Coverage for src/wiktextract/extractor/en/form

1# Code for parsing linguistic form descriptions and tags for word senses

2# (both the word entry head - initial part and parenthesized parts -

3# and tags at the beginning of word senses)

7import functools

8import re

9import unicodedata

10from typing import (

11 Any,

12 Literal,

13 Optional,

14 Sequence,

15 Union,

16)

18import Levenshtein

19from nltk import TweetTokenizer # type:ignore[import-untyped]

20from wikitextprocessor.parser import WikiNode

22from ...datautils import data_append, data_extend, split_at_comma_semi

23from ...page import extract_links_from_node

24from ...tags import (

25 alt_of_tags,

26 form_of_tags,

27 head_final_bantu_langs,

28 head_final_bantu_map,

29 head_final_numeric_langs,

30 head_final_other_langs,

31 head_final_other_map,

32 head_final_semitic_langs,

33 head_final_semitic_map,

34 uppercase_tags,

35 valid_tags,

36 xlat_descs_map,

37 xlat_head_map,

38 xlat_tags_map,

39)

40from ...topics import topic_generalize_map, valid_topics

41from ...wxr_context import WiktextractContext

42from .english_words import (

43 english_words,

44 not_english_words,

45 potentially_english_words,

46)

47from .form_descriptions_known_firsts import known_firsts

48from .taxondata import known_species

49from .type_utils import (

50 AltOf,

51 FormData,

52 LinkageData,

53 SenseData,

54 SoundData,

55 TemplateData,

56 TranslationData,

57 WordData,

58)

60# Tokenizer for classify_desc()

61tokenizer = TweetTokenizer()

63# These are ignored as the value of a related form in form head.

64IGNORED_RELATED: set[str] = set(

65 [

66 "-",

67 "־",

68 "᠆",

69 "‐",

70 "‑",

71 "‒",

72 "–",

73 "—",

74 "―",

75 "−",

76 "⸺",

77 "⸻",

78 "﹘",

79 "﹣",

80 "－",

81 "?",

82 "(none)",

83 ]

84)

87# First words of unicodedata.name() that indicate scripts that cannot be

88# accepted in romanizations or english (i.e., should be considered "other"

89# in classify_desc()).

90non_latin_scripts: list[str] = [

91 "ADLAM",

92 "ARABIC",

93 "ARABIC-INDIC",

94 "ARMENIAN",

95 "BALINESE",

96 "BENGALI",

97 "BRAHMI",

98 "BRAILLE",

99 "CANADIAN",

100 "CHAKMA",

101 "CHAM",

102 "CHEROKEE",

103 "CJK",

104 "COPTIC",

105 "COUNTING ROD",

106 "CUNEIFORM",

107 "CYRILLIC",

108 "DOUBLE-STRUCK",

109 "EGYPTIAN",

110 "ETHIOPIC",

111 "EXTENDED ARABIC-INDIC",

112 "GEORGIAN",

113 "GLAGOLITIC",

114 "GOTHIC",

115 "GREEK",

116 "GUJARATI",

117 "GURMUKHI",

118 "HANGUL",

119 "HANIFI ROHINGYA",

120 "HEBREW",

121 "HIRAGANA",

122 "JAVANESE",

123 "KANNADA",

124 "KATAKANA",

125 "KAYAH LI",

126 "KHMER",

127 "KHUDAWADI",

128 "LAO",

129 "LEPCHA",

130 "LIMBU",

131 "MALAYALAM",

132 "MEETEI",

133 "MYANMAR",

134 "NEW TAI LUE",

135 "NKO",

136 "OL CHIKI",

137 "OLD PERSIAN",

138 "OLD SOUTH ARABIAN",

139 "ORIYA",

140 "OSMANYA",

141 "PHOENICIAN",

142 "SAURASHTRA",

143 "SHARADA",

144 "SINHALA",

145 "SUNDANESE",

146 "SYLOTI",

147 "TAI THAM",

148 "TAKRI",

149 "TAMIL",

150 "TELUGU",

151 "THAANA",

152 "THAI",

153 "TIBETAN",

154 "TIFINAGH",

155 "TIRHUTA",

156 "UGARITIC",

157 "WARANG CITI",

158 "YI",

159]

160non_latin_scripts_re = re.compile(

161 r"(" + r"|".join(re.escape(x) for x in non_latin_scripts) + r")\b"

162)

163

164# Sanity check xlat_head_map values

165for k, v in xlat_head_map.items():

166 if v.startswith("?"):

167 v = v[1:]

168 for tag in v.split():

169 if tag not in valid_tags: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 print(

171 "WARNING: xlat_head_map[{}] contains"

172 " unrecognized tag {}".format(k, tag)

173 )

174

175# Regexp for finding nested translations from translation items (these are

176# used in, e.g., year/English/Translations/Arabic). This is actually used

177# in page.py.

178nested_translations_re = re.compile(

179 r"\s+$(({}): ([^()]|\([^()]+$)+)\)".format(

180 "|".join(

181 re.escape(x.removeprefix("?"))

182 for x in sorted(xlat_head_map.values(), key=len, reverse=True)

183 if x and not x.startswith("class-")

184 )

185 )

186)

187

188# Regexp that matches head tag specifiers. Used to match tags from end of

189# translations and linkages

190head_final_re_text = r"( -)?( ({}))+".format(

191 "|".join(

192 re.escape(x)

193 for x in

194 # The sort is to put longer ones first, preferring them in

195 # the regexp match

196 sorted(xlat_head_map.keys(), key=len, reverse=True)

197 )

198)

199head_final_re = re.compile(head_final_re_text + r"$")

200

201# Regexp used to match head tag specifiers at end of a form for certain

202# Bantu languages (particularly Swahili and similar languages).

203head_final_bantu_re_text = r" ({})".format(

204 "|".join(re.escape(x) for x in head_final_bantu_map.keys())

205)

206head_final_bantu_re = re.compile(head_final_bantu_re_text + "$")

207

208# Regexp used to match head tag specifiers at end of a form for certain

209# Semitic languages (particularly Arabic and similar languages).

210head_final_semitic_re_text = r" ({})".format(

211 "|".join(re.escape(x) for x in head_final_semitic_map.keys())

212)

213head_final_semitic_re = re.compile(head_final_semitic_re_text + "$")

214

215# Regexp used to match head tag specifiers at end of a form for certain

216# other languages (e.g., Lithuanian, Finnish, French).

217head_final_other_re_text = r" ({})".format(

218 "|".join(re.escape(x) for x in head_final_other_map.keys())

219)

220head_final_other_re = re.compile(head_final_other_re_text + "$")

221

222# Regexp for splitting heads. See parse_word_head().

223head_split_re_text_part_1 = (

224 "("

225 + head_final_re_text

226 + "|"

227 + head_final_bantu_re_text

228 + "|"

229 + head_final_semitic_re_text

230 + "|"

231 + head_final_other_re_text

232)

233

234head_split_re_text = head_split_re_text_part_1 + ")?( or |[,;]+| *$)"

235

236head_split_re_text_no_semicolon = head_split_re_text_part_1 + ")?( or |,+| *$)"

237

238head_split_re = re.compile(head_split_re_text)

239head_split_no_semicolon_re = re.compile(head_split_re_text_no_semicolon)

240

241head_split_re_parens = 0

242for m in re.finditer(r"(^|[^\\])[(]+", head_split_re_text):

243 head_split_re_parens += m.group(0).count("(")

244

245# Parenthesized parts that are ignored in translations

246tr_ignored_parens: set[str] = set(

247 [

248 "please verify",

249 "(please verify)",

250 "transliteration needed",

251 "(transliteration needed)",

252 "in words with back vowel harmony",

253 "(in words with back vowel harmony)",

254 "in words with front vowel harmony",

255 "(in words with front vowel harmony)",

256 "see below",

257 "see usage notes below",

258 ]

259)

260tr_ignored_parens_re = re.compile(

261 r"^("

262 + "|".join(re.escape(x) for x in tr_ignored_parens)

263 + ")$"

264 + r"|^(Can we clean up|Can we verify|for other meanings see "

265 r"lit\. )"

266)

267

268# Translations that are ignored

269ignored_translations: set[str] = set(

270 [

271 "[script needed]",

272 "please add this translation if you can",

273 ]

274)

275

276# Put english text into the "note" field in a translation if it contains one

277# of these words

278tr_note_re = re.compile(

288 r"indicative|progressive|conditional|potential|"

292 r"periphrastically|no equivalent|not used|not always used|"

295 r"may be replaced|stricter sense|for nonhumans|"

297 r"not restricted to|pertaining to|or optionally with|are optional|"

298 r"in conjunction with|in compounds|depending on the relationship|"

299 r"person addressed|one person|multiple persons|may be replaced with|"

300 r"optionally completed with|in the phrase|in response to|"

302 r"with verb|with uncountable|with the objects|with stative|"

303 r"can be replaced by|often after|used before|used after|"

305 r"short form|shortening of|shortened form|initialism of|"

307 r"previously mentioned|uncountable noun|countable noun|"

308 r"countable nouns|uncountable nouns|"

309 r"with predicative|with -|with imperfect|with a negated|"

311 r'"|'

312 r"general term|after a vowel|before a vowel|"

313 r"form|regular|irregular|alternative)"

314 r")($|[) ])|^("

315 # Following are only matched at the beginning of the string

316 r"pl|pl\.|see:|pl:|sg:|plurals:|e\.g\.|e\.g\.:"

317 r"|e\.g\.,|cf\.|compare|such as|"

318 r"see|only|often|usually|used|usage:|of|not|in|compare|usu\.|"

321 r"\+|with) "

322)

323# \b does not work at the end???

324

325# Related forms matching this regexp will be considered suspicious if the

326# page title does not also match one of these.

327suspicious_related_re = re.compile(

328 r"(^| )(f|m|n|c|or|pl|sg|inan|anim|pers|anml|impf|pf|vir|nvir)( |$)"

329 r"|[][:=<>&#*|]"

330 r"| \d+$"

331)

332

333# Word forms (head forms, translations, etc) that will be considered ok and

334# silently accepted even if they would otherwise trigger a suspicious

335# form warning.

336ok_suspicious_forms: set[str] = set(

337 [

338 "but en or", # "golden goal"/English/Tr/French

339 "cœur en or", # "heart of gold"/Eng/Tr/French

340 "en or", # golden/Eng/Tr/French

341 "men du", # jet/Etym2/Noun/Tr/Cornish

342 "parachute en or", # "golden parachute"/Eng/Tr/French

343 "vieil or", # "old gold"/Eng/Tr/French

344 # "all that glitters is not gold"/Eng/Tr/French

345 "tout ce qui brille n’est pas or",

346 "μη αποκλειστικό or", # inclusive or/Eng/Tr/Greek

347 "period or full stop",

348 ]

349)

350

351

352# Replacements to be done in classify_desc before tokenizing. This is a

353# workaround for shortcomings in TweetTokenizer.

354tokenizer_fixup_map = {

355 r"a.m.": "AM",

356 r"p.m.": "PM",

357}

358tokenizer_fixup_re = re.compile(

359 r"\b("

360 + "|".join(

361 re.escape(x)

362 for x in sorted(

363 tokenizer_fixup_map.keys(), key=lambda x: len(x), reverse=True

364 )

365 )

366 + r")"

367)

368

369# Unknown tags starting with these words will be silently ignored.

370ignored_unknown_starts: set[str] = set(

371 [

372 "originally",

373 "e.g.",

374 "c.f.",

375 "supplanted by",

376 "supplied by",

377 ]

378)

379

380ignored_unknown_starts_re = re.compile(

381 r"^("

382 + "|".join(

383 re.escape(x)

384 for x in sorted(ignored_unknown_starts, key=lambda x: -len(x))

385 )

386 + ") "

387)

388

389# If an unknown sequence starts with one of these, it will continue as an

390# unknown sequence until the end, unless it turns out to have a replacement.

391allowed_unknown_starts: set[str] = set(

392 [

393 "Relating",

394 "accompanied",

395 "added",

396 "after",

397 "answering",

398 "as",

399 "based",

400 "before",

401 "conjugated",

402 "conjunction",

403 "construed",

404 "especially",

405 "expression:",

406 "figurative:",

407 "followed",

408 "for",

409 "forms",

410 "from",

411 "governs",

412 "in",

413 "indicating",

414 "modifying",

415 "normally",

416 "not",

417 "of",

418 "preceding",

419 "prefixed",

420 "referring",

421 "relating",

422 "revived",

423 "said",

424 "since",

425 "takes",

426 "used",

427 "with",

428 "With",

429 "without",

430 ]

431)

432# Allow the ignored unknown starts without complaining

433allowed_unknown_starts.update(ignored_unknown_starts)

434

435# Full unknown tags that will be ignored in decode_tags()

436# XXX this is unused, ask Tatu where the contents is now

437ignored_unknown_tags: set[str] = set([])

438

439# Head endings that are mapped to tags

440head_end_map = {

441 " 1st conj.": "conjugation-1",

442 " 2nd conj.": "conjugation-2",

443 " 3rd conj.": "conjugation-3",

444 " 4th conj.": "conjugation-4",

445 " 5th conj.": "conjugation-5",

446 " 6th conj.": "conjugation-6",

447 " 7th conj.": "conjugation-7",

448}

449head_end_re = re.compile(

450 r"(" + "|".join(re.escape(x) for x in head_end_map.keys()) + r")$"

451)

452

453# Transitivity and inflection class labels at the end of Japanese heads,

454# e.g. 走る "intransitive godan", 給ふ "transitive or intransitive ^†yodan"

455ja_head_end_re = re.compile(

457 r"|irregular|uninflectable"

458 r"|-i|-na|-nari|-tari|-ku|-shiku|-ka|-ri|-se|-ko|-n)(?: or)?$"

459)

460

461

462# Dictionary of language-specific parenthesized head part starts that

463# either introduce new tags or modify previous tags. The value for each

464# language is a dictionary that maps the first word of the head part to

465# (rem_tags, add_tags), where ``rem_tags`` can be True to remove all previous

466# tags or a space-separated string of tags to remove, and ``add_tags`` should

467# be a string of tags to add.

468lang_specific_head_map: dict[

469 str, dict[str, Union[tuple[str, str], tuple[Literal[True], str]]]

470] = {

471 "Danish": {

472 # prefix: (rem_tags space separate string/True, add_tags s-sep str)

473 "c": ("neuter", "common-gender"),

474 "n": ("common-gender", "neuter"),

475 "pl": ("singular neuter common-gender", "plural"),

476 "sg": ("plural neuter common-gender", "singular"),

477 },

478}

479

480

481# Regular expression used to strip additional stuff from the end of alt_of and

482# form_of.

483alt_of_form_of_clean_re = re.compile(

484 r"(?s)("

485 + "|".join(

486 [

487 r":",

488 r'[“"]',

489 r";",

490 r" \(",

491 r" - ",

492 r" ־ ",

493 r" ᠆ ",

494 r" ‐ ",

495 r" ‑ ",

496 r" ‒ ",

497 r" – ",

498 r" — ",

499 r" ― ",

500 r" − ",

501 r" ⸺ ",

502 r" ⸻ ",

503 r" ﹘ ",

504 r" ﹣ ",

505 r" － ",

506 r" \+ ",

507 r" \(with ",

508 r" with -ra/-re",

509 r"\. Used ",

510 r"\. Also ",

511 r"\. Since ",

512 r"\. A ",

513 r"\.\. A ",

514 r"\. An ",

515 r"\.\. An ",

516 r"\. an ",

517 r"\. The ",

518 r"\. Spanish ",

519 r"\. Language ",

520 r"\. former name of ",

521 r"\. AIM",

522 r"\. OT",

523 r"\. Not ",

524 r"\. Now ",

525 r"\. Nowadays ",

526 r"\. Early ",

527 r"\. ASEAN",

528 r"\. UN",

529 r"\. IMF",

530 r"\. WHO",

531 r"\. WIPO",

532 r"\. AC",

533 r"\. DC",

534 r"\. DNA",

535 r"\. RNA",

536 r"\. SOB",

537 r"\. IMO",

538 r"\. Behavior",

539 r"\. Income ",

540 r"\. More ",

541 r"\. Most ",

542 r"\. Only ",

543 r"\. Also ",

544 r"\. From ",

545 r"\. Of ",

546 r"\.\. Of ",

547 r"\. To ",

548 r"\. For ",

549 r"\. If ",

550 r"\. Praenominal ",

551 r"\. This ",

552 r"\. Replaced ",

553 r"\. CHCS is the ",

554 r"\. Equivalent ",

555 r"\. Initialism ",

556 r"\. Note ",

557 r"\. Alternative ",

558 r"\. Compare ",

559 r"\. Cf\. ",

560 r"\. Comparable ",

561 r"\. Involves ",

562 r"\. Sometimes ",

563 r"\. Commonly ",

564 r"\. Often ",

565 r"\. Typically ",

566 r"\. Possibly ",

567 r"\. Although ",

568 r"\. Rare ",

569 r"\. Instead ",

570 r"\. Integrated ",

571 r"\. Distinguished ",

572 r"\. Given ",

573 r"\. Found ",

574 r"\. Was ",

575 r"\. In ",

576 r"\. It ",

577 r"\.\. It ",

578 r"\. One ",

579 r"\. Any ",

580 r"\. They ",

581 r"\. Members ",

582 r"\. Each ",

583 r"\. Original ",

584 r"\. Especially ",

585 r"\. Usually ",

586 r"\. Known ",

587 r"\.\. Known ",

588 r"\. See ",

589 r"\. see ",

590 r"\. target was not ",

591 r"\. Popular ",

592 r"\. Pedantic ",

593 r"\. Positive ",

594 r"\. Society ",

595 r"\. Plan ",

596 r"\. Environmentally ",

597 r"\. Affording ",

598 r"\. Encompasses ",

599 r"\. Expresses ",

600 r"\. Indicates ",

601 r"\. Text ",

602 r"\. Large ",

603 r"\. Sub-sorting ",

604 r"\. Sax",

605 r"\. First-person ",

606 r"\. Second-person ",

607 r"\. Third-person ",

608 r"\. 1st ",

609 r"\. 2nd ",

610 r"\. 3rd ",

611 r"\. Term ",

612 r"\. Northeastern ",

613 r"\. Northwestern ",

614 r"\. Southeast ",

615 r"\. Egyptian ",

616 r"\. English ",

617 r"\. Cape Province was split into ",

618 r"\. Pañcat",

619 r"\. of the ",

620 r"\. is ",

621 r"\. after ",

622 r"\. or ",

623 r"\. chromed",

624 r"\. percussion",

625 r"\. with his ",

626 r"\. a\.k\.a\. ",

627 r"\. comparative form ",

628 r"\. singular ",

629 r"\. plural ",

630 r"\. present ",

631 r"\. his ",

632 r"\. her ",

633 r"\. equivalent ",

634 r"\. measuring ",

635 r"\. used in ",

636 r"\. cutely ",

637 r"\. Protects",

638 r'\. "',

639 r"\.^",

640 r"\. \+ ",

641 r"\., ",

642 r". — ",

643 r", a ",

644 r", an ",

645 r", the ",

646 r", obsolete ",

647 r", possessed", # 'd/English

648 r", imitating", # 1/English

649 r", derived from",

650 r", called ",

651 r", especially ",

652 r", slang for ",

653 r", used to", # c/o /English

654 r", commonly", # b/w /English

655 r" corresponding to ",

656 r" equivalent to ",

657 r" popularized by ",

658 r" denoting ",

659 r" in its various senses\.",

660 r" used by ",

661 r" but not for ",

662 r" since ",

663 r" i\.e\. ",

664 r" i\. e\. ",

665 r" e\.g\. ",

666 r" eg\. ",

667 r" etc\. ",

668 r"\[http",

669 r" — used as ",

670 r" by K\. Forsyth ",

671 r" by J\. R\. Allen ",

672 r" by S\. Ferguson ",

673 r" by G\. Donaldson ",

674 r" May refer to ",

675 r" An area or region ",

676 ]

677 )

678 + r").*$"

679)

680

681

682class ValidNode:

683 """Node in the valid_sequences tree. Each node is part of a chain

684 or chains that form sequences built out of keys in key->tags

685 maps like xlat_tags, etc. The ValidNode's 'word' is the key

686 by which it is refered to in the root dict or a `children` dict,

687 `end` marks that the node is the end-terminus of a sequence (but

688 it can still continue if the sequence is shared by the start of

689 other sequences: "nominative$" and "nominative plural$" for example),

690 `tags` and `topics` are the dicts containing tag and topic strings

691 for terminal nodes (end==True)."""

692

693 __slots__ = (

694 "end",

695 "tags",

696 "topics",

697 "children",

698 )

699

700 def __init__(

701 self,

702 end=False,

703 tags: Optional[list[str]] = None,

704 topics: Optional[list[str]] = None,

705 children: Optional[dict[str, "ValidNode"]] = None,

706 ) -> None:

707 self.end = end

708 self.tags: list[str] = tags or []

709 self.topics: list[str] = topics or []

710 self.children: dict[str, "ValidNode"] = children or {}

711

712

713def add_to_valid_tree(tree: ValidNode, desc: str, v: Optional[str]) -> None:

714 """Helper function for building trees of valid tags/sequences during

715 initialization."""

716 assert isinstance(tree, ValidNode)

717 assert isinstance(desc, str)

718 assert v is None or isinstance(v, str)

719 node = tree

720

721 # Build the tree structure: each node has children nodes

722 # whose names are denoted by their dict key.

723 for w in desc.split(" "):

724 if w in node.children:

725 node = node.children[w]

726 else:

727 new_node = ValidNode()

728 node.children[w] = new_node

729 node = new_node

730 if not node.end:

731 node.end = True

732 if not v:

733 return None # Terminate early because there are no tags

734

735 tagslist = []

736 topicslist = []

737 for vv in v.split():

738 if vv in valid_tags:

739 tagslist.append(vv)

740 elif vv in valid_topics: 740 ↛ 743line 740 didn't jump to line 743 because the condition on line 740 was always true

741 topicslist.append(vv)

742 else:

743 print(

744 "WARNING: tag/topic {!r} maps to unknown {!r}".format(desc, vv)

745 )

746 topics = " ".join(topicslist)

747 tags = " ".join(tagslist)

748 # Changed to "_tags" and "_topics" to avoid possible key-collisions.

749 if topics:

750 node.topics.extend([topics])

751 if tags:

752 node.tags.extend([tags])

753

754

755def add_to_valid_tree1(

756 tree: ValidNode,

757 k: str,

758 v: Union[list[str], tuple[str, ...], str],

759 valid_values: Union[set[str], dict[str, Any]],

760) -> list[str]:

761 assert isinstance(tree, ValidNode)

762 assert isinstance(k, str)

763 assert v is None or isinstance(v, (list, tuple, str))

764 assert isinstance(valid_values, (set, dict))

765 if not v: 765 ↛ 766line 765 didn't jump to line 766 because the condition on line 765 was never true

766 add_to_valid_tree(valid_sequences, k, None)

767 return []

768 elif isinstance(v, str):

769 v = [v]

770 q = []

771 for vv in v:

772 assert isinstance(vv, str)

773 add_to_valid_tree(valid_sequences, k, vv)

774 vvs = vv.split()

775 for x in vvs:

776 q.append(x)

777 # return each individual tag

778 return q

779

780

781def add_to_valid_tree_mapping(

782 tree: ValidNode,

783 mapping: Union[dict[str, Union[list[str], str]], dict[str, str]],

784 valid_values: Union[set[str], dict[str, Any]],

785 recurse: bool,

786) -> None:

787 assert isinstance(tree, ValidNode)

788 assert isinstance(mapping, dict)

789 assert isinstance(valid_values, (set, dict))

790 assert recurse in (True, False)

791 for k, v in mapping.items():

792 assert isinstance(k, str)

793 assert isinstance(v, (list, str))

794 if isinstance(v, str):

795 q = add_to_valid_tree1(tree, k, [v], valid_values)

796 else:

797 q = add_to_valid_tree1(tree, k, v, valid_values)

798 if recurse:

799 visited = set()

800 while q:

801 v = q.pop()

802 if v in visited:

803 continue

804 visited.add(v)

805 if v not in mapping:

806 continue

807 vv = mapping[v]

808 qq = add_to_valid_tree1(tree, k, vv, valid_values)

809 q.extend(qq)

810

811

812# Tree of sequences considered to be tags (includes sequences that are

813# mapped to something that becomes one or more valid tags)

814valid_sequences = ValidNode()

815sequences_with_slashes: set[str] = set()

816for tag in valid_tags:

817 # The basic tags used in our tag system; some are a bit weird, but easier

818 # to implement this with 'false' positives than filter out stuff no one else

819 # uses.

820 if "/" in tag:

821 sequences_with_slashes.add(tag)

822 add_to_valid_tree(valid_sequences, tag, tag)

823for tag in uppercase_tags:

824 hyphenated = re.sub(r"\s+", "-", tag)

825 if "/" in tag:

826 sequences_with_slashes.add(tag)

827 add_to_valid_tree(valid_sequences, tag, hyphenated)

828

829# xlat_tags_map!

830add_to_valid_tree_mapping(valid_sequences, xlat_tags_map, valid_tags, False)

831for k in xlat_tags_map:

832 if "/" in k:

833 sequences_with_slashes.add(k)

834# Add topics to the same table, with all generalized topics also added

835for topic in valid_topics:

836 assert " " not in topic

837 if "/" in topic: 837 ↛ 838line 837 didn't jump to line 838 because the condition on line 837 was never true

838 sequences_with_slashes.add(topic)

839 add_to_valid_tree(valid_sequences, topic, topic)

840# Let each original topic value stand alone. These are not generally on

841# valid_topics. We add the original topics with spaces replaced by hyphens.

842for topic in topic_generalize_map.keys():

843 hyphenated = re.sub(r"\s+", "-", topic)

844 if "/" in topic: 844 ↛ 845line 844 didn't jump to line 845 because the condition on line 844 was never true

845 sequences_with_slashes.add(topic)

846 add_to_valid_tree(valid_sequences, topic, hyphenated)

847# Add canonicalized/generalized topic values

848add_to_valid_tree_mapping(

849 valid_sequences, topic_generalize_map, valid_topics, True

850)

851

852# Regex used to divide a decode candidate into parts that shouldn't

853# have their slashes turned into spaces

854slashes_re = re.compile(

855 r"(" + "|".join((re.escape(s) for s in sequences_with_slashes)) + r")"

856)

857

858# Regexp used to find "words" from word heads and linguistic descriptions

859word_pattern = (

860 r"[^ ,;()\u200e]+|"

861 r"$[^ ,;()\u200e]+$[^ ,;()\u200e]+|"

862 r"[\u2800-\u28ff]|" # Braille characters

863 r"$([^()]|\([^()]*$)*\)"

864)

865

866word_re_global = re.compile(word_pattern)

867

868

869def distw(titleparts: Sequence[str], word: str) -> float:

870 """Computes how distinct ``word`` is from the most similar word in

871 ``titleparts``. Returns 1 if words completely distinct, 0 if

872 identical, or otherwise something in between."""

873 assert isinstance(titleparts, (list, tuple))

874 assert isinstance(word, str)

875 w = min(

876 Levenshtein.distance(word, tw) / max(len(tw), len(word))

877 for tw in titleparts

878 )

879 return w

880

881

882def map_with(

883 ht: dict[str, str | list[str]] | dict[str, str],

884 lst: Sequence[str],

885) -> list[str]:

886 """Takes alternatives from ``lst``, maps them using ``ht`` to zero or

887 more alternatives each, and returns a combined list of alternatives."""

888 assert isinstance(ht, dict)

889 assert isinstance(lst, (list, tuple))

890 ret = []

891 for x in lst:

892 assert isinstance(x, str)

893 x = x.strip()

894 x = ht.get(x, x)

895 if isinstance(x, str): 895 ↛ 898line 895 didn't jump to line 898 because the condition on line 895 was always true

896 if x: 896 ↛ 891line 896 didn't jump to line 891 because the condition on line 896 was always true

897 ret.append(x)

898 elif isinstance(x, (list, tuple)):

899 ret.extend(x)

900 else:

901 raise RuntimeError("map_with unexpected value: {!r}".format(x))

902 return ret

903

904

905TagList = list[str]

906PosPathStep = tuple[int, TagList, TagList]

907

908

909def check_unknown(

910 from_i: int,

911 to_i: int,

912 i: int,

913 wordlst: Sequence[str],

914 allow_any: bool,

915 no_unknown_starts: bool,

916) -> list[PosPathStep]:

917 """Check if the current section from_i->to_i is actually unknown

918 or if it needs some special handling. We already presupposed that

919 this is UNKNOWN; this is just called to see what *kind* of UNKNOWN."""

920 assert isinstance(to_i, int)

921 assert isinstance(from_i, int)

922 assert isinstance(i, int)

923 # Adds unknown tag if needed. Returns new last_i

924 # print("check_unknown to_i={} from_i={} i={}"

925 # .format(to_i, from_i, i))

926 if from_i >= to_i:

927 return []

928 words = wordlst[from_i:to_i]

929 tag = " ".join(words)

930 assert tag

931 # print(f"{tag=}")

932 if re.match(ignored_unknown_starts_re, tag):

933 # Tags with this start are to be ignored

934 return [(from_i, ["UNKNOWN"], [])]

935 if tag in ignored_unknown_tags: 935 ↛ 936line 935 didn't jump to line 936 because the condition on line 935 was never true

936 return [] # One of the tags listed as to be ignored

937 if tag in ("and", "or"):

938 return []

939 if (

940 not allow_any

941 and not words[0].startswith("~")

942 and (

943 no_unknown_starts

944 or words[0] not in allowed_unknown_starts

945 or len(words) <= 1

946 )

947 ):

948 # print("ERR allow_any={} words={}"

949 # .format(allow_any, words))

950 return [

951 (from_i, ["UNKNOWN"], ["error-unknown-tag"])

952 ] # Add ``tag`` here to include

953 else:

954 return [(from_i, ["UNKNOWN"], [tag])]

955

956

957def add_new1(

958 node: ValidNode,

959 i: int,

960 start_i: int,

961 last_i: int,

962 new_paths: list[list[PosPathStep]],

963 new_nodes: list[tuple[ValidNode, int, int]],

964 pos_paths: list[list[list[PosPathStep]]],

965 wordlst: list[str],

966 allow_any: bool,

967 no_unknown_starts: bool,

968 max_last_i: int,

969) -> int:

970 assert isinstance(new_paths, list)

971 # print("add_new: start_i={} last_i={}".format(start_i, last_i))

972 # print("$ {} last_i={} start_i={}"

973 # .format(w, last_i, start_i))

974 max_last_i = max(max_last_i, last_i) # if last_i has grown

975 if (node, start_i, last_i) not in new_nodes:

976 new_nodes.append((node, start_i, last_i))

977 if node.end:

978 # We can see a terminal point in the search tree.

979 u = check_unknown(

980 last_i, start_i, i, wordlst, allow_any, no_unknown_starts

981 )

982 # Create new paths candidates based on different past possible

983 # paths; pos_path[last_i] contains possible paths, so add this

984 # new one at the beginning(?)

985 # The list comprehension inside the parens generates an iterable

986 # of lists, so this is .extend( [(last_i...)], [(last_i...)], ... )

987 # XXX: this is becoming impossible to annotate, nodes might

988 # need to become classed objects and not just dicts, or at least

989 # a TypedDict with a "children" node

990 new_paths.extend(

991 [(last_i, node.tags, node.topics)] + u + x

992 for x in pos_paths[last_i]

993 )

994 max_last_i = i + 1

995 return max_last_i

996

997

998@functools.lru_cache(maxsize=65536)

999def decode_tags(

1000 src: str,

1001 allow_any=False,

1002 no_unknown_starts=False,

1003) -> tuple[list[tuple[str, ...]], list[str]]:

1004 tagsets, topics = decode_tags1(src, allow_any, no_unknown_starts)

1005 # print(f"decode_tags: {src=}, {tagsets=}")

1006

1007 # Insert retry-code here that modifies the text source

1008 if (

1009 any(s.startswith("error-") for tagset in tagsets for s in tagset)

1010 # I hate Python's *nested* list comprehension syntax ^

1011 or any(s.startswith("error-") for s in topics)

1012 ):

1013 new_tagsets: list[tuple[str, ...]] = []

1014 new_topics: list[str] = []

1015

1016 if "/" in src:

1017 # slashes_re contains valid key entries with slashes; we're going

1018 # to skip them by splitting the string and skipping handling every

1019 # second entry, which contains the splitting group like "masculine/

1020 # feminine" style keys.

1021 split_parts = re.split(slashes_re, src)

1022 new_parts: list[str] = []

1023 if len(split_parts) > 1:

1024 for i, s in enumerate(split_parts):

1025 if i % 2 == 0:

1026 new_parts.append(s.replace("/", " "))

1027 else:

1028 new_parts.append(s)

1029 new_src = "".join(new_parts)

1030 else:

1031 new_src = src

1032 new_tagsets, new_topics = decode_tags1(

1033 new_src, allow_any, no_unknown_starts

1034 )

1035 elif " or " in src or " and " in src:

1036 # Annoying kludge.

1037 new_src = src.replace(" and ", " ")

1038 new_src = new_src.replace(" or ", " ")

1039 new_tagsets, new_topics = decode_tags1(

1040 new_src, allow_any, no_unknown_starts

1041 )

1042 # print(f"{new_tagsets=}")

1043

1044 if new_tagsets or new_topics:

1045 old_errors = sum(

1046 1 for tagset in tagsets for s in tagset if s.startswith("error")

1047 )

1048 old_errors += sum(1 for s in topics if s.startswith("error"))

1049 new_errors = sum(

1050 1

1051 for new_tagset in new_tagsets

1052 for s in new_tagset

1053 if s.startswith("error")

1054 )

1055 new_errors += sum(1 for s in new_topics if s.startswith("error"))

1056

1057 if new_errors <= old_errors: 1057 ↛ 1060line 1057 didn't jump to line 1060 because the condition on line 1057 was always true

1058 return new_tagsets, new_topics

1059

1060 return tagsets, topics

1061

1062

1063def decode_tags1(

1064 src: str,

1065 allow_any=False,

1066 no_unknown_starts=False,

1067) -> tuple[list[tuple[str, ...]], list[str]]:

1068 """Decodes tags, doing some canonicalizations. This returns a list of

1069 lists of tags and a list of topics."""

1070 assert isinstance(src, str)

1071

1072 # print("decode_tags: src={!r}".format(src))

1073

1074 pos_paths: list[list[list[PosPathStep]]] = [[[]]]

1075 wordlst: list[str] = []

1076 max_last_i = 0 # pre-initialized here so that it can be used as a ref

1077

1078 add_new = functools.partial(

1079 add_new1, # pre-set parameters and references for function

1080 pos_paths=pos_paths,

1081 wordlst=wordlst,

1082 allow_any=allow_any,

1083 no_unknown_starts=no_unknown_starts,

1084 max_last_i=max_last_i,

1085 )

1086 # First split the tags at commas and semicolons. Their significance is that

1087 # a multi-word sequence cannot continue across them.

1088 parts = split_at_comma_semi(src, extra=[";", ":"])

1089

1090 for part in parts:

1091 max_last_i = len(wordlst) # "how far have we gone?"

1092 lst1 = part.split()

1093 if not lst1:

1094 continue

1095 wordlst.extend(lst1)

1096 cur_nodes: list[tuple[ValidNode, int, int]] = [] # Currently seen

1097 for w in lst1:

1098 i = len(pos_paths) - 1

1099 new_nodes: list[tuple[ValidNode, int, int]] = []

1100 # replacement nodes for next loop

1101 new_paths: list[list[PosPathStep]] = []

1102 # print("ITER i={} w={} max_last_i={} wordlst={}"

1103 # .format(i, w, max_last_i, wordlst))

1104 node: ValidNode

1105 start_i: int

1106 last_i: int

1107 for node, start_i, last_i in cur_nodes:

1108 # ValidNodes are part of a search tree that checks if a

1109 # phrase is found in xlat_tags_map and other text->tags dicts.

1110 if w in node.children:

1111 # the phrase continues down the tree

1112 # print("INC", w)

1113 max_last_i = add_new(

1114 node.children[w],

1115 i,

1116 start_i,

1117 last_i,

1118 new_paths,

1119 new_nodes,

1120 )

1121 if node.end:

1122 # we've hit an end point, the tags and topics have already

1123 # been gathered at some point, don't do anything with the

1124 # old stuff

1125 if w in valid_sequences.children:

1126 # This starts a *new* possible section

1127 max_last_i = add_new(

1128 valid_sequences.children[w], # root->

1129 i,

1130 i,

1131 i,

1132 new_paths,

1133 new_nodes,

1134 )

1135 if w not in node.children and not node.end:

1136 # print("w not in node and $: i={} last_i={} wordlst={}"

1137 # .format(i, last_i, wordlst))

1138 # If i == last_i == 0, for example (beginning)

1139 if (

1140 i == last_i

1141 or no_unknown_starts

1142 or wordlst[last_i] not in allowed_unknown_starts

1143 ):

1144 # print("NEW", w)

1145 if w in valid_sequences.children:

1146 # Start new sequences here

1147 max_last_i = add_new(

1148 valid_sequences.children[w],

1149 i,

1150 i,

1151 last_i,

1152 new_paths,

1153 new_nodes,

1154 )

1155 if not new_nodes:

1156 # This is run at the start when i == max_last_i == 0,

1157 # which is what populates the first node in new_nodes.

1158 # Some initial words cause the rest to be interpreted as unknown

1159 # print("not new nodes: i={} last_i={} wordlst={}"

1160 # .format(i, max_last_i, wordlst))

1161 if (

1162 i == max_last_i

1163 or no_unknown_starts

1164 or wordlst[max_last_i] not in allowed_unknown_starts

1165 ):

1166 # print("RECOVER w={} i={} max_last_i={} wordlst={}"

1167 # .format(w, i, max_last_i, wordlst))

1168 if w in valid_sequences.children:

1169 max_last_i = add_new(

1170 # new sequence from root

1171 valid_sequences.children[w],

1172 i,

1173 i,

1174 max_last_i,

1175 new_paths,

1176 new_nodes,

1177 )

1178 cur_nodes = new_nodes # Completely replace nodes!

1179 # 2023-08-18, fix to improve performance

1180 # Decode tags does a big search of the best-shortest matching

1181 # sequences of tags, but the original algorithm didn't have

1182 # any culling happen during operation, so in a case with

1183 # a lot of tags (for example, big blocks of text inserted

1184 # somewhere by mistake that is processed by decode_tags),

1185 # it would lead to exponential growth of new_paths contents.

1186 # This culling, using the same weighting algorithm code as

1187 # in the original is just applied to new_paths before it is

1188 # added to pos_paths. Basically it's "take the 10 best paths".

1189 # This *can* cause bugs if it gets stuck in a local minimum

1190 # or something, but this whole process is one-dimensional

1191 # and not that complex, so hopefully it works out...

1192 pw = []

1193 path: list[PosPathStep]

1194 for path in new_paths:

1195 weight = len(path)

1196 if any(x[1] == ["UNKNOWN"] for x in path):

1197 weight += 100 # Penalize unknown paths

1198 pw.append((weight, path))

1199 new_paths = [weightpath[1] for weightpath in sorted(pw)[:10]]

1200 pos_paths.append(new_paths)

1201

1202 # print("END max_last_i={} len(wordlst)={} len(pos_paths)={}"

1203 # .format(max_last_i, len(wordlst), len(pos_paths)))

1204

1205 if cur_nodes:

1206 # print("END HAVE_NODES")

1207 for node, start_i, last_i in cur_nodes:

1208 if node.end:

1209 # print("$ END start_i={} last_i={}"

1210 # .format(start_i, last_i))

1211 for path in pos_paths[start_i]:

1212 pos_paths[-1].append(

1213 [(last_i, node.tags, node.topics)] + path

1214 )

1215 else:

1216 # print("UNK END start_i={} last_i={} wordlst={}"

1217 # .format(start_i, last_i, wordlst))

1218 u = check_unknown(

1219 last_i,

1220 len(wordlst),

1221 len(wordlst),

1222 wordlst,

1223 allow_any,

1224 no_unknown_starts,

1225 )

1226 if pos_paths[start_i]:

1227 for path in pos_paths[start_i]:

1228 pos_paths[-1].append(u + path)

1229 else:

1230 pos_paths[-1].append(u)

1231 else:

1232 # Check for a final unknown tag

1233 # print("NO END NODES max_last_i={}".format(max_last_i))

1234 paths = pos_paths[max_last_i] or [[]]

1235 u = check_unknown(

1236 max_last_i,

1237 len(wordlst),

1238 len(wordlst),

1239 wordlst,

1240 allow_any,

1241 no_unknown_starts,

1242 )

1243 if u:

1244 # print("end max_last_i={}".format(max_last_i))

1245 for path in list(paths): # Copy in case it is the last pos

1246 pos_paths[-1].append(u + path)

1247

1248 # import json

1249 # print("POS_PATHS:", json.dumps(pos_paths, indent=2, sort_keys=True))

1250

1251 if not pos_paths[-1]:

1252 # print("decode_tags: {}: EMPTY POS_PATHS[-1]".format(src))

1253 return [], []

1254

1255 # Find the best path

1256 pw = []

1257 for path in pos_paths[-1]:

1258 weight = len(path)

1259 if any(x[1] == ["UNKNOWN"] for x in path):

1260 weight += 100 # Penalize unknown paths

1261 pw.append((weight, path))

1262 path = min(pw)[1]

1263

1264 # Convert the best path to tagsets and topics

1265 tagsets: list[list[str]] = [[]]

1266 topics: list[str] = []

1267 for i, tagspec, topicspec in path:

1268 if len(tagsets or "") > 16:

1269 # ctx.error("Too many tagsets! This is probably exponential",

1270 # sortid="form_descriptions/20230818")

1271 return [("error-unknown-tag", "error-exponential-tagsets")], []

1272 if tagspec == ["UNKNOWN"]:

1273 new_tagsets = []

1274 for x in tagsets:

1275 new_tagsets.append(x + topicspec)

1276 tagsets = new_tagsets

1277 continue

1278 if tagspec:

1279 new_tagsets = []

1280 for x in tagsets:

1281 for t in tagspec:

1282 if t: 1282 ↛ 1289line 1282 didn't jump to line 1289 because the condition on line 1282 was always true

1283 new_tags = list(x)

1284 for tag in t.split():

1285 if tag not in new_tags:

1286 new_tags.append(tag)

1287 new_tagsets.append(new_tags)

1288 else:

1289 new_tagsets.append(x)

1290 tagsets = new_tagsets

1291 if topicspec:

1292 for t in topicspec:

1293 for topic in t.split():

1294 if topic not in topics:

1295 topics.append(topic)

1296

1297 # print("unsorted tagsets:", tagsets)

1298 ret_tagsets = sorted(set(tuple(sorted(set(tags))) for tags in tagsets))

1299 # topics = list(sorted(set(topics))) XXX tests expect not sorted

1300 # print("decode_tags: {} -> {} topics {}".format(src, tagsets, topics))

1301 # Yes, ret_tagsets is a list of tags in tuples, while topics is a LIST

1302 # of tags. Turning topics into a tuple breaks tests, turning the tuples

1303 # inside tagsets into lists breaks tests, I'm leaving them mismatched

1304 # for now. XXX

1305 return ret_tagsets, topics

1306

1307

1308def parse_head_final_tags(

1309 wxr: WiktextractContext, lang: str, form: str

1310) -> tuple[str, list[str]]:

1311 """Parses tags that are allowed at the end of a form head from the end

1312 of the form. This can also be used for parsing the final gender etc tags

1313 from translations and linkages."""

1314 assert isinstance(wxr, WiktextractContext)

1315 assert isinstance(lang, str) # Should be language that "form" is for

1316 assert isinstance(form, str)

1317

1318 # print("parse_head_final_tags: lang={} form={!r}".format(lang, form))

1319

1320 # Make sure there are no double spaces in the form as this code does not

1321 # handle them otherwise.

1322 form = re.sub(r"\s+", " ", form.strip())

1323 if not form:

1324 return form, []

1325

1326 origform = form

1327

1328 tags = []

1329

1330 # If parsing for certain Bantu languages (e.g., Swahili), handle

1331 # some extra head-final tags first

1332 if lang in head_final_bantu_langs:

1333 m = re.search(head_final_bantu_re, form)

1334 if m is not None:

1335 tagkeys = m.group(1)

1336 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1336 ↛ 1351line 1336 didn't jump to line 1351 because the condition on line 1336 was always true

1337 form = form[: m.start()]

1338 v = head_final_bantu_map[tagkeys]

1339 if v.startswith("?"): 1339 ↛ 1340line 1339 didn't jump to line 1340 because the condition on line 1339 was never true

1340 v = v[1:]

1341 wxr.wtp.debug(

1342 "suspicious suffix {!r} in language {}: {}".format(

1343 tagkeys, lang, origform

1344 ),

1345 sortid="form_descriptions/1028",

1346 )

1347 tags.extend(v.split())

1348

1349 # If parsing for certain Semitic languages (e.g., Arabic), handle

1350 # some extra head-final tags first

1351 if lang in head_final_semitic_langs:

1352 m = re.search(head_final_semitic_re, form)

1353 if m is not None:

1354 tagkeys = m.group(1)

1355 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1355 ↛ 1370line 1355 didn't jump to line 1370 because the condition on line 1355 was always true

1356 form = form[: m.start()]

1357 v = head_final_semitic_map[tagkeys]

1358 if v.startswith("?"): 1358 ↛ 1359line 1358 didn't jump to line 1359 because the condition on line 1358 was never true

1359 v = v[1:]

1360 wxr.wtp.debug(

1361 "suspicious suffix {!r} in language {}: {}".format(

1362 tagkeys, lang, origform

1363 ),

1364 sortid="form_descriptions/1043",

1365 )

1366 tags.extend(v.split())

1367

1368 # If parsing for certain other languages (e.g., Lithuanian,

1369 # French, Finnish), handle some extra head-final tags first

1370 if lang in head_final_other_langs:

1371 m = re.search(head_final_other_re, form)

1372 if m is not None:

1373 tagkeys = m.group(1)

1374 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1374 ↛ 1380line 1374 didn't jump to line 1380 because the condition on line 1374 was always true

1375 form = form[: m.start()]

1376 tags.extend(head_final_other_map[tagkeys].split(" "))

1377

1378 # Handle normal head-final tags

1379 # Loop this until nothing is found

1380 while True:

1381 prev_form = form

1382 m = re.search(head_final_re, form)

1383 if m is not None:

1384 # print(f"{m=}, {m.groups()=}")

1385 tagkeys = m.group(3)

1386 # Only replace tags ending with numbers in languages that have

1387 # head-final numeric tags (e.g., Bantu classes); also, don't replace

1388 # tags if the main title ends with them (then presume they are part

1389 # of the word)

1390 # print("head_final_tags form={!r} tagkeys={!r} lang={}"

1391 # .format(form, tagkeys, lang))

1392 tagkeys_contains_digit = re.search(r"\d", tagkeys)

1393 if (

1394 (not tagkeys_contains_digit or lang in head_final_numeric_langs)

1395 and not wxr.wtp.title.endswith(" " + tagkeys) # type:ignore[union-attr]

1396 and

1397 # XXX the above test does not capture when the whole word is a

1398 # xlat_head_map key, so I added the below test to complement

1399 # it; does this break anything?

1400 not wxr.wtp.title == tagkeys

1401 ): # defunct/English,

1402 # "more defunct" -> "more" ["archaic"]

1403 if ( 1403 ↛ 1421line 1403 didn't jump to line 1421 because the condition on line 1403 was always true

1404 not tagkeys_contains_digit

1405 or lang in head_final_numeric_langs

1406 ):

1407 # m.start(3) gets the start of what is in m.group(3), handy

1408 form = form[: m.start(3)].strip()

1409 v = xlat_head_map[tagkeys]

1410 if v.startswith("?"): 1410 ↛ 1411line 1410 didn't jump to line 1411 because the condition on line 1410 was never true

1411 v = v[1:]

1412 wxr.wtp.debug(

1413 "suspicious suffix {!r} in language {}: {}".format(

1414 tagkeys, lang, origform

1415 ),

1416 sortid="form_descriptions/1077",

1417 )

1418 tags.extend(v.split())

1419 else:

1420 break

1421 if prev_form == form:

1422 break

1423

1424 # Generate warnings about words ending in " or" after processing

1425 if (

1426 (form.endswith(" or") and not origform.endswith(" or"))

1427 or re.search(

1428 r" (1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|"

1429 r"1a|2a|9a|10a|m1|f1|f2|m2|f3|m3|f4|m4|f5|m5|or|\?)"

1430 r"($|/| (f|m|sg|pl|anim|inan))",

1431 form,

1432 )

1433 or form.endswith(" du")

1434 ):

1435 if form not in ok_suspicious_forms:

1436 wxr.wtp.debug(

1437 "suspicious unhandled suffix in {}:"

1438 " {!r}, originally {!r}".format(lang, form, origform),

1439 sortid="form_descriptions/1089",

1440 )

1441

1442 # print("parse_head_final_tags: form={!r} tags={}".format(form, tags))

1443 return form, tags

1444

1445

1446def quote_kept_parens(s: str) -> str:

1447 """Changes certain parenthesized expressions so that they won't be

1448 interpreted as parentheses. This is used for parts that are kept as

1449 part of the word, such as "read admiral (upper half)"."""

1450 return re.sub(

1451 r"\((lower half|upper half|k|s|n|II|III|A|C|G|U|Y|"

1452 r"vinyl|p-phenylene vinylene|$\(\s*$\))\)",

1453 r"__lpar__\1__rpar__",

1454 s,

1455 )

1456

1457

1458def quote_kept_ruby(

1459 wxr: WiktextractContext,

1460 ruby_tuples: list[

1461 tuple[

1462 str,

1463 str,

1464 ]

1465 ],

1466 s: str,

1467) -> str:

1468 if len(ruby_tuples) < 1: 1468 ↛ 1469line 1468 didn't jump to line 1469 because the condition on line 1468 was never true

1469 wxr.wtp.debug(

1470 "quote_kept_ruby called with no ruby",

1471 sortid="form_description/1114/20230517",

1472 )

1473 return s

1474 ks = []

1475 rs = []

1476 for k, r in ruby_tuples:

1477 ks.append(re.escape(k))

1478 rs.append(re.escape(r))

1479 if not (ks and rs): 1479 ↛ 1480line 1479 didn't jump to line 1480 because the condition on line 1479 was never true

1480 wxr.wtp.debug(

1481 f"empty column in ruby_tuples: {ruby_tuples}",

1482 sortid="form_description/1124/20230606",

1483 )

1484 return s

1485 newm = re.compile(

1486 r"({})\s*$\s*({})\s*$".format("|".join(ks), "|".join(rs))

1487 )

1488 rub_re = re.compile(

1489 r"({})".format(

1490 r"|".join(

1491 r"{}$*{}$*".format(

1492 re.escape(k),

1493 re.escape(r),

1494 )

1495 for k, r in ruby_tuples

1496 )

1497 )

1498 )

1499

1500 def paren_replace(m: re.Match) -> str:

1501 return re.sub(newm, r"\1__lrub__\2__rrub__", m.group(0))

1502

1503 return re.sub(rub_re, paren_replace, s)

1504

1505

1506def unquote_kept_parens(s: str) -> str:

1507 """Converts the quoted parentheses back to normal parentheses."""

1508 return re.sub(r"__lpar__(.*?)__rpar__", r"(\1)", s)

1509

1510

1511def add_romanization(

1512 wxr: WiktextractContext,

1513 data: WordData,

1514 roman: str,

1515 text: str,

1516 is_reconstruction: bool,

1517 head_group: Optional[int],

1518 ruby: Sequence[tuple[str, str]],

1519) -> None:

1520 tags_lst = ["romanization"]

1521 m = re.match(r"([^:]+):(.+)", roman)

1522 # This function's purpose is to intercept broken romanizations,

1523 # like "Yale: hēnpyeng" style tags. Most romanization styles

1524 # are already present as tags, so we can use decode_tags to find

1525 # them.

1526 if m: 1526 ↛ 1527line 1526 didn't jump to line 1527 because the condition on line 1526 was never true

1527 tagsets, topics = decode_tags(m.group(1))

1528 if tagsets:

1529 for tags in tagsets:

1530 tags_lst.extend(tags)

1531 roman = m.group(2)

1532 add_related(

1533 wxr,

1534 data,

1535 tags_lst,

1536 [roman],

1537 text,

1538 True,

1539 is_reconstruction,

1540 head_group,

1541 ruby,

1542 )

1543

1544

1545def add_related(

1546 wxr: WiktextractContext,

1547 data: WordData,

1548 tags_lst: Union[list[str], tuple[str, ...]],

1549 related_list: list[str],

1550 origtext: str,

1551 add_all_canonicals: bool,

1552 is_reconstruction: bool,

1553 head_group: Optional[int],

1554 ruby_data: Optional[Sequence[tuple[str, str]]] = None,

1555 links: list[tuple[str, str]] | None = None,

1556 link_dict: dict[str, list[str]] | None = None,

1557) -> Optional[list[tuple[str, ...]]]:

1558 """Internal helper function for some post-processing entries for related

1559 forms (e.g., in word head). This returns a list of list of tags to be

1560 added to following related forms or None (cf. walrus/English word head,

1561 parenthesized part starting with "both")."""

1562 assert isinstance(wxr, WiktextractContext)

1563 assert isinstance(tags_lst, (list, tuple))

1564 for x in tags_lst:

1565 assert isinstance(x, str)

1566 assert isinstance(related_list, (list, tuple))

1567 assert isinstance(origtext, str)

1568 assert add_all_canonicals in (True, False)

1569 assert isinstance(ruby_data, (list, tuple)) or ruby_data is None

1570 if ruby_data is None: 1570 ↛ 1571line 1570 didn't jump to line 1571 because the condition on line 1570 was never true

1571 ruby_data = []

1572 related = " ".join(related_list)

1573 # print("add_related: tags_lst={} related={}".format(tags_lst, related))

1574 if related == "[please provide]": 1574 ↛ 1575line 1574 didn't jump to line 1575 because the condition on line 1574 was never true

1575 return None

1576 if related in IGNORED_RELATED: 1576 ↛ 1577line 1576 didn't jump to line 1577 because the condition on line 1576 was never true

1577 return None

1578 if is_reconstruction and related.startswith("*") and len(related) > 1:

1579 related = related[1:]

1580

1581 # print(f"{links=}, {link_dict=}")

1582 # Get title word, with any reconstruction prefix removed

1583 titleword = re.sub(r"^Reconstruction:[^/]*/", "", wxr.wtp.title) # type:ignore[arg-type]

1584

1585 def check_related(related: str) -> None:

1586 # Warn about some suspicious related forms

1587 m = re.search(suspicious_related_re, related)

1588 if (m and m.group(0) not in titleword) or (

1589 related in ("f", "m", "n", "c") and len(titleword) >= 3

1590 ):

1591 if "eumhun" in tags_lst: 1591 ↛ 1592line 1591 didn't jump to line 1592 because the condition on line 1591 was never true

1592 return

1593 if "cangjie-input" in tags_lst: 1593 ↛ 1594line 1593 didn't jump to line 1594 because the condition on line 1593 was never true

1594 return

1595 if "class" in tags_lst: 1595 ↛ 1596line 1595 didn't jump to line 1596 because the condition on line 1595 was never true

1596 return

1597 if wxr.wtp.section == "Korean" and re.search( 1597 ↛ 1601line 1597 didn't jump to line 1601 because the condition on line 1597 was never true

1598 r"^\s*\w*>\w*\s*$", related

1599 ):

1600 # ignore Korean "i>ni" / "라>나" values

1601 return

1602 if ( 1602 ↛ 1609line 1602 didn't jump to line 1609 because the condition on line 1602 was never true

1603 wxr.wtp.section == "Burmese"

1604 and "romanization" in tags_lst

1605 and re.search(r":", related)

1606 ):

1607 # ignore Burmese with ":", that is used in Burmese

1608 # translitteration of "း", the high-tone visarga.

1609 return

1610 wxr.wtp.debug(

1611 "suspicious related form tags {}: {!r} in {!r}".format(

1612 tags_lst, related, origtext

1613 ),

1614 sortid="form_descriptions/1147",

1615 )

1616

1617 following_tagsets = None # Tagsets to add to following related forms

1618 roman = None

1619 tagsets1: list[tuple[str, ...]] = [tuple()]

1620 topics1: list[str] = []

1621

1622 m = re.match(r"$(([^()]|\([^()]*$)*)\)\s+", related)

1623 if m:

1624 paren = m.group(1)

1625 related = related[m.end() :]

1626 m = re.match(r"^(all|both) (.*)", paren)

1627 if m: 1627 ↛ 1628line 1627 didn't jump to line 1628 because the condition on line 1627 was never true

1628 tagsets1, topics1 = decode_tags(m.group(2))

1629 following_tagsets = tagsets1

1630 else:

1631 tagsets1, topics1 = decode_tags(paren)

1632 else:

1633 m = re.search(r"\s+$(([^()]|\([^()]*$)*)\)$", related)

1634 if m:

1635 paren = m.group(1)

1636 if paren.startswith("U+"): 1636 ↛ 1637line 1636 didn't jump to line 1637 because the condition on line 1636 was never true

1637 related = related[: m.start()]

1638 else:

1639 cls = classify_desc(paren)

1640 if ( 1640 ↛ 1647line 1640 didn't jump to line 1647 because the condition on line 1640 was always true

1641 cls in ("romanization", "english")

1642 and classify_desc(related[: m.start()]) == "other"

1643 ):

1644 roman = paren

1645 related = related[: m.start()]

1646 else:

1647 related = related[: m.start()]

1648 tagsets1, topics1 = decode_tags(paren)

1649 if related and related.startswith("{{"): 1649 ↛ 1650line 1649 didn't jump to line 1650 because the condition on line 1649 was never true

1650 wxr.wtp.debug(

1651 "`{{` in word head form - possible Wiktionary error: {!r}".format(

1652 related

1653 ),

1654 sortid="form_descriptions/1177",

1655 )

1656 return None # Likely Wiktionary coding error

1657 related = unquote_kept_parens(related)

1658 # Split related by "/" (e.g., grande/Spanish) superlative in head

1659 # Do not split if / in word title, see π//Japanese

1660 if len(related) > 5 and "/" not in wxr.wtp.title: # type:ignore[operator]

1661 alts = split_at_comma_semi(related, separators=["/"])

1662 else:

1663 alts = [related]

1664 if ruby_data:

1665 # prepare some regex stuff in advance

1666 ks, rs = [], []

1667 for k, r in ruby_data:

1668 ks.append(re.escape(k))

1669 rs.append(re.escape(r))

1670 splitter = r"((?:{})__lrub__(?:{})__rrub__)".format(

1671 "|".join(ks), "|".join(rs)

1672 )

1673 for related in alts:

1674 ruby: list[tuple[str, str]] = []

1675 if ruby_data:

1676 new_related = []

1677 rub_split = re.split(splitter, related)

1678 for s in rub_split:

1679 m = re.match(r"(.+)__lrub__(.+)__rrub__", s)

1680 if m:

1681 # add ruby with (\1, \2)

1682 ruby.append((m.group(1), m.group(2)))

1683 new_related.append(m.group(1))

1684 else:

1685 new_related.append(s)

1686 related = "".join(new_related)

1687 tagsets2, topics2 = decode_tags(" ".join(tags_lst))

1688 for tags1 in tagsets1:

1689 assert isinstance(tags1, (list, tuple))

1690 for tags2 in tagsets2:

1691 assert isinstance(tags1, (list, tuple))

1692 dt: LinkageData = {"word": related}

1693 if roman:

1694 dt["roman"] = roman

1695 if ruby:

1696 dt["ruby"] = ruby

1697 if "alt-of" in tags2: 1697 ↛ 1698line 1697 didn't jump to line 1698 because the condition on line 1697 was never true

1698 check_related(related)

1699 data_extend(data, "tags", tags1)

1700 data_extend(data, "tags", tags2)

1701 data_extend(data, "topics", topics1)

1702 data_extend(data, "topics", topics2)

1703 data_append(data, "alt_of", dt)

1704 elif "form-of" in tags2: 1704 ↛ 1705line 1704 didn't jump to line 1705 because the condition on line 1704 was never true

1705 check_related(related)

1706 data_extend(data, "tags", tags1)

1707 data_extend(data, "tags", tags2)

1708 data_extend(data, "topics", topics1)

1709 data_extend(data, "topics", topics2)

1710 data_append(data, "form_of", dt)

1711 elif "compound-of" in tags2: 1711 ↛ 1712line 1711 didn't jump to line 1712 because the condition on line 1711 was never true

1712 check_related(related)

1713 data_extend(data, "tags", tags1)

1714 data_extend(data, "tags", tags2)

1715 data_extend(data, "topics", topics1)

1716 data_extend(data, "topics", topics2)

1717 data_append(data, "compound", related)

1718 else:

1719 lang = wxr.wtp.section or "LANG_MISSING"

1720 related, final_tags = parse_head_final_tags(

1721 wxr, lang, related

1722 )

1723 # print("add_related: related={!r} tags1={!r} tags2={!r} "

1724 # "final_tags={!r}"

1725 # .format(related, tags1, tags2, final_tags))

1726 tags = list(tags1) + list(tags2) + list(final_tags)

1727 check_related(related)

1728 form: FormData = {"form": related}

1729 if (

1730 links

1731 and link_dict

1732 and (

1733 form_links := match_links_to_form(

1734 wxr, related, links, link_dict

1735 )

1736 )

1737 ):

1738 form["links"] = form_links

1739 if head_group:

1740 form["head_nr"] = head_group

1741 if roman:

1742 form["roman"] = roman

1743 if ruby:

1744 form["ruby"] = ruby

1745 data_extend(form, "topics", topics1)

1746 data_extend(form, "topics", topics2)

1747 if topics1 or topics2: 1747 ↛ 1748line 1747 didn't jump to line 1748 because the condition on line 1747 was never true

1748 wxr.wtp.debug(

1749 "word head form has topics: {}".format(form),

1750 sortid="form_descriptions/1233",

1751 )

1752 # Add tags from canonical form into the main entry

1753 if "canonical" in tags:

1754 if related in ("m", "f") and len(titleword) > 1: 1754 ↛ 1755line 1754 didn't jump to line 1755 because the condition on line 1754 was never true

1755 wxr.wtp.debug(

1756 "probably incorrect canonical form "

1757 "{!r} ignored (probably tag combination "

1758 "missing from xlat_head_map)".format(related),

1759 sortid="form_descriptions/1241",

1760 )

1761 continue

1762 if (

1763 related != titleword

1764 or add_all_canonicals

1765 or topics1

1766 or topics2

1767 or ruby

1768 ):

1769 data_extend(form, "tags", sorted(set(tags)))

1770 else:

1771 # We won't add canonical form here

1772 filtered_tags = list(

1773 x for x in tags if x != "canonical"

1774 )

1775 data_extend(data, "tags", filtered_tags)

1776 continue

1777 else:

1778 data_extend(form, "tags", sorted(set(tags)))

1779 # Only insert if the form is not already there

1780 for old in data.get("forms", ()):

1781 if form == old: 1781 ↛ 1782line 1781 didn't jump to line 1782 because the condition on line 1781 was never true

1782 break

1783 else:

1784 data_append(data, "forms", form)

1785

1786 # If this form had pre-tags that started with "both" or "all", add those

1787 # tags also to following related forms that don't have their own tags

1788 # specified.

1789 return following_tagsets

1790

1791

1792def match_links_to_form(

1793 wxr: WiktextractContext,

1794 form: str,

1795 links: list[tuple[str, str]],

1796 link_dict: dict[str, list[str]] | None,

1797) -> list[tuple[str, str]] | None:

1798 if not links: 1798 ↛ 1799line 1798 didn't jump to line 1799 because the condition on line 1798 was never true

1799 return None

1800 if link_dict is None:

1801 link_dict = {}

1802 for ltxt, ltrg in links:

1803 if ltxt not in link_dict:

1804 link_dict[ltxt] = [

1805 ltrg,

1806 ]

1807 else:

1808 link_dict[ltxt].append(ltrg)

1809 ret: list[tuple[str, str]] = []

1810 if form in link_dict:

1811 if len(link_dict[form]) > 1 and any( 1811 ↛ 1814line 1811 didn't jump to line 1814 because the condition on line 1811 was never true

1812 x != link_dict[form][0] for x in link_dict[form]

1813 ):

1814 wxr.wtp.warning(

1815 f"{form=} has many different "

1816 f"link candidates `{link_dict[form]}`, "

1817 f"which can't be disambiguated.",

1818 sortid="form_descriptions/match_links_to_form",

1819 )

1820 for ltarg in link_dict[form]:

1821 ret.append((form, ltarg))

1822 elif " " in form:

1823 # split and search for a sequence of links...

1824 split_forms = form.split()

1825 found = False

1826 for i, (ltext, ltarg) in enumerate(links):

1827 if ltext == split_forms[0]:

1828 for j, f in enumerate(split_forms):

1829 if i + j >= len(links):

1830 break

1831 if f.strip(",;() ") != links[i + j][0].strip(",;() "):

1832 break

1833 if i + j == len(links): 1833 ↛ 1834line 1833 didn't jump to line 1834 because the condition on line 1833 was never true

1834 break

1835 else:

1836 found = True

1837 if found:

1838 ret = links[i : i + len(split_forms)]

1839 break

1840 # We only care about weird links

1841 # print(f"{len(ret)=}, {ret}")

1842 for txt, tar in ret:

1843 if txt != tar and txt != tar[: tar.find("#")]:

1844 break

1845 else:

1846 return None

1847 return ret or None

1848

1849

1850# Issue #967, in English word forms sometimes forms are skipped because

1851# they are taggable words and their distw() is too big, like clipping from clip

1852WORDS_WITH_FALSE_POSITIVE_TAGS: dict[str, list[str]] = {

1853 "clip": ["clipping"], # XXX remember to change me back to clipping after

1854 "English": ["English", "Englishes"],

1855 "common": ["common", "commoner"],

1856 # tests.

1857}

1858

1859WORDS_WITH_FALSE_POSITIVE_FORMS: dict[str, list[str]] = {

1860 "unaccountability": ["countable", "uncountable"],

1861 "uncountability": ["countable", "uncountable"],

1862}

1863

1864FALSE_POSITIVE_MISSING_FORMS: dict[str, list[str]] = {}

1865

1866FORM_ASSOCIATED_TAG_WORDS: set[str] = {

1867 "participle",

1868 "past",

1869 "present",

1870 "singular",

1871 "plural",

1872 "first-person",

1873 "second-person",

1874 "third-person",

1875 "gerund",

1876}

1877

1878SEMICOLON_REPLACEMENT = "__SEMICOLON__"

1879

1880

1881def parse_word_head(

1882 wxr: WiktextractContext,

1883 word: str,

1884 pos: str,

1885 text: str,

1886 data: WordData,

1887 is_reconstruction: bool,

1888 head_group: Optional[int],

1889 original_header_nodes: list[WikiNode | str] | None = None,

1890 ruby=None,

1891 links: list[

1892 tuple[

1893 str,

1894 str,

1895 ]

1896 ]

1897 | None = None,

1898 label_templates: list[TemplateData] | None = None,

1899) -> None:

1900 """Parses the head line for a word for in a particular language and

1901 part-of-speech, extracting tags and related forms."""

1902 assert isinstance(wxr, WiktextractContext)

1903 assert isinstance(pos, str)

1904 assert isinstance(text, str)

1905 assert isinstance(data, dict)

1906 assert isinstance(ruby, (list, tuple)) or ruby is None

1907 if ruby is None:

1908 ruby = []

1909 assert is_reconstruction in (True, False)

1910 # print("PARSE_WORD_HEAD: {}: {!r}".format(wxr.wtp.section, text))

1911 # print(f"PARSE_WORD_HEAD: {data=}")

1912 # print(f"PARSE_WORD_HEAD: {links=}")

1913

1914 # Save original text for if we want to look for mismatched form-links

1915

1916 link_dict: dict[str, list[str]] | None

1917 if links is not None:

1918 link_dict = {}

1919 for ltxt, ltrg in links:

1920 if ltxt not in link_dict:

1921 link_dict[ltxt] = [

1922 ltrg,

1923 ]

1924 else:

1925 link_dict[ltxt].append(ltrg)

1926 else:

1927 link_dict = None

1928

1929 # print(f"MAIN: {links=}")

1930 link_words_not_alnum = []

1931 if not word.isalnum() and not word.replace("-", "").isalnum():

1932 # `-` is kosher, add more of these if needed.

1933 # if the word contains non-letter or -number characters, it

1934 # might have something that messes with split-at-semi-comma; we

1935 # collect links so that we can skip splitting them.

1936 if links is None and original_header_nodes is not None:

1937 links, _ = extract_links_from_node(

1938 wxr,

1939 original_header_nodes,

1940 remove_anchor_tags=True,

1941 expand_nodes=True,

1942 )

1943 if links is not None: 1943 ↛ 1947line 1943 didn't jump to line 1947 because the condition on line 1943 was always true

1944 for ltext, ltar in links:

1945 if not ltext.isalnum():

1946 link_words_not_alnum.append(ltext)

1947 if word not in link_words_not_alnum: 1947 ↛ 1950line 1947 didn't jump to line 1950 because the condition on line 1947 was always true

1948 link_words_not_alnum.append(word)

1949

1950 if link_words_not_alnum is None: 1950 ↛ 1951line 1950 didn't jump to line 1951 because the condition on line 1950 was never true

1951 link_words_not_alnum = []

1952

1953 if len(link_words_not_alnum) > 0:

1954 # if we have link data (that is, links with stuff like commas and

1955 # spaces, replace word_re with a modified local scope pattern

1956 # print(f"links {list((c, ord(c)) for link in links for c in link)=}")

1957 word_re = re.compile(

1958 r"\b" # In case we have forms that are longer and contain links

1959 +

1960 # or words as a substring...

1961 r"\b|\b".join(

1962 sorted(

1963 (re.escape(s) for s in link_words_not_alnum),

1964 key=lambda x: -len(x),

1965 )

1966 )

1967 + r"\b|"

1968 + word_pattern

1969 )

1970 else:

1971 word_re = word_re_global

1972

1973 if "Lua execution error" in text or "Lua timeout error" in text: 1973 ↛ 1974line 1973 didn't jump to line 1974 because the condition on line 1973 was never true

1974 return

1975

1976 # Fix words with "superlative:" or "comparative:" at end of head

1977 # e.g. grande/Spanish/Adj

1978 text = re.sub(r" (superlative|comparative): (.*)", r" (\1 \2)", text)

1979

1980 # Parse Arabic non-past forms, e.g. أبلع/Arabic/Verb

1981 m = re.search(r", non-past ([^)]+ $[^)]+$)", text)

1982 if m:

1983 add_related(

1984 wxr,

1985 data,

1986 ["non-past"],

1987 [m.group(1)],

1988 text,

1989 True,

1990 is_reconstruction,

1991 head_group,

1992 ruby,

1993 links,

1994 link_dict,

1995 )

1996 text = text[: m.start()] + text[m.end() :]

1997

1998 language = wxr.wtp.section

1999 titleword = re.sub(

2000 r"^Reconstruction:[^/]*/", "", wxr.wtp.title or "MISSING_TITLE"

2001 )

2002 titleparts = list(

2003 m.group(0)

2004 for m in re.finditer(word_re, wxr.wtp.title or "MISSING_TITLE")

2005 )

2006 if not titleparts: 2006 ↛ 2007line 2006 didn't jump to line 2007 because the condition on line 2006 was never true

2007 return

2008

2009 # Remove " or" from the end to prevent weird canonical forms

2010 if text.endswith(" or"):

2011 for tp in titleparts:

2012 if text.endswith(tp): 2012 ↛ 2013line 2012 didn't jump to line 2013 because the condition on line 2012 was never true

2013 break

2014 else:

2015 text = text.removesuffix(" or").rstrip()

2016

2017 # Handle the part of the head that is not in parentheses. However, certain

2018 # parenthesized parts are part of word, and those must be handled

2019 # specially here.

2020 if ruby:

2021 text = quote_kept_ruby(wxr, ruby, text)

2022 base = text

2023 base = quote_kept_parens(base)

2024 base = remove_text_in_parentheses(base)

2025 base = base.replace("?", "") # Removes uncertain articles etc

2026 base = re.sub(r"\s+", " ", base)

2027 base = re.sub(r" ([,;])", r"\1", base)

2028 base = re.sub(r" • ", r" ", base)

2029 # Many languages use • as a punctuation mark separating the base

2030 # from the rest of the head. στάδιος/Ancient Greek, issue #176

2031 base = base.strip()

2032 # print(f"{base=}, {text=}")

2033

2034 # Check for certain endings in head (mostly for compatibility with weird

2035 # heads, e.g. rata/Romanian "1st conj." at end)

2036 m = re.search(head_end_re, base)

2037 tags: Union[tuple[str, ...], list[str]] = []

2038 if m: 2038 ↛ 2039line 2038 didn't jump to line 2039 because the condition on line 2038 was never true

2039 tags = head_end_map[m.group(1).lower()].split()

2040 data_extend(data, "tags", tags)

2041 base = base[: m.start()]

2042

2043 if language == "Japanese":

2044 # Remove the leftover "^" of the historical kana superscript,

2045 # e.g. 静か "^(←しづか)-na"

2046 base = re.sub(r" \^(?!†)", " ", base).strip()

2047 while m := ja_head_end_re.search(base):

2048 tag = m.group(1)

2049 if tag == "uninflectable": 2049 ↛ 2050line 2049 didn't jump to line 2050 because the condition on line 2049 was never true

2050 tag = "uninflected"

2051 data_extend(data, "tags", [tag])

2052 base = base[: m.start()]

2053

2054 # Special case: handle Hán Nôm readings for Vietnamese characters

2055 m = re.match(

2056 r"{}: (Hán Nôm) readings: (.*)".format(re.escape(titleword)), base

2057 )

2058 if m: 2058 ↛ 2059line 2058 didn't jump to line 2059 because the condition on line 2058 was never true

2059 tag, readings = m.groups()

2060 tag = re.sub(r"\s+", "-", tag)

2061 for reading in split_at_comma_semi(

2062 readings, skipped=link_words_not_alnum

2063 ):

2064 add_related(

2065 wxr,

2066 data,

2067 [tag],

2068 [reading],

2069 text,

2070 True,

2071 is_reconstruction,

2072 head_group,

2073 ruby,

2074 links,

2075 link_dict,

2076 )

2077 return

2078

2079 # Special case: Hebrew " [pattern: nnn]" ending

2080 m = re.search(r"\s+\[pattern: ([^]]+)\]", base)

2081 if m: 2081 ↛ 2082line 2081 didn't jump to line 2082 because the condition on line 2081 was never true

2082 add_related(

2083 wxr,

2084 data,

2085 ["class"],

2086 [m.group(1)],

2087 text,

2088 True,

2089 is_reconstruction,

2090 head_group,

2091 ruby,

2092 links,

2093 link_dict,

2094 )

2095 base = base[: m.start()] + base[m.end() :]

2096

2097 # Clean away some messy "Upload an image" template text used in

2098 # American Sign Language:

2099 # S@NearBaseForearm-PalmUp Frontandback S@BaseForearm-PalmUp

2100 m = re.search(r"Upload .+ gif image.", base)

2101 if m: 2101 ↛ 2102line 2101 didn't jump to line 2102 because the condition on line 2101 was never true

2102 base = base[: m.start()] + base[m.end() :]

2103

2104 semicolon_present = False

2105 # Split the head into alternatives. This is a complicated task, as

2106 # we do not want so split on "or" or "," when immediately followed by more

2107 # head-final tags, but otherwise do want to split by them.

2108 # 20230907 added "or" to this to handle 'true or false', titles with 'or'

2109 if wxr.wtp.title and (

2110 "," in wxr.wtp.title or ";" in wxr.wtp.title or " or " in wxr.wtp.title

2111 ):

2112 # If the title has ";", we don't want to split on that and can remove

2113 # the ; from the splitting regex pretty easily because it's uncommon.

2114 # However, commas are so common that not splitting on them is just

2115 # not feasible, and we have to just deal with that if there are

2116 # alternative forms or variations with stray commas that shouldn't

2117 # be split.

2118 if ";" in wxr.wtp.title:

2119 semicolon_present = True

2120 base = base.replace(";", SEMICOLON_REPLACEMENT)

2121 default_splitter = head_split_no_semicolon_re

2122 else:

2123 default_splitter = head_split_re

2124 # A kludge to handle article titles/phrases with commas.

2125 # Preprocess splits to first capture the title, then handle

2126 # all the others as usual.

2127 presplits = re.split(r"({})".format(wxr.wtp.title), base)

2128 splits = []

2129 for psplit in presplits:

2130 if psplit == wxr.wtp.title:

2131 splits.append(psplit)

2132 else:

2133 splits.extend(re.split(default_splitter, psplit))

2134 else:

2135 # Do the normal split; previous behavior.

2136 splits = re.split(head_split_re, base)

2137 # print("BASE: ", repr(base))

2138 # print("SPLITS:", splits)

2139 alts: list[str] = []

2140 # print("parse_word_head: splits:", splits,

2141 # "head_split_re_parens:", head_split_re_parens)

2142 for i in range(

2143 0, len(splits) - head_split_re_parens, head_split_re_parens + 1

2144 ):

2145 v = splits[i]

2146 ending = splits[i + 1] or "" # XXX is this correct???

2147 # print("parse_word_head alts v={!r} ending={!r} alts={}"

2148 # .format(v, ending, alts))

2149 if alts and (v == "" and ending):

2150 assert ending[0] == " "

2151 alts[-1] += " or" + ending # endings starts with space

2152 elif v or ending:

2153 alts.append((v or "") + (ending or ""))

2154 last = splits[-1].strip()

2155 conn = "" if len(splits) < 3 else splits[-2]

2156 # print("parse_word_head alts last={!r} conn={!r} alts={}"

2157 # .format(last, conn, alts))

2158 if ( 2158 ↛ 2169line 2158 didn't jump to line 2169 because the condition on line 2158 was never true

2159 alts

2160 and last

2161 and (

2162 last.split()[0] in xlat_head_map

2163 or (

2164 conn == " or "

2165 and (alts[-1] + " or " + last).strip() in xlat_head_map

2166 )

2167 )

2168 ):

2169 alts[-1] += " or " + last

2170 elif last: 2170 ↛ 2171line 2170 didn't jump to line 2171 because the condition on line 2170 was never true

2171 alts.append(last)

2172

2173 # print("parse_word_head alts: {}".format(alts))

2174 # print(f"{base=}")

2175

2176 # Process the head alternatives

2177 canonicals: list[tuple[list[str], list[str]]] = []

2178 mode: Optional[str] = None

2179 for alt_i, alt in enumerate(alts):

2180 alt = alt.strip()

2181 if alt.startswith("compound form:"): 2181 ↛ 2182line 2181 didn't jump to line 2182 because the condition on line 2181 was never true

2182 mode = "compound-form"

2183 alt = alt[14:].strip()

2184 if ((dash_i := alt.find(" -")) > 0) and ( 2184 ↛ 2190line 2184 didn't jump to line 2190 because the condition on line 2184 was never true

2185 dash_i > (wxr.wtp.title or "").find(" -")

2186 ):

2187 # test_en_head / test_suffixes_at_end_of_form1

2188 # Some heads have suffixes that end up attached to the form

2189 # like in https://en.wiktionary.org/wiki/%E6%A5%BD%E3%81%97%E3%81%84

2190 alt = alt[:dash_i]

2191 if mode == "compound-form": 2191 ↛ 2192line 2191 didn't jump to line 2192 because the condition on line 2191 was never true

2192 add_related(

2193 wxr,

2194 data,

2195 ["in-compounds"],

2196 [alt],

2197 text,

2198 True,

2199 is_reconstruction,

2200 head_group,

2201 ruby,

2202 links,

2203 link_dict,

2204 )

2205 continue

2206 # For non-first parts, see if it can be treated as tags-only

2207 if alt_i == 0:

2208 expanded_alts = [alt]

2209 else:

2210 expanded_alts = map_with(xlat_descs_map, [alt])

2211 # print("EXPANDED_ALTS:", expanded_alts)

2212 tagsets: Optional[list[tuple[str, ...]]]

2213 for alt in expanded_alts:

2214 baseparts = list(m.group(0) for m in word_re.finditer(alt))

2215 if alt_i > 0:

2216 tagsets, topics = decode_tags(" ".join(baseparts))

2217 if not any("error-unknown-tag" in x for x in tagsets):

2218 data_extend(data, "topics", topics)

2219 for tags1 in tagsets:

2220 data_extend(data, "tags", tags1)

2221 continue

2222

2223 alt, tags = parse_head_final_tags(

2224 wxr, language or "MISSING_LANG", alt

2225 )

2226 tags = list(tags) # Make sure we don't modify anything cached

2227 tags.append("canonical")

2228 if alt_i == 0 and "," in wxr.wtp.title or ";" in wxr.wtp.title: # type:ignore[operator]

2229 # Kludge to handle article titles/phrases with commas.

2230 # basepart's regex strips commas, which leads to a

2231 # canonical form that is the title phrase without a comma.

2232 # basepart in add_related is almost immediately joined with

2233 # spaces anyhow. XXX not exactly sure why it's

2234 # canonicals.append((tags, baseparts)) and not (tags, [alt])

2235 baseparts = [alt]

2236 canonicals.append((tags, baseparts))

2237

2238 # If more of this kind of replace-and-return-original kind of stuff is

2239 # needed, make semicolon_present into a flag enum, something like `modified`

2240 if semicolon_present:

2241 new_cans = []

2242 for tags, baseparts in canonicals:

2243 new_cans.append(

2244 (

2245 tags,

2246 [s.replace(SEMICOLON_REPLACEMENT, ";") for s in baseparts],

2247 )

2248 )

2249 canonicals = new_cans

2250 for tags, baseparts in canonicals:

2251 add_related(

2252 wxr,

2253 data,

2254 tags,

2255 baseparts,

2256 text,

2257 len(canonicals) > 1,

2258 is_reconstruction,

2259 head_group,

2260 ruby,

2261 links,

2262 link_dict,

2263 )

2264

2265 # Handle parenthesized descriptors for the word form and links to

2266 # related words

2267 text = quote_kept_parens(text)

2268 parens = list(

2269 m.group(2)

2270 for m in re.finditer(r"(^|\s)$(([^()]|\([^()]*$)*)\)", text)

2271 )

2272 parens.extend(

2273 m.group(1)

2274 for m in re.finditer(r"[^\s]$(([^()]|\([^()]*$)*)\)($|\s)", text)

2275 )

2276 have_romanization = False

2277 have_ruby = False

2278 hiragana = ""

2279 katakana = ""

2280 for paren in parens:

2281 paren = paren.strip()

2282 if not paren: 2282 ↛ 2283line 2282 didn't jump to line 2283 because the condition on line 2282 was never true

2283 continue

2284 can_be_form = True

2285 if label_templates is not None and paren.startswith(

2286 "__LABEL_TEMPLATE_"

2287 ):

2288 # wxr.wtp.warning("Found label template in head")

2289 # continue

2290 can_be_form = False

2291 m = re.match(r"__LABEL_TEMPLATE_(\d+)__", paren)

2292 if m is None: 2292 ↛ 2293line 2292 didn't jump to line 2293 because the condition on line 2292 was never true

2293 wxr.wtp.warning(

2294 f"Label template list magic phrase is broken: `{paren}`",

2295 sortid="20260508/label list index broken",

2296 )

2297 continue

2298 ht = label_templates[int(m.group(1))]

2299 desc = ht.get("expansion", "").strip()

2300 if desc: 2300 ↛ 2303line 2300 didn't jump to line 2303 because the condition on line 2300 was always true

2301 paren = desc

2302 else:

2303 wxr.wtp.warning(

2304 f"Label template seems to have no text contents: {ht=}",

2305 sortid="20260508/label_templates",

2306 )

2307 continue

2308 if paren.startswith("see "):

2309 continue

2310 if paren.startswith("U+"): 2310 ↛ 2311line 2310 didn't jump to line 2311 because the condition on line 2310 was never true

2311 continue

2312 # In some rare cases, strip word that inflects form the form

2313 # description, e.g. "look through rose-tinted glasses"/English.

2314 # `([looks])`

2315 paren = re.sub(r"\s*$\[[^])]*\]$", "", paren)

2316

2317 # If it starts with hiragana or katakana, treat as such form. Note

2318 # that each hiragana/katakana character is in separate parentheses,

2319 # so we must concatenate them.

2320 try:

2321 un = unicodedata.name(paren[0]).split()[0]

2322 except ValueError:

2323 un = "INVALID"

2324 if un == "KATAKANA": 2324 ↛ 2325line 2324 didn't jump to line 2325 because the condition on line 2324 was never true

2325 katakana += paren

2326 have_ruby = True

2327 continue

2328 if un == "HIRAGANA": 2328 ↛ 2329line 2328 didn't jump to line 2329 because the condition on line 2328 was never true

2329 hiragana += paren

2330 have_ruby = True

2331 continue

2332

2333 # Parse format ", 16 (Japan, Mainland), 17 (Hong Kong, Taiwan) strokes,"

2334 # in the middle of the parenthesized expression, e.g. 薄

2335 def strokes_repl(m: re.Match) -> str:

2336 strokes1, tags1, strokes2, tags2 = m.groups()

2337 for strokes, tags in [[strokes1, tags1], [strokes2, tags2]]:

2338 tags = tags.split(", ")

2339 tags = list(

2340 "Mainland China" if t == "Mainland" else t for t in tags

2341 )

2342 tags.append("strokes")

2343 add_related(

2344 wxr,

2345 data,

2346 tags,

2347 [strokes],

2348 text,

2349 True,

2350 is_reconstruction,

2351 head_group,

2352 ruby,

2353 links,

2354 link_dict,

2355 )

2356 return ", "

2357

2358 if can_be_form is True:

2359 paren = re.sub(

2360 r", (\d+) $([^()]+)$, (\d+) $([^()]+)$ strokes, ",

2361 strokes_repl,

2362 paren,

2363 )

2364

2365 descriptors = map_with(xlat_descs_map, [paren])

2366 new_desc = []

2367 for desc in descriptors:

2368 new_desc.extend(

2369 map_with(

2370 xlat_tags_map,

2371 split_at_comma_semi(

2372 desc, extra=[", or "], skipped=link_words_not_alnum

2373 ),

2374 )

2375 )

2376 prev_tags: Union[list[list[str]], list[tuple[str, ...]], None] = None

2377 following_tags = None # Added to prev_tags from previous parenthesized

2378 # part, e.g. walrus/English

2379 # "(both nonstandard, proscribed, uncommon)"

2380 for desc_i, desc in enumerate(new_desc):

2381 # print("HEAD DESC: {!r}".format(desc))

2382

2383 # Abort on certain descriptors (assume remaining values are

2384 # examples or uninteresting, cf. gaan/Navajo, horior/Latin)

2385 if re.match(r"^(per |e\.g\.$)", desc): 2385 ↛ 2386line 2385 didn't jump to line 2386 because the condition on line 2385 was never true

2386 break

2387

2388 # If it all consists of CJK characters, add it with the

2389 # CJK tag. This is used at least for some Vietnamese

2390 # words (e.g., ba/Vietnamese)

2391 try:

2392 if ( 2392 ↛ 2396line 2392 didn't jump to line 2396 because the condition on line 2392 was never true

2393 all(unicodedata.name(x).startswith("CJK ") for x in desc)

2394 and can_be_form

2395 ):

2396 add_related(

2397 wxr,

2398 data,

2399 ["CJK"],

2400 [desc],

2401 text,

2402 True,

2403 is_reconstruction,

2404 head_group,

2405 ruby,

2406 links,

2407 link_dict,

2408 )

2409 continue

2410 except ValueError:

2411 pass

2412

2413 # Handle some special cases

2414 splitdesc = desc.split()

2415 if ( 2415 ↛ 2425line 2415 didn't jump to line 2425 because the condition on line 2415 was never true

2416 len(splitdesc) >= 3

2417 and splitdesc[1] == "superlative"

2418 and classify_desc(splitdesc[0]) != "tags"

2419 and prev_tags

2420 and can_be_form

2421 ):

2422 # Handle the special case of second comparative after comma,

2423 # followed by superlative without comma. E.g.

2424 # mal/Portuguese/Adv

2425 for ts in prev_tags:

2426 add_related(

2427 wxr,

2428 data,

2429 ts,

2430 [splitdesc[0]],

2431 text,

2432 True,

2433 is_reconstruction,

2434 head_group,

2435 ruby,

2436 links,

2437 link_dict,

2438 )

2439 desc = " ".join(splitdesc[1:])

2440 elif ( 2440 ↛ 2449line 2440 didn't jump to line 2449 because the condition on line 2440 was never true

2441 len(splitdesc) == 2

2442 and splitdesc[0] in ("also", "and")

2443 and prev_tags

2444 and classify_desc(splitdesc[1]) != "tags"

2445 and can_be_form

2446 ):

2447 # Sometimes alternative forms are prefixed with "also" or

2448 # "and"

2449 for ts in prev_tags:

2450 add_related(

2451 wxr,

2452 data,

2453 ts,

2454 [splitdesc[1]],

2455 text,

2456 True,

2457 is_reconstruction,

2458 head_group,

2459 ruby,

2460 links,

2461 link_dict,

2462 )

2463 continue

2464 elif len(splitdesc) >= 2 and splitdesc[0] in ("including",): 2464 ↛ 2465line 2464 didn't jump to line 2465 because the condition on line 2464 was never true

2465 continue

2466

2467 # If only one word, assume it is comma-separated alternative

2468 # to the previous one

2469 if len(splitdesc) == 1:

2470 cls = classify_desc(desc)

2471 if cls != "tags" and can_be_form:

2472 if prev_tags: 2472 ↛ 2474line 2472 didn't jump to line 2474 because the condition on line 2472 was never true

2473 # Assume comma-separated alternative to previous one

2474 for ts in prev_tags:

2475 add_related(

2476 wxr,

2477 data,

2478 ts,

2479 [desc],

2480 text,

2481 True,

2482 is_reconstruction,

2483 head_group,

2484 ruby,

2485 links,

2486 link_dict,

2487 )

2488 continue

2489 elif distw(titleparts, desc) <= 0.5: 2489 ↛ 2492line 2489 didn't jump to line 2492 because the condition on line 2489 was never true

2490 # Similar to head word, assume a dialectal variation to

2491 # the base form. Cf. go/Alemannic German/Verb

2492 add_related(

2493 wxr,

2494 data,

2495 ["alternative"],

2496 [desc],

2497 text,

2498 True,

2499 is_reconstruction,

2500 head_group,

2501 ruby,

2502 links,

2503 link_dict,

2504 )

2505 continue

2506 elif (

2507 cls in ("romanization", "english")

2508 and not have_romanization

2509 and classify_desc(titleword) == "other"

2510 and not (

2511 "categories" in data and desc in data["categories"]

2512 )

2513 ):

2514 # Assume it to be a romanization

2515 add_romanization(

2516 wxr,

2517 data,

2518 desc,

2519 text,

2520 is_reconstruction,

2521 head_group,

2522 ruby,

2523 )

2524 have_romanization = True

2525 continue

2526

2527 m = re.match(r"^(\d+) strokes?$", desc)

2528 if m and can_be_form:

2529 # Special case, used to give #strokes for Han characters

2530 add_related(

2531 wxr,

2532 data,

2533 ["strokes"],

2534 [m.group(1)],

2535 text,

2536 True,

2537 is_reconstruction,

2538 head_group,

2539 ruby,

2540 links,

2541 link_dict,

2542 )

2543 continue

2544

2545 # See if it is radical+strokes

2546 m = re.match(

2547 r"^([\u2F00-\u2FDF\u2E80-\u2EFF\U00018800-\U00018AFF"

2548 r"\uA490-\uA4CF\u4E00-\u9FFF]\+\d+)"

2549 r"( in (Japanese|Chinese|traditional Chinese|"

2550 r"simplified Chinese))?$",

2551 desc,

2552 )

2553 if m and can_be_form: 2553 ↛ 2556line 2553 didn't jump to line 2556 because the condition on line 2553 was never true

2554 # Special case, used to give radical + strokes for Han

2555 # characters

2556 radical_strokes = m.group(1)

2557 lang = m.group(3)

2558 t = ["radical+strokes"]

2559 if lang:

2560 t.extend(lang.split())

2561 add_related(

2562 wxr,

2563 data,

2564 t,

2565 [radical_strokes],

2566 text,

2567 True,

2568 is_reconstruction,

2569 head_group,

2570 ruby,

2571 links,

2572 link_dict,

2573 )

2574 prev_tags = None

2575 following_tags = None

2576 continue

2577

2578 # See if it indicates historical Katakana ortography (←) or

2579 # just otherwise katakana/hiragana form

2580 m = re.match(r"←\s*|kana\s+", desc)

2581 if m: 2581 ↛ 2582line 2581 didn't jump to line 2582 because the condition on line 2581 was never true

2582 if desc.startswith("←"):

2583 t1 = "historical "

2584 else:

2585 t1 = ""

2586 x = desc[m.end() :]

2587 if x.endswith("?"):

2588 x = x[:-1]

2589 # XXX should we add a tag indicating uncertainty?

2590 if x:

2591 name = unicodedata.name(x[0])

2592 if name.startswith("HIRAGANA "):

2593 desc = t1 + "hiragana " + x

2594 elif name.startswith("KATAKANA "):

2595 desc = t1 + "katakana " + x

2596

2597 # See if it is "n strokes in Chinese" or similar

2598 m = re.match(

2599 r"(\d+) strokes in (Chinese|Japanese|"

2600 r"traditional Chinese|simplified Chinese)$",

2601 desc,

2602 )

2603 if m and can_be_form: 2603 ↛ 2605line 2603 didn't jump to line 2605 because the condition on line 2603 was never true

2604 # Special case, used to give just strokes for some Han chars

2605 strokes = m.group(1)

2606 lang = m.group(2)

2607 t = ["strokes"]

2608 t.extend(lang.split())

2609 add_related(

2610 wxr,

2611 data,

2612 t,

2613 [strokes],

2614 text,

2615 True,

2616 is_reconstruction,

2617 head_group,

2618 ruby,

2619 links,

2620 link_dict,

2621 )

2622 prev_tags = None

2623 following_tags = None

2624 continue

2625

2626 # American Sign Language has images (or requests for image)

2627 # as heads, + this ASL gloss after.

2628 m2 = re.search(r"$ASL gloss:\s+(.*)$", text)

2629 if m2 and can_be_form: 2629 ↛ 2630line 2629 didn't jump to line 2630 because the condition on line 2629 was never true

2630 add_related(

2631 wxr,

2632 data,

2633 ["ASL-gloss"],

2634 [m2.group(1)],

2635 text,

2636 True,

2637 is_reconstruction,

2638 head_group,

2639 ruby,

2640 links,

2641 link_dict,

2642 )

2643 continue

2644

2645 parts = list(m.group(0) for m in re.finditer(word_re, desc))

2646 if not parts: 2646 ↛ 2647line 2646 didn't jump to line 2647 because the condition on line 2646 was never true

2647 prev_tags = None

2648 following_tags = None

2649 continue

2650

2651 # Check for certain language-specific header part starts that

2652 # modify

2653 if ( 2653 ↛ 2658line 2653 didn't jump to line 2658 because the condition on line 2653 was never true

2654 len(parts) == 2

2655 and language in lang_specific_head_map

2656 and can_be_form

2657 ):

2658 ht2 = lang_specific_head_map[language]

2659 if parts[0] in ht2:

2660 rem_tags, add_tags = ht2[parts[0]]

2661 new_prev_tags1: list[list[str]] = []

2662 tags2: Union[tuple[str, ...], list[str]]

2663 for tags2 in prev_tags or [()]:

2664 if rem_tags is True: # Remove all old tags

2665 tsets = set()

2666 else:

2667 tsets = set(tags2) - set(rem_tags.split())

2668 tsets = tsets | set(add_tags.split())

2669 tags = list(sorted(tsets))

2670 add_related(

2671 wxr,

2672 data,

2673 tags,

2674 [parts[1]],

2675 text,

2676 True,

2677 is_reconstruction,

2678 head_group,

2679 ruby,

2680 links,

2681 link_dict,

2682 )

2683 new_prev_tags1.append(tags)

2684 prev_tags = new_prev_tags1

2685 following_tags = None

2686 continue

2687

2688 # Handle the special case of descriptors that are parenthesized,

2689 # e.g., (archaic or Scotland)

2690 m = re.match(r"$([^)]+)$\s+(.*)$", desc)

2691 if m is not None and classify_desc(m.group(1)) == "tags": 2691 ↛ 2692line 2691 didn't jump to line 2692 because the condition on line 2691 was never true

2692 tagpart = m.group(1)

2693 related = [m.group(2)]

2694 tagsets, topics = decode_tags(tagpart, no_unknown_starts=True)

2695 if topics:

2696 wxr.wtp.debug(

2697 "parenthized head part {!r} contains topics: {}".format(

2698 tagpart, topics

2699 ),

2700 sortid="form_descriptions/1647",

2701 )

2702 elif m is not None and re.match(r"in the sense ", m.group(1)): 2702 ↛ 2705line 2702 didn't jump to line 2705 because the condition on line 2702 was never true

2703 # Handle certain ignored cases

2704 # e.g. bord/Danish: in the sense "plank"

2705 related = [m.group(2)]

2706 tagsets = [()]

2707 else:

2708 # Normal parsing of the descriptor

2709 alt_related = None

2710 alt_tagsets = None

2711 tagsets = None

2712 for i in range(len(parts), 0, -1):

2713 related = parts[i:]

2714 tagparts = parts[:i]

2715 # print(" i={} related={} tagparts={}"

2716 # .format(i, related, tagparts))

2717 tagsets, topics = decode_tags(

2718 " ".join(tagparts), no_unknown_starts=True

2719 )

2720 # print("tagparts={!r} tagsets={} topics={} related={} "

2721 # "alt_related={} distw={:.2f}"

2722 # .format(tagparts, tagsets, topics, related,

2723 # alt_related,

2724 # distw(titleparts, parts[i - 1])))

2725 if (

2726 topics

2727 or not tagsets

2728 or any("error-unknown-tag" in x for x in tagsets)

2729 ):

2730 if alt_related is not None: 2730 ↛ 2732line 2730 didn't jump to line 2732 because the condition on line 2730 was never true

2731 # We already had a good division, so let's stop.

2732 break

2733 # Bad division, try deeper

2734 continue

2735 # print(f"{parts[i-1]=}, {parts=}")

2736 if (

2737 i > 1

2738 and len(parts[i - 1]) >= 4

2739 and (

2740 distw(titleparts, parts[i - 1]) <= 0.4

2741 or (

2742 wxr.wtp.section == "English"

2743 and wxr.wtp.title

2744 in WORDS_WITH_FALSE_POSITIVE_TAGS

2745 and parts[i - 1]

2746 in WORDS_WITH_FALSE_POSITIVE_TAGS[wxr.wtp.title]

2747 )

2748 )

2749 # Fixes 'unaccountability' wiktext #1196

2750 and not (

2751 wxr.wtp.section == "English"

2752 and wxr.wtp.title in WORDS_WITH_FALSE_POSITIVE_FORMS

2753 and parts[i - 1]

2754 in WORDS_WITH_FALSE_POSITIVE_FORMS[wxr.wtp.title]

2755 )

2756 # Fixes wiktextract #983, where "participle"

2757 # was too close to "Martinize" and so this accepted

2758 # ["participle", "Martinize"] as matching; this

2759 # kludge prevents this from happening if titleparts

2760 # is shorter than what would be 'related'.

2761 # This breaks if we want to detect stuff that

2762 # actually gets an extra space-separated word when

2763 # 'inflected'.

2764 and (

2765 len(titleparts) >= len(parts[i - 1 :])

2766 or "or" in parts[i - 1 :]

2767 )

2768 ):

2769 # print(f"Reached; {parts=}, {parts[i-1]=}")

2770 alt_related = related

2771 alt_tagsets = tagsets

2772 continue

2773 alt_related = None

2774 alt_tagsets = None

2775 break

2776 # for-else

2777 else:

2778 if alt_related is None and can_be_form:

2779 # Check if the parenthesized part is likely a

2780 # romanization

2781 if ( 2781 ↛ 2789line 2781 didn't jump to line 2789 because the condition on line 2781 was never true

2782 (have_ruby or classify_desc(base) == "other")

2783 and classify_desc(paren) == "romanization"

2784 and not (

2785 "categories" in data

2786 and desc in data["categories"]

2787 )

2788 ):

2789 for r in split_at_comma_semi(

2790 paren,

2791 extra=[" or "],

2792 skipped=link_words_not_alnum,

2793 ):

2794 add_romanization(

2795 wxr,

2796 data,

2797 r,

2798 text,

2799 is_reconstruction,

2800 head_group,

2801 ruby,

2802 )

2803 have_romanization = True

2804 continue

2805 tagsets = [("error-unrecognized-head-form",)]

2806 wxr.wtp.debug(

2807 "unrecognized head form: {}".format(desc),

2808 sortid="form_descriptions/1698",

2809 )

2810 continue

2811

2812 if alt_related is not None: 2812 ↛ 2813line 2812 didn't jump to line 2813 because the condition on line 2812 was never true

2813 related = alt_related

2814 tagsets = alt_tagsets

2815

2816 # print("FORM END: tagsets={} related={}".format(tagsets, related))

2817 # print("==================")

2818

2819 if ( 2819 ↛ 2840line 2819 didn't jump to line 2840 because the condition on line 2819 was never true

2820 len(related) <= 0

2821 and wxr.wtp.section == "English"

2822 and tagsets is not None

2823 and len(tagsets) > 0

2824 and not any(

2825 s.startswith("error-") for tagset in tagsets for s in tagset

2826 )

2827 and any(

2828 s in FORM_ASSOCIATED_TAG_WORDS

2829 for tagset in tagsets

2830 for s in tagset

2831 )

2832 and (

2833 wxr.wtp.title not in FALSE_POSITIVE_MISSING_FORMS

2834 and not any(

2835 rel in FALSE_POSITIVE_MISSING_FORMS[wxr.wtp.title or ""]

2836 for rel in related

2837 )

2838 )

2839 ):

2840 wxr.wtp.debug(

2841 f"Form tags without form: {desc=}, {tagsets=}",

2842 sortid="form_description/20250107",

2843 )

2844 if not tagsets: 2844 ↛ 2845line 2844 didn't jump to line 2845 because the condition on line 2844 was never true

2845 continue

2846

2847 # print(f"{alts=}, {related=}")

2848

2849 assert isinstance(related, (list, tuple))

2850 related_str = " ".join(related)

2851 if "or" in titleparts:

2852 alts = [related_str]

2853 else:

2854 alts = split_at_comma_semi(

2855 related_str,

2856 separators=[r"\bor\b"],

2857 skipped=link_words_not_alnum,

2858 )

2859 # print(f"{related_str=}, {alts=}")

2860 if not alts:

2861 alts = [""]

2862 for related_str in alts:

2863 if related_str and can_be_form:

2864 if prev_tags and (

2865 all(

2866 all(

2867 t in ["nonstandard", "dialectal"]

2868 or valid_tags[t] == "dialect"

2869 for t in tags

2870 )

2871 for ts in tagsets

2872 )

2873 or (

2874 any("participle" in ts for ts in prev_tags)

2875 and all(

2876 "attributive" in ts

2877 or any(valid_tags[t] == "gender" for t in ts)

2878 for ts in tagsets

2879 )

2880 )

2881 ):

2882 # Merged with previous tags. Don't update previous

2883 # tags here; cf. burn/English/Verb

2884 for tags_l in tagsets:

2885 for ts in prev_tags:

2886 tags_l1 = sorted(set(tags_l) | set(ts))

2887 add_related(

2888 wxr,

2889 data,

2890 tags_l1,

2891 [related_str],

2892 text,

2893 True,

2894 is_reconstruction,

2895 head_group,

2896 ruby,

2897 links,

2898 link_dict,

2899 )

2900 else:

2901 # Not merged with previous tags

2902 for tags_l in tagsets:

2903 if following_tags is not None: 2903 ↛ 2904line 2903 didn't jump to line 2904 because the condition on line 2903 was never true

2904 for ts in following_tags:

2905 tags_l1 = list(

2906 sorted(set(tags_l) | set(ts))

2907 )

2908 add_related(

2909 wxr,

2910 data,

2911 tags_l1,

2912 [related_str],

2913 text,

2914 True,

2915 is_reconstruction,

2916 head_group,

2917 ruby,

2918 links,

2919 link_dict,

2920 )

2921 else:

2922 ret = add_related(

2923 wxr,

2924 data,

2925 tags_l,

2926 [related_str],

2927 text,

2928 True,

2929 is_reconstruction,

2930 head_group,

2931 ruby,

2932 links,

2933 link_dict,

2934 )

2935 if ret is not None: 2935 ↛ 2936line 2935 didn't jump to line 2936 because the condition on line 2935 was never true

2936 following_tags = ret

2937 prev_tags = tagsets

2938 else:

2939 if desc_i < len(new_desc) - 1 and all( 2939 ↛ 2946line 2939 didn't jump to line 2946 because the condition on line 2939 was never true

2940 "participle" in ts or "infinitive" in ts

2941 for ts in tagsets

2942 ):

2943 # Interpret it as a standalone form description

2944 # in the middle, probably followed by forms or

2945 # language-specific descriptors. cf. drikke/Danish

2946 new_prev_tags2 = []

2947 for ts1 in prev_tags or [()]:

2948 for ts2 in tagsets:

2949 ts = tuple(sorted(set(ts1) | set(ts2)))

2950 new_prev_tags2.append(ts)

2951 prev_tags = new_prev_tags2

2952 continue

2953 for tags in tagsets:

2954 data_extend(data, "tags", tags)

2955 prev_tags = tagsets

2956 following_tags = None

2957

2958 # Finally, if we collected hirakana/katakana, add them now

2959 if hiragana: 2959 ↛ 2960line 2959 didn't jump to line 2960 because the condition on line 2959 was never true

2960 add_related(

2961 wxr,

2962 data,

2963 ["hiragana"],

2964 [hiragana],

2965 text,

2966 True,

2967 is_reconstruction,

2968 head_group,

2969 ruby,

2970 )

2971 if katakana: 2971 ↛ 2972line 2971 didn't jump to line 2972 because the condition on line 2971 was never true

2972 add_related(

2973 wxr,

2974 data,

2975 ["katakana"],

2976 [katakana],

2977 text,

2978 True,

2979 is_reconstruction,

2980 head_group,

2981 ruby,

2982 )

2983

2984 # XXX check if this is actually relevant, tags in word root data

2985 # is extremely rare (not sure where they slip through).

2986 tags = data.get("tags", []) # type:ignore

2987 if len(tags) > 0:

2988 # wxr.wtp.debug(

2989 # f"Tags appear in word root data: {data['tags']=}", # type:ignore

2990 # sortid="form_descriptions/2620/20240606",

2991 # ) # Messes up tests.

2992 data["tags"] = sorted(set(tags)) # type:ignore

2993

2994

2995def parse_sense_qualifier(

2996 wxr: WiktextractContext, text: str, data: Union[SenseData, LinkageData]

2997) -> None:

2998 """Parses tags or topics for a sense or some other data. The values are

2999 added into the dictionary ``data``."""

3000 assert isinstance(wxr, WiktextractContext)

3001 assert isinstance(text, str)

3002 assert isinstance(data, dict)

3003 # print("parse_sense_qualifier:", text)

3004 if re.match(r"$[^()]+$$", text): 3004 ↛ 3005line 3004 didn't jump to line 3005 because the condition on line 3004 was never true

3005 text = text[1:-1]

3006 if re.match(r'"[^"]+"$', text): 3006 ↛ 3007line 3006 didn't jump to line 3007 because the condition on line 3006 was never true

3007 text = text[1:-1]

3008 lst = map_with(xlat_descs_map, [text])

3009 sense_tags: list[str] = []

3010 for text in lst:

3011 for semi in split_at_comma_semi(text):

3012 if not semi: 3012 ↛ 3013line 3012 didn't jump to line 3013 because the condition on line 3012 was never true

3013 continue

3014 orig_semi = semi

3015 idx = semi.find(":")

3016 if idx >= 0: 3016 ↛ 3017line 3016 didn't jump to line 3017 because the condition on line 3016 was never true

3017 semi = semi[:idx]

3018 cls = classify_desc(semi, allow_unknown_tags=True)

3019 # print("parse_sense_qualifier: classify_desc: {} -> {}"

3020 # .format(semi, cls))

3021 if cls == "tags":

3022 tagsets, topics = decode_tags(semi)

3023 data_extend(data, "topics", topics)

3024 # XXX should think how to handle distinct options better,

3025 # e.g., "singular and plural genitive"; that can't really be

3026 # done with changing the calling convention of this function.

3027 # Should split sense if more than one category of tags differs.

3028 for tags in tagsets:

3029 sense_tags.extend(tags)

3030 elif cls == "taxonomic": 3030 ↛ 3031line 3030 didn't jump to line 3031 because the condition on line 3030 was never true

3031 if re.match(r"×[A-Z]", semi):

3032 sense_tags.append("extinct")

3033 semi = semi[1:]

3034 data["taxonomic"] = semi

3035 elif cls == "english":

3036 if "qualifier" in data and data["qualifier"] != orig_semi: 3036 ↛ 3037line 3036 didn't jump to line 3037 because the condition on line 3036 was never true

3037 data["qualifier"] += "; " + orig_semi

3038 else:

3039 data["qualifier"] = orig_semi

3040 else:

3041 wxr.wtp.debug(

3042 "unrecognized sense qualifier: {}".format(text),

3043 sortid="form_descriptions/1831",

3044 )

3045 sense_tags = sorted(set(sense_tags))

3046 data_extend(data, "tags", sense_tags)

3047

3048

3049def parse_pronunciation_tags(

3050 wxr: WiktextractContext, text: str, data: SoundData

3051) -> None:

3052 assert isinstance(wxr, WiktextractContext)

3053 assert isinstance(text, str)

3054 assert isinstance(data, dict)

3055 text = text.strip()

3056 if not text:

3057 return

3058 cls = classify_desc(text)

3059 notes = []

3060 if cls == "tags":

3061 tagsets, topics = decode_tags(text)

3062 data_extend(data, "topics", topics)

3063 for tagset in tagsets:

3064 for t in tagset:

3065 if " " in t: 3065 ↛ 3066line 3065 didn't jump to line 3066 because the condition on line 3065 was never true

3066 notes.append(t)

3067 else:

3068 data_append(data, "tags", t)

3069 else:

3070 notes.append(text)

3071 if notes:

3072 data["note"] = "; ".join(notes)

3073

3074

3075def parse_translation_desc(

3076 wxr: WiktextractContext, lang: str, text: str, tr: TranslationData

3077) -> None:

3078 assert isinstance(wxr, WiktextractContext)

3079 assert isinstance(lang, str) # The language of ``text``

3080 assert isinstance(text, str)

3081 assert isinstance(tr, dict)

3082 # print("parse_translation_desc:", text)

3083

3084 # Process all parenthesized parts from the translation item

3085 note = None

3086 restore_beginning = ""

3087 restore_end = ""

3088 while True:

3089 beginning = False

3090 # See if we can find a parenthesized expression at the end

3091 m = re.search(r"\s*$(([^()]|\([^()]+$)+)\)\.?$", text)

3092 if m:

3093 par = m.group(1)

3094 text = text[: m.start()]

3095 if par.startswith(("literally ", "lit.")):

3096 continue # Not useful for disambiguation in many idioms

3097 else:

3098 # See if we can find a parenthesized expression at the start

3099 m = re.match(r"^\^?$(([^()]|\([^()]+$)+)\):?(\s+|$)", text)

3100 if m:

3101 par = m.group(1)

3102 text = text[m.end() :]

3103 beginning = True

3104 if re.match(r"^(\d|\s|,| or | and )+$", par): 3104 ↛ 3109line 3104 didn't jump to line 3109 because the condition on line 3104 was never true

3105 # Looks like this beginning parenthesized expression only

3106 # contains digits or their combinations. We assume such

3107 # to be sense descriptions if no sense has been selected,

3108 # or otherwise just ignore them.

3109 if not tr.get("sense"):

3110 tr["sense"] = par

3111 continue

3112 else:

3113 # See if we can find a parenthesized expression in the middle.

3114 # Romanizations are sometimes between word and gender marker,

3115 # e.g. wife/English/Tr/Yiddish.

3116 m = re.search(r"\s+$(([^()]|\([^()]+$)+)\)", text)

3117 if m:

3118 par = m.group(1)

3119 text = text[: m.start()] + text[m.end() :]

3120 else:

3121 # No more parenthesized expressions - break out of the loop

3122 break

3123

3124 # Some cleanup of artifacts that may result from skipping some templates

3125 # in earlier stages

3126 if par.startswith(": "): 3126 ↛ 3127line 3126 didn't jump to line 3127 because the condition on line 3126 was never true

3127 par = par[2:]

3128 if par.endswith(","): 3128 ↛ 3129line 3128 didn't jump to line 3129 because the condition on line 3128 was never true

3129 par = par[:-1]

3130 if re.match(r'^[“"]([^“”"]*)[“”"]$', par): 3130 ↛ 3131line 3130 didn't jump to line 3131 because the condition on line 3130 was never true

3131 par = par[1:-1]

3132 par = par.strip()

3133

3134 # Check for special script pronunciation followed by romanization,

3135 # used in many Asian languages.

3136 lst = par.split(", ")

3137 if len(lst) == 2:

3138 a, r = lst

3139 if classify_desc(a) == "other":

3140 cls = classify_desc(r)

3141 # print("parse_translation_desc: r={} cls={}".format(r, cls))

3142 if cls == "romanization" or (

3143 cls == "english" and len(r.split()) == 1 and r[0].islower()

3144 ):

3145 if tr.get("alt") and tr.get("alt") != a: 3145 ↛ 3146line 3145 didn't jump to line 3146 because the condition on line 3145 was never true

3146 wxr.wtp.debug(

3147 'more than one value in "alt": {} vs. {}'.format(

3148 tr["alt"], a

3149 ),

3150 sortid="form_descriptions/1930",

3151 )

3152 tr["alt"] = a

3153 if tr.get("roman") and tr.get("roman") != r: 3153 ↛ 3154line 3153 didn't jump to line 3154 because the condition on line 3153 was never true

3154 wxr.wtp.debug(

3155 'more than one value in "roman": {} vs. {}'.format(

3156 tr["roman"], r

3157 ),

3158 sortid="form_descriptions/1936",

3159 )

3160 tr["roman"] = r

3161 continue

3162

3163 # Check for certain comma-separated tags combined with English text

3164 # at the beginning or end of a comma-separated parenthesized list

3165 while len(lst) > 1:

3166 cls = classify_desc(lst[0])

3167 if cls == "tags": 3167 ↛ 3168line 3167 didn't jump to line 3168 because the condition on line 3167 was never true

3168 tagsets, topics = decode_tags(lst[0])

3169 for t in tagsets:

3170 data_extend(tr, "tags", t)

3171 data_extend(tr, "topics", topics)

3172 lst = lst[1:]

3173 continue

3174 cls = classify_desc(lst[-1])

3175 if cls == "tags":

3176 tagsets, topics = decode_tags(lst[-1])

3177 for t in tagsets:

3178 data_extend(tr, "tags", t)

3179 data_extend(tr, "topics", topics)

3180 lst = lst[:-1]

3181 continue

3182 break

3183 par = ", ".join(lst)

3184

3185 if not par: 3185 ↛ 3186line 3185 didn't jump to line 3186 because the condition on line 3185 was never true

3186 continue

3187 if re.search(tr_ignored_parens_re, par): 3187 ↛ 3188line 3187 didn't jump to line 3188 because the condition on line 3187 was never true

3188 continue

3189 if par.startswith("numeral:"):

3190 par = par[8:].strip()

3191

3192 # Classify the part in parenthesis and process accordingly

3193 cls = classify_desc(par)

3194 # print("parse_translation_desc classify: {!r} -> {}"

3195 # .format(par, cls))

3196 if par == text:

3197 pass

3198 if par == "f": 3198 ↛ 3199line 3198 didn't jump to line 3199 because the condition on line 3198 was never true

3199 data_append(tr, "tags", "feminine")

3200 elif par == "m": 3200 ↛ 3201line 3200 didn't jump to line 3201 because the condition on line 3200 was never true

3201 data_append(tr, "tags", "masculine")

3202 elif cls == "tags":

3203 tagsets, topics = decode_tags(par)

3204 for tags in tagsets:

3205 data_extend(tr, "tags", tags)

3206 data_extend(tr, "topics", topics)

3207 elif cls == "english":

3208 # If the text contains any of certain grammatical words, treat it

3209 # as a "note" instead of "english"

3210 if re.search(tr_note_re, par):

3211 if par.endswith(":"): 3211 ↛ 3212line 3211 didn't jump to line 3212 because the condition on line 3211 was never true

3212 par = par[:-1]

3213 if par not in ("see entry for forms",): 3213 ↛ 3088line 3213 didn't jump to line 3088 because the condition on line 3213 was always true

3214 if note: 3214 ↛ 3215line 3214 didn't jump to line 3215 because the condition on line 3214 was never true

3215 note = note + ";" + par

3216 else:

3217 note = par

3218 else:

3219 # There can be more than one parenthesized english item, see

3220 # e.g. Aunt/English/Translations/Tamil

3221 if "translation" in tr and "english" in tr:

3222 tr["english"] += "; " + par # DEPRECATED for "translation"

3223 tr["translation"] += "; " + par

3224 else:

3225 tr["english"] = par # DEPRECATED for "translation"

3226 tr["translation"] = par

3227 elif cls == "romanization":

3228 # print("roman text={!r} text cls={}"

3229 # .format(text, classify_desc(text)))

3230 if classify_desc(text) in (

3231 "english",

3232 "romanization",

3233 ) and lang not in ("Egyptian",):

3234 if beginning:

3235 restore_beginning += "({}) ".format(par)

3236 else:

3237 restore_end = " ({})".format(par) + restore_end

3238 else:

3239 if tr.get("roman"): 3239 ↛ 3240line 3239 didn't jump to line 3240 because the condition on line 3239 was never true

3240 wxr.wtp.debug(

3241 'more than one value in "roman": {} vs. {}'.format(

3242 tr["roman"], par

3243 ),

3244 sortid="form_descriptions/2013",

3245 )

3246 tr["roman"] = par

3247 elif cls == "taxonomic": 3247 ↛ 3248line 3247 didn't jump to line 3248 because the condition on line 3247 was never true

3248 if tr.get("taxonomic"):

3249 wxr.wtp.debug(

3250 'more than one value in "taxonomic": {} vs. {}'.format(

3251 tr["taxonomic"], par

3252 ),

3253 sortid="form_descriptions/2019",

3254 )

3255 if re.match(r"×[A-Z]", par):

3256 data_append(tr, "tags", "extinct")

3257 par = par[1:]

3258 tr["taxonomic"] = par

3259 elif cls == "other": 3259 ↛ 3269line 3259 didn't jump to line 3269 because the condition on line 3259 was always true

3260 if tr.get("alt"): 3260 ↛ 3261line 3260 didn't jump to line 3261 because the condition on line 3260 was never true

3261 wxr.wtp.debug(

3262 'more than one value in "alt": {} vs. {}'.format(

3263 tr["alt"], par

3264 ),

3265 sortid="form_descriptions/2028",

3266 )

3267 tr["alt"] = par

3268 else:

3269 wxr.wtp.debug(

3270 "parse_translation_desc unimplemented cls {}: {}".format(

3271 cls, par

3272 ),

3273 sortid="form_descriptions/2033",

3274 )

3275

3276 # Check for gender indications in suffix

3277 text, final_tags = parse_head_final_tags(wxr, lang, text)

3278 data_extend(tr, "tags", final_tags)

3279

3280 # Restore those parts that we did not want to remove (they are often

3281 # optional words or words that are always used with the given translation)

3282 text = restore_beginning + text + restore_end

3283

3284 if note:

3285 tr["note"] = note.strip()

3286 if text and text not in ignored_translations:

3287 tr["word"] = text.strip()

3288

3289 # Sometimes gender seems to be at the end of "roman" field, see e.g.

3290 # fire/English/Noun/Translations/Egyptian (for "oxidation reaction")

3291 roman = tr.get("roman")

3292 if roman:

3293 if roman.endswith(" f"): 3293 ↛ 3294line 3293 didn't jump to line 3294 because the condition on line 3293 was never true

3294 data_append(tr, "tags", "feminine")

3295 tr["roman"] = roman[:-2].strip()

3296 elif roman.endswith(" m"): 3296 ↛ 3297line 3296 didn't jump to line 3297 because the condition on line 3296 was never true

3297 data_append(tr, "tags", "masculine")

3298 tr["roman"] = roman[:-2].strip()

3299

3300 # If the word now has "translation" field but no "roman" field, and

3301 # the word would be classified "other" (generally non-latin

3302 # characters), and the value in "translation" is only one lowercase

3303 # word, move it to "roman". This happens semi-frequently when the

3304 # translation is transliterated the same as some English word.

3305 roman = tr.get("roman")

3306 english = tr.get("translation")

3307 if english and not roman and "word" in tr:

3308 cls = classify_desc(tr["word"])

3309 if cls == "other" and " " not in english and english[0].islower():

3310 del tr["translation"]

3311 if "english" in tr: # DEPRECATED for "translation" 3311 ↛ 3313line 3311 didn't jump to line 3313 because the condition on line 3311 was always true

3312 del tr["english"]

3313 tr["roman"] = english

3314

3315 # If the entry now has both tr["roman"] and tr["word"] and they have

3316 # the same value, delete tr["roman"] (e.g., man/English/Translations

3317 # Evenki)

3318 if tr.get("word") and tr.get("roman") == tr.get("word"): 3318 ↛ 3319line 3318 didn't jump to line 3319 because the condition on line 3318 was never true

3319 del tr["roman"]

3320

3321

3322def parse_alt_or_inflection_of(

3323 wxr: WiktextractContext, gloss: str, gloss_template_args: set[str]

3324) -> Optional[tuple[list[str], Optional[list[AltOf]]]]:

3325 """Tries to parse an inflection-of or alt-of description. If successful,

3326 this returns (tags, alt-of/inflection-of-dict). If the description cannot

3327 be parsed, this returns None. This may also return (tags, None) when the

3328 gloss describes a form (or some other tags were extracted from it), but

3329 there was no alt-of/form-of/synonym-of word."""

3330 # print("parse_alt_or_inflection_of: {!r}".format(gloss))

3331 # Occasionally inflection_of/alt_of have "A(n) " etc. at the beginning.

3332

3333 # Never interpret a gloss that is equal to the word itself as a tag

3334 # (e.g., instrumental/Romanian, instrumental/Spanish).

3335 if gloss.lower() == wxr.wtp.title.lower() or ( # type:ignore[union-attr]

3336 len(gloss) >= 5 and distw([gloss.lower()], wxr.wtp.title.lower()) < 0.2 # type:ignore[union-attr]

3337 ):

3338 return None

3339

3340 # First try parsing it as-is

3341 parsed = parse_alt_or_inflection_of1(wxr, gloss, gloss_template_args)

3342 if parsed is not None:

3343 return parsed

3344

3345 # Next try parsing it with the first character converted to lowercase if

3346 # it was previously uppercase.

3347 if gloss and gloss[0].isupper():

3348 gloss = gloss[0].lower() + gloss[1:]

3349 parsed = parse_alt_or_inflection_of1(wxr, gloss, gloss_template_args)

3350 if parsed is not None:

3351 return parsed

3352

3353 return None

3354

3355

3356# These tags are not allowed in alt-or-inflection-of parsing

3357alt_infl_disallowed: set[str] = set(

3358 [

3359 "error-unknown-tag",

3360 "place", # Not in inflected forms and causes problems e.g. house/

3361 # English

3362 ]

3363)

3364

3365

3366def parse_alt_or_inflection_of1(

3367 wxr: WiktextractContext, gloss: str, gloss_template_args: set[str]

3368) -> Optional[tuple[list[str], Optional[list[AltOf]]]]:

3369 """Helper function for parse_alt_or_inflection_of. This handles a single

3370 capitalization."""

3371 if not gloss or not gloss.strip(): 3371 ↛ 3372line 3371 didn't jump to line 3372 because the condition on line 3371 was never true

3372 return None

3373

3374 # Prevent some common errors where we would parse something we shouldn't

3375 if re.search(r"(?i)form of address ", gloss): 3375 ↛ 3376line 3375 didn't jump to line 3376 because the condition on line 3375 was never true

3376 return None

3377

3378 gloss = re.sub(r"only used in [^,]+, ", "", gloss)

3379

3380 # First try all formats ending with "of" (or other known last words that

3381 # can end a form description)

3382 matches = list(re.finditer(r"\b(of|for|by|as|letter|number) ", gloss))

3383 m: Optional[re.Match]

3384 for m in reversed(matches):

3385 desc = gloss[: m.end()].strip()

3386 base = gloss[m.end() :].strip()

3387 tagsets, topics = decode_tags(desc, no_unknown_starts=True)

3388 if not topics and any(

3389 not (alt_infl_disallowed & set(ts)) for ts in tagsets

3390 ):

3391 # Successfully parsed, including "of" etc.

3392 tags: list[str] = []

3393 # If you have ("Western-Armenian", ..., "form-of") as your

3394 # tag set, it's most probable that it's something like

3395 # "Western Armenian form of խոսել (xosel)", which should

3396 # get "alt-of" instead of "form-of" (inflection).

3397 # խօսիլ/Armenian

3398 for ts_t in tagsets:

3399 if "form-of" in ts_t and any(

3400 valid_tags.get(tk) == "dialect" for tk in ts_t

3401 ):

3402 ts_s = (set(ts_t) - {"form-of"}) | {"alt-of"}

3403 else:

3404 ts_s = set(ts_t)

3405 if not (alt_infl_disallowed & ts_s): 3405 ↛ 3398line 3405 didn't jump to line 3398 because the condition on line 3405 was always true

3406 tags.extend(ts_s)

3407 if (

3408 "alt-of" in tags

3409 or "form-of" in tags

3410 or "synonym-of" in tags

3411 or "compound-of" in tags

3412 ):

3413 break

3414 if m.group(1) == "of":

3415 # Try parsing without the final "of". This is commonly used in

3416 # various form-of expressions.

3417 desc = gloss[: m.start()]

3418 base = gloss[m.end() :]

3419 tagsets, topics = decode_tags(desc, no_unknown_starts=True)

3420 # print("ALT_OR_INFL: desc={!r} base={!r} tagsets={} topics={}"

3421 # .format(desc, base, tagsets, topics))

3422 if not topics and any(

3423 not (alt_infl_disallowed & set(t)) for t in tagsets

3424 ):

3425 tags = []

3426 for t in tagsets:

3427 if not (alt_infl_disallowed & set(t)): 3427 ↛ 3426line 3427 didn't jump to line 3426 because the condition on line 3427 was always true

3428 tags.extend(t)

3429 # It must have at least one tag from form_of_tags

3430 if set(tags) & form_of_tags:

3431 # Accept this as form-of

3432 tags.append("form-of")

3433 break

3434 if set(tags) & alt_of_tags:

3435 # Accept this as alt-of

3436 tags.append("alt-of")

3437 break

3438

3439 else:

3440 # Did not find a form description based on last word; see if the

3441 # whole description is tags

3442 tagsets, topics = decode_tags(gloss, no_unknown_starts=True)

3443 if not topics and any(

3444 not (alt_infl_disallowed & set(ts)) and form_of_tags & set(ts)

3445 for ts in tagsets

3446 ):

3447 tags = []

3448 for ts in tagsets:

3449 if not (alt_infl_disallowed & set(ts)) and form_of_tags & set( 3449 ↛ 3448line 3449 didn't jump to line 3448 because the condition on line 3449 was always true

3450 ts

3451 ):

3452 tags.extend(ts)

3453 base = ""

3454 else:

3455 return None

3456

3457 # kludge for Spanish (again): 'x of [word] combined with [clitic]'

3458 m = re.search(r"combined with \w+$", base)

3459 if m: 3459 ↛ 3460line 3459 didn't jump to line 3460 because the condition on line 3459 was never true

3460 tagsets, topics = decode_tags(m.group(0), no_unknown_starts=True)

3461 if not topics:

3462 for ts in tagsets:

3463 tags.extend(ts)

3464 base = base[: m.start()]

3465

3466 # It is fairly common for form_of glosses to end with something like

3467 # "ablative case" or "in instructive case". Parse that ending.

3468 base = base.strip()

3469 lst = base.split()

3470 # print("parse_alt_or_inflection_of: lst={}".format(lst))

3471 if len(lst) >= 3 and lst[-1] in ("case", "case."): 3471 ↛ 3472line 3471 didn't jump to line 3472 because the condition on line 3471 was never true

3472 node = valid_sequences.children.get(lst[-2])

3473 if node and node.end:

3474 for s in node.tags:

3475 tags.extend(s.split(" "))

3476 lst = lst[:-2]

3477 if lst[-1] == "in" and len(lst) > 1:

3478 lst = lst[:-1]

3479

3480 # Eliminate empty and duplicate tags

3481 tags = sorted(set(t for t in tags if t))

3482

3483 # Clean up some extra stuff from the linked word, separating the text

3484 # into ``base`` (the linked word) and ``extra`` (additional information,

3485 # such as English translation or clarifying word sense information).

3486 orig_base = base

3487 base = re.sub(alt_of_form_of_clean_re, "", orig_base)

3488 base = re.sub(r" [(⟨][^()]*[)⟩]", "", base) # Remove all (...) groups

3489 extra = orig_base[len(base) :]

3490 extra = re.sub(r"^[- :;.,，—]+", "", extra)

3491 if extra.endswith(".") and extra.count(".") == 1:

3492 extra = extra[:-1].strip()

3493 m = re.match(r"^$([^()]*)$$", extra)

3494 if m: 3494 ↛ 3495line 3494 didn't jump to line 3495 because the condition on line 3494 was never true

3495 extra = m.group(1)

3496 else:

3497 # These weird backets used in "slash mark"

3498 m = re.match(r"^⟨([^()]*)⟩$", extra)

3499 if m: 3499 ↛ 3500line 3499 didn't jump to line 3500 because the condition on line 3499 was never true

3500 extra = m.group(1)

3501 m = re.match(r'^[“"]([^"“”]*)["”]$', extra)

3502 if m: 3502 ↛ 3503line 3502 didn't jump to line 3503 because the condition on line 3502 was never true

3503 extra = m.group(1)

3504 # Note: base might still contain comma-separated values and values

3505 # separated by "and"

3506 base = base.strip()

3507 if base.endswith(",") and len(base) > 2: 3507 ↛ 3508line 3507 didn't jump to line 3508 because the condition on line 3507 was never true

3508 base = base[:-1].strip()

3509 while (

3510 base.endswith(".")

3511 and not wxr.wtp.page_exists(base)

3512 and base not in gloss_template_args

3513 ):

3514 base = base[:-1].strip()

3515 if base.endswith('(\u201cconjecture")'): 3515 ↛ 3516line 3515 didn't jump to line 3516 because the condition on line 3515 was never true

3516 base = base[:-14].strip()

3517 tags.append("conjecture")

3518 while ( 3518 ↛ 3523line 3518 didn't jump to line 3523 because the condition on line 3518 was never true

3519 base.endswith(".")

3520 and not wxr.wtp.page_exists(base)

3521 and base not in gloss_template_args

3522 ):

3523 base = base[:-1].strip()

3524 if ( 3524 ↛ 3529line 3524 didn't jump to line 3529 because the condition on line 3524 was never true

3525 base.endswith(".")

3526 and base not in gloss_template_args

3527 and base[:-1] in gloss_template_args

3528 ):

3529 base = base[:-1]

3530 base = base.strip()

3531 if not base:

3532 return tags, None

3533

3534 # Kludge: Spanish verb forms seem to have a dot added at the end.

3535 # Remove it; we know of no Spanish verbs ending with a dot.

3536 language = wxr.wtp.section

3537 pos = wxr.wtp.subsection

3538 # print("language={} pos={} base={}".format(language, pos, base))

3539 if ( 3539 ↛ 3545line 3539 didn't jump to line 3545 because the condition on line 3539 was never true

3540 base.endswith(".")

3541 and len(base) > 1

3542 and base[-2].isalpha()

3543 and (language == "Spanish" and pos == "Verb")

3544 ):

3545 base = base[:-1]

3546

3547 # Split base to alternatives when multiple alternatives provided

3548 parts = split_at_comma_semi(base, extra=[" / ", "／", r" \+ "])

3549 titleword = re.sub(r"^Reconstruction:[^/]*/", "", wxr.wtp.title or "")

3550 if (

3551 len(parts) <= 1

3552 or base.startswith("/")

3553 or base.endswith("/")

3554 or "/" in titleword

3555 ):

3556 parts = [base]

3557 # Split base to alternatives when of form "a or b" and "a" and "b" are

3558 # similar (generally spelling variants of the same word or similar words)

3559 if len(parts) == 1:

3560 pp = base.split()

3561 if len(pp) == 3 and pp[1] == "or" and distw([pp[0]], pp[2]) < 0.4:

3562 parts = [pp[0], pp[2]]

3563

3564 # Create form-of/alt-of entries based on the extracted data

3565 dt_lst: list[AltOf] = []

3566 for p in parts:

3567 # Check for some suspicious base forms

3568 m = re.search(r"[.,] |[{}()]", p)

3569 if m and not wxr.wtp.page_exists(p): 3569 ↛ 3570line 3569 didn't jump to line 3570 because the condition on line 3569 was never true

3570 wxr.wtp.debug(

3571 "suspicious alt_of/form_of with {!r}: {}".format(m.group(0), p),

3572 sortid="form_descriptions/2278",

3573 )

3574 if p.startswith("*") and len(p) >= 3 and p[1].isalpha(): 3574 ↛ 3575line 3574 didn't jump to line 3575 because the condition on line 3574 was never true

3575 p = p[1:]

3576 dt: AltOf = {"word": p}

3577 if extra:

3578 dt["extra"] = extra

3579 dt_lst.append(dt)

3580 # print("alt_or_infl_of returning tags={} lst={} base={!r}"

3581 # .format(tags, lst, base))

3582 return tags, dt_lst

3583

3584

3585@functools.lru_cache(maxsize=65536)

3586def classify_desc(

3587 desc: str,

3588 allow_unknown_tags=False,

3589 no_unknown_starts=False,

3590 accepted: Union[tuple[str, ...], frozenset[str]] = tuple(),

3591) -> str:

3592 """Determines whether the given description is most likely tags, english,

3593 a romanization, or something else. Returns one of: "tags", "english",

3594 "romanization", or "other". If ``allow_unknown_tags`` is True, then

3595 allow "tags" classification even when the only tags are those starting

3596 with a word in allowed_unknown_starts."""

3597 assert isinstance(desc, str)

3598 # Empty and whitespace-only strings are treated as "other"

3599 desc = desc.strip()

3600 if not desc:

3601 return "other"

3602

3603 normalized_desc = unicodedata.normalize("NFKD", desc)

3604

3605 # If it can be fully decoded as tags without errors, treat as tags

3606 tagsets, topics = decode_tags(desc, no_unknown_starts=no_unknown_starts)

3607 for tagset in tagsets:

3608 assert isinstance(tagset, (list, tuple, set))

3609 if "error-unknown-tag" not in tagset and (

3610 topics or allow_unknown_tags or any(" " not in x for x in tagset)

3611 ):

3612 return "tags"

3613

3614 # Check if it looks like the taxonomic name of a species

3615 if desc in known_species:

3616 return "taxonomic"

3617 desc1 = re.sub(r"^×([A-Z])", r"\1", desc)

3618 desc1 = re.sub(r"\s*×.*", "", desc1)

3619 lst = desc1.split()

3620 if len(lst) > 1 and len(lst) <= 5 and lst[0] in known_firsts:

3621 have_non_english = 1 if lst[0].lower() not in english_words else 0

3622 for x in lst[1:]:

3623 if x in ("A", "B", "C", "D", "E", "F", "I", "II", "III", "IV", "V"):

3624 continue

3625 if x[0].isupper():

3626 break

3627 if x not in english_words:

3628 have_non_english += 1

3629 else:

3630 # Starts with known taxonomic term, does not contain uppercase

3631 # words (except allowed letters) and at least one word is not

3632 # English

3633 if have_non_english >= len(lst) - 1 and have_non_english > 0: 3633 ↛ 3639line 3633 didn't jump to line 3639 because the condition on line 3633 was always true

3634 return "taxonomic"

3635

3636 # If all words are in our English dictionary, interpret as English.

3637 # [ -~] is regex black magic, "ALL CHARACTERS from space to tilde"

3638 # in ASCII. Took me a while to figure out.

3639 if re.match(r"[ -~―—“”…'‘’ʹ€]+$", normalized_desc) and len(desc) > 1:

3640 if desc in english_words and desc[0].isalpha():

3641 return "english" # Handles ones containing whitespace

3642 desc1 = re.sub(

3643 tokenizer_fixup_re, lambda m: tokenizer_fixup_map[m.group(0)], desc

3644 )

3645 tokens = tokenizer.tokenize(desc1)

3646 if not tokens: 3646 ↛ 3647line 3646 didn't jump to line 3647 because the condition on line 3646 was never true

3647 return "other"

3648 lst_bool = list(

3649 x not in not_english_words

3650 and

3651 # not x.isdecimal() and

3652 (

3653 x in english_words

3654 or x.lower() in english_words

3655 or x in known_firsts

3656 or x[0].isdigit()

3657 or x in accepted

3658 or

3659 # (x[0].isupper() and x.find("-") < 0 and x.isascii()) or

3660 (

3661 x.endswith("s") and len(x) >= 4 and x[:-1] in english_words

3662 ) # Plural

3663 or (

3664 x.endswith("ies")

3665 and len(x) >= 5

3666 and x[:-3] + "y" in english_words

3667 ) # E.g. lily - lilies

3668 or (

3669 x.endswith("ing")

3670 and len(x) >= 5

3671 and x[:-3] in english_words

3672 ) # E.g. bring - bringing

3673 or (

3674 x.endswith("ing")

3675 and len(x) >= 5

3676 and x[:-3] + "e" in english_words

3677 ) # E.g., tone - toning

3678 or (

3679 x.endswith("ed") and len(x) >= 5 and x[:-2] in english_words

3680 ) # E.g. hang - hanged

3681 or (

3682 x.endswith("ed")

3683 and len(x) >= 5

3684 and x[:-2] + "e" in english_words

3685 ) # E.g. atone - atoned

3686 or (x.endswith("'s") and x[:-2] in english_words)

3687 or (x.endswith("s'") and x[:-2] in english_words)

3688 or (

3689 x.endswith("ise")

3690 and len(x) >= 5

3691 and x[:-3] + "ize" in english_words

3692 )

3693 or (

3694 x.endswith("ised")

3695 and len(x) >= 6

3696 and x[:-4] + "ized" in english_words

3697 )

3698 or (

3699 x.endswith("ising")

3700 and len(x) >= 7

3701 and x[:-5] + "izing" in english_words

3702 )

3703 or (

3704 re.search(r"[-/]", x)

3705 and all(

3706 ((y in english_words and len(y) > 2) or not y)

3707 for y in re.split(r"[-/]", x)

3708 )

3709 )

3710 )

3711 for x in tokens

3712 )

3713 cnt = lst_bool.count(True)

3714 rejected_words = tuple(

3715 x for i, x in enumerate(tokens) if not lst_bool[i]

3716 )

3717 if (

3718 any(

3719 lst_bool[i] and x[0].isalpha() and len(x) > 1

3720 for i, x in enumerate(tokens)

3721 )

3722 and not desc.startswith("-")

3723 and not desc.endswith("-")

3724 and re.search(r"\w+", desc)

3725 and (

3726 cnt == len(lst_bool)

3727 or (

3728 any(

3729 lst_bool[i] and len(x) > 3 for i, x in enumerate(tokens)

3730 )

3731 and cnt >= len(lst_bool) - 1

3732 )

3733 or cnt / len(lst_bool) >= 0.8

3734 or (

3735 all(x in potentially_english_words for x in rejected_words)

3736 and cnt / len(lst_bool) >= 0.50

3737 )

3738 )

3739 ):

3740 return "english"

3741 # Some translations have apparent pronunciation descriptions in /.../

3742 # which we'll put in the romanization field (even though they probably are

3743 # not exactly romanizations).

3744 if desc.startswith("/") and desc.endswith("/"):

3745 return "romanization"

3746 # If all characters are in classes that could occur in romanizations,

3747 # treat as romanization

3748 classes = list(

3749 unicodedata.category(x) if x not in ("-", ",", ":", "/", '"') else "OK"

3750 for x in normalized_desc

3751 )

3752 classes1 = []

3753 num_latin = 0

3754 num_greek = 0

3755 # part = ""

3756 # for ch, cl in zip(normalized_desc, classes):

3757 # part += f"{ch}({cl})"

3758 # print(part)

3759 for ch, cl in zip(normalized_desc, classes):

3760 if ch in (

3761 "'", # ' in Arabic, / in IPA-like parenthesized forms

3762 ".", # e.g., "..." in translations

3763 ";",

3764 ":",

3765 "!",

3766 "‘",

3767 "’",

3768 '"',

3769 "“",

3770 "”",

3771 "/",

3772 "?",

3773 "…", # alternative to "..."

3774 "⁉", # 見る/Japanese automatic transcriptions...

3775 "？",

3776 "！",

3777 "⁻", # superscript -, used in some Cantonese roman, e.g. "we"

3778 "ʔ",

3779 "ʼ",

3780 "ʾ",

3781 "ʹ",

3782 ): # ʹ e.g. in understand/English/verb Russian transl

3783 classes1.append("OK")

3784 continue

3785 if cl not in ("Ll", "Lu"):

3786 classes1.append(cl)

3787 continue

3788 try:

3789 name = unicodedata.name(ch)

3790 first = name.split()[0]

3791 if first == "LATIN":

3792 num_latin += 1

3793 elif first == "GREEK":

3794 num_greek += 1

3795 elif first == "COMBINING": # Combining diacritic 3795 ↛ 3796line 3795 didn't jump to line 3796 because the condition on line 3795 was never true

3796 cl = "OK"

3797 elif re.match(non_latin_scripts_re, name): 3797 ↛ 3801line 3797 didn't jump to line 3801 because the condition on line 3797 was always true

3798 cl = "NO" # Not acceptable in romanizations

3799 except ValueError:

3800 cl = "NO" # Not acceptable in romanizations

3801 classes1.append(cl)

3802 # print("classify_desc: {!r} classes1: {}".format(desc, classes1))

3803 # print(set(classes1) )

3804 if all(

3805 x in ("Ll", "Lu", "Lt", "Lm", "Mn", "Mc", "Zs", "Nd", "OK")

3806 for x in classes1

3807 ):

3808 if (

3809 (num_latin >= num_greek + 2 or num_greek == 0)

3810 and classes1.count("OK") < len(classes1)

3811 and classes1.count("Nd") < len(classes1)

3812 ):

3813 return "romanization"

3814 # Otherwise it is something else, such as hanji version of the word

3815 return "other"

3816

3817

3818def remove_text_in_parentheses(text: str) -> str:

3819 parentheses = 0

3820 new_text = ""

3821 for c in text:

3822 if c == "(":

3823 parentheses += 1

3824 elif c == ")":

3825 parentheses -= 1

3826 elif parentheses == 0:

3827 new_text += c

3828 return new_text

Coverage for src/wiktextract/extractor/en/form_descriptions.py: 79%

1424 statements