Coverage for src/wiktextract/extractor/en/form_descriptions.py: 79%

1442 statements  

« prev     ^ index     » next       coverage.py v7.17.2, created at 2026-11-12 07:03 +0000

1# Code for parsing linguistic form descriptions and tags for word senses 

2# (both the word entry head - initial part and parenthesized parts - 

3# and tags at the beginning of word senses) 

4# 

5# Copyright (c) 2020-2022 Tatu Ylonen. See file LICENSE and https://ylonen.org 

6 

7import functools 

8import re 

9import unicodedata 

10from typing import ( 

11 Any, 

12 Literal, 

13 Optional, 

14 Sequence, 

15 Union, 

16) 

17 

18import Levenshtein 

19from nltk import TweetTokenizer # type:ignore[import-untyped] 

20from wikitextprocessor.parser import WikiNode 

21 

22from ...datautils import data_append, data_extend, split_at_comma_semi 

23from ...page import extract_links_from_node 

24from ...tags import ( 

25 alt_of_tags, 

26 form_of_tags, 

27 head_final_bantu_langs, 

28 head_final_bantu_map, 

29 head_final_numeric_langs, 

30 head_final_other_langs, 

31 head_final_other_map, 

32 head_final_semitic_langs, 

33 head_final_semitic_map, 

34 uppercase_tags, 

35 valid_tags, 

36 xlat_descs_map, 

37 xlat_head_map, 

38 xlat_tags_map, 

39) 

40from ...topics import topic_generalize_map, valid_topics 

41from ...wxr_context import WiktextractContext 

42from .english_words import ( 

43 english_words, 

44 not_english_words, 

45 potentially_english_words, 

46) 

47from .form_descriptions_known_firsts import known_firsts 

48from .taxondata import known_species 

49from .type_utils import ( 

50 AltOf, 

51 FormData, 

52 LinkageData, 

53 SenseData, 

54 SoundData, 

55 TemplateData, 

56 TranslationData, 

57 WordData, 

58) 

59 

60# Tokenizer for classify_desc() 

61tokenizer = TweetTokenizer() 

62 

63# These are ignored as the value of a related form in form head. 

64IGNORED_RELATED: set[str] = set( 

65 [ 

66 "-", 

67 "־", 

68 "᠆", 

69 "‐", 

70 "‑", 

71 "‒", 

72 "–", 

73 "—", 

74 "―", 

75 "−", 

76 "⸺", 

77 "⸻", 

78 "﹘", 

79 "﹣", 

80 "-", 

81 "?", 

82 "(none)", 

83 ] 

84) 

85 

86 

87# First words of unicodedata.name() that indicate scripts that cannot be 

88# accepted in romanizations or english (i.e., should be considered "other" 

89# in classify_desc()). 

90non_latin_scripts: list[str] = [ 

91 "ADLAM", 

92 "ARABIC", 

93 "ARABIC-INDIC", 

94 "ARMENIAN", 

95 "BALINESE", 

96 "BENGALI", 

97 "BRAHMI", 

98 "BRAILLE", 

99 "CANADIAN", 

100 "CHAKMA", 

101 "CHAM", 

102 "CHEROKEE", 

103 "CJK", 

104 "COPTIC", 

105 "COUNTING ROD", 

106 "CUNEIFORM", 

107 "CYRILLIC", 

108 "DOUBLE-STRUCK", 

109 "EGYPTIAN", 

110 "ETHIOPIC", 

111 "EXTENDED ARABIC-INDIC", 

112 "GEORGIAN", 

113 "GLAGOLITIC", 

114 "GOTHIC", 

115 "GREEK", 

116 "GUJARATI", 

117 "GURMUKHI", 

118 "HANGUL", 

119 "HANIFI ROHINGYA", 

120 "HEBREW", 

121 "HIRAGANA", 

122 "JAVANESE", 

123 "KANNADA", 

124 "KATAKANA", 

125 "KAYAH LI", 

126 "KHMER", 

127 "KHUDAWADI", 

128 "LAO", 

129 "LEPCHA", 

130 "LIMBU", 

131 "MALAYALAM", 

132 "MEETEI", 

133 "MYANMAR", 

134 "NEW TAI LUE", 

135 "NKO", 

136 "OL CHIKI", 

137 "OLD PERSIAN", 

138 "OLD SOUTH ARABIAN", 

139 "ORIYA", 

140 "OSMANYA", 

141 "PHOENICIAN", 

142 "SAURASHTRA", 

143 "SHARADA", 

144 "SINHALA", 

145 "SUNDANESE", 

146 "SYLOTI", 

147 "TAI THAM", 

148 "TAKRI", 

149 "TAMIL", 

150 "TELUGU", 

151 "THAANA", 

152 "THAI", 

153 "TIBETAN", 

154 "TIFINAGH", 

155 "TIRHUTA", 

156 "UGARITIC", 

157 "WARANG CITI", 

158 "YI", 

159] 

160non_latin_scripts_re = re.compile( 

161 r"(" + r"|".join(re.escape(x) for x in non_latin_scripts) + r")\b" 

162) 

163 

164# Sanity check xlat_head_map values 

165for k, v in xlat_head_map.items(): 

166 if v.startswith("?"): 

167 v = v[1:] 

168 for tag in v.split(): 

169 if tag not in valid_tags: 169 ↛ 170line 169 didn't jump to line 170 because the condition on line 169 was never true

170 print( 

171 "WARNING: xlat_head_map[{}] contains" 

172 " unrecognized tag {}".format(k, tag) 

173 ) 

174 

175# Regexp for finding nested translations from translation items (these are 

176# used in, e.g., year/English/Translations/Arabic). This is actually used 

177# in page.py. 

178nested_translations_re = re.compile( 

179 r"\s+\((({}): ([^()]|\([^()]+\))+)\)".format( 

180 "|".join( 

181 re.escape(x.removeprefix("?")) 

182 for x in sorted(xlat_head_map.values(), key=len, reverse=True) 

183 if x and not x.startswith("class-") 

184 ) 

185 ) 

186) 

187 

188# Regexp that matches head tag specifiers. Used to match tags from end of 

189# translations and linkages 

190head_final_re_text = r"( -)?( ({}))+".format( 

191 "|".join( 

192 re.escape(x) 

193 for x in 

194 # The sort is to put longer ones first, preferring them in 

195 # the regexp match 

196 sorted(xlat_head_map.keys(), key=len, reverse=True) 

197 ) 

198) 

199head_final_re = re.compile(head_final_re_text + r"$") 

200 

201# Regexp used to match head tag specifiers at end of a form for certain 

202# Bantu languages (particularly Swahili and similar languages). 

203head_final_bantu_re_text = r" ({})".format( 

204 "|".join(re.escape(x) for x in head_final_bantu_map.keys()) 

205) 

206head_final_bantu_re = re.compile(head_final_bantu_re_text + "$") 

207 

208# Regexp used to match head tag specifiers at end of a form for certain 

209# Semitic languages (particularly Arabic and similar languages). 

210head_final_semitic_re_text = r" ({})".format( 

211 "|".join(re.escape(x) for x in head_final_semitic_map.keys()) 

212) 

213head_final_semitic_re = re.compile(head_final_semitic_re_text + "$") 

214 

215# Regexp used to match head tag specifiers at end of a form for certain 

216# other languages (e.g., Lithuanian, Finnish, French). 

217head_final_other_re_text = r" ({})".format( 

218 "|".join(re.escape(x) for x in head_final_other_map.keys()) 

219) 

220head_final_other_re = re.compile(head_final_other_re_text + "$") 

221 

222# Regexp for splitting heads. See parse_word_head(). 

223head_split_re_text_part_1 = ( 

224 "(" 

225 + head_final_re_text 

226 + "|" 

227 + head_final_bantu_re_text 

228 + "|" 

229 + head_final_semitic_re_text 

230 + "|" 

231 + head_final_other_re_text 

232) 

233 

234head_split_re_text = head_split_re_text_part_1 + ")?( or |[,;]+| *$)" 

235 

236head_split_re_text_no_semicolon = head_split_re_text_part_1 + ")?( or |,+| *$)" 

237 

238head_split_re = re.compile(head_split_re_text) 

239head_split_no_semicolon_re = re.compile(head_split_re_text_no_semicolon) 

240 

241head_split_re_parens = 0 

242for m in re.finditer(r"(^|[^\\])[(]+", head_split_re_text): 

243 head_split_re_parens += m.group(0).count("(") 

244 

245# Parenthesized parts that are ignored in translations 

246tr_ignored_parens: set[str] = set( 

247 [ 

248 "please verify", 

249 "(please verify)", 

250 "transliteration needed", 

251 "(transliteration needed)", 

252 "in words with back vowel harmony", 

253 "(in words with back vowel harmony)", 

254 "in words with front vowel harmony", 

255 "(in words with front vowel harmony)", 

256 "see below", 

257 "see usage notes below", 

258 ] 

259) 

260tr_ignored_parens_re = re.compile( 

261 r"^(" 

262 + "|".join(re.escape(x) for x in tr_ignored_parens) 

263 + ")$" 

264 + r"|^(Can we clean up|Can we verify|for other meanings see " 

265 r"lit\. )" 

266) 

267 

268# Translations that are ignored 

269ignored_translations: set[str] = set( 

270 [ 

271 "[script needed]", 

272 "please add this translation if you can", 

273 ] 

274) 

275 

276# Put english text into the "note" field in a translation if it contains one 

277# of these words 

278tr_note_re = re.compile( 

279 r"(\b(article|definite|indefinite|superlative|comparative|pattern|" 

280 r"adjective|adjectives|clause|clauses|pronoun|pronouns|preposition|prep|" 

281 r"postposition|postp|action|actions|articles|" 

282 r"adverb|adverbs|noun|nouns|verb|verbs|before|" 

283 r"after|placed|prefix|suffix|used with|translated|" 

284 r"nominative|genitive|dative|infinitive|participle|past|perfect|imperfect|" 

285 r"perfective|imperfective|auxiliary|negative|future|present|tense|aspect|" 

286 r"conjugation|declension|class|category|plural|singular|positive|" 

287 r"seldom used|formal|informal|familiar|unspoken|spoken|written|" 

288 r"indicative|progressive|conditional|potential|" 

289 r"accusative|adessive|inessive|superessive|elative|allative|" 

290 r"dialect|dialects|object|subject|predicate|movies|recommended|language|" 

291 r"locative|continuous|simple|continuousness|gerund|subjunctive|" 

292 r"periphrastically|no equivalent|not used|not always used|" 

293 r"used only with|not applicable|use the|signifying|wordplay|pronounced|" 

294 r"preconsonantal|spelled|spelling|respelling|respellings|phonetic|" 

295 r"may be replaced|stricter sense|for nonhumans|" 

296 r"sense:|used:|in full:|informally used|followed by|" 

297 r"not restricted to|pertaining to|or optionally with|are optional|" 

298 r"in conjunction with|in compounds|depending on the relationship|" 

299 r"person addressed|one person|multiple persons|may be replaced with|" 

300 r"optionally completed with|in the phrase|in response to|" 

301 r"before a|before an|preceded by|verbs ending|very common|after a verb|" 

302 r"with verb|with uncountable|with the objects|with stative|" 

303 r"can be replaced by|often after|used before|used after|" 

304 r"used in|clipping of|spoken|somewhat|capitalized|" 

305 r"short form|shortening of|shortened form|initialism of|" 

306 r"said to|rare:|rarer also|is rarer|negatively connoted|" 

307 r"previously mentioned|uncountable noun|countable noun|" 

308 r"countable nouns|uncountable nouns|" 

309 r"with predicative|with -|with imperfect|with a negated|" 

310 r"colloquial|misspelling|holophrastic|frequently|esp\.|especially|" 

311 r'"|' 

312 r"general term|after a vowel|before a vowel|" 

313 r"form|regular|irregular|alternative)" 

314 r")($|[) ])|^(" 

315 # Following are only matched at the beginning of the string 

316 r"pl|pl\.|see:|pl:|sg:|plurals:|e\.g\.|e\.g\.:" 

317 r"|e\.g\.,|cf\.|compare|such as|" 

318 r"see|only|often|usually|used|usage:|of|not|in|compare|usu\.|" 

319 r"as|about|abbrv\.|abbreviation|abbr\.|that:|optionally|" 

320 r"mainly|from|for|also|also:|acronym|" 

321 r"\+|with) " 

322) 

323# \b does not work at the end??? 

324 

325# Related forms matching this regexp will be considered suspicious if the 

326# page title does not also match one of these. 

327suspicious_related_re = re.compile( 

328 r"(^| )(f|m|n|c|or|pl|sg|inan|anim|pers|anml|impf|pf|vir|nvir)( |$)" 

329 r"|[][:=<>&#*|]" 

330 r"| \d+$" 

331) 

332 

333# Word forms (head forms, translations, etc) that will be considered ok and 

334# silently accepted even if they would otherwise trigger a suspicious 

335# form warning. 

336ok_suspicious_forms: set[str] = set( 

337 [ 

338 "but en or", # "golden goal"/English/Tr/French 

339 "cœur en or", # "heart of gold"/Eng/Tr/French 

340 "en or", # golden/Eng/Tr/French 

341 "men du", # jet/Etym2/Noun/Tr/Cornish 

342 "parachute en or", # "golden parachute"/Eng/Tr/French 

343 "vieil or", # "old gold"/Eng/Tr/French 

344 # "all that glitters is not gold"/Eng/Tr/French 

345 "tout ce qui brille n’est pas or", 

346 "μη αποκλειστικό or", # inclusive or/Eng/Tr/Greek 

347 "period or full stop", 

348 ] 

349) 

350 

351 

352# Replacements to be done in classify_desc before tokenizing. This is a 

353# workaround for shortcomings in TweetTokenizer. 

354tokenizer_fixup_map = { 

355 r"a.m.": "AM", 

356 r"p.m.": "PM", 

357} 

358tokenizer_fixup_re = re.compile( 

359 r"\b(" 

360 + "|".join( 

361 re.escape(x) 

362 for x in sorted( 

363 tokenizer_fixup_map.keys(), key=lambda x: len(x), reverse=True 

364 ) 

365 ) 

366 + r")" 

367) 

368 

369# Unknown tags starting with these words will be silently ignored. 

370ignored_unknown_starts: set[str] = set( 

371 [ 

372 "originally", 

373 "e.g.", 

374 "c.f.", 

375 "supplanted by", 

376 "supplied by", 

377 ] 

378) 

379 

380ignored_unknown_starts_re = re.compile( 

381 r"^(" 

382 + "|".join( 

383 re.escape(x) 

384 for x in sorted(ignored_unknown_starts, key=lambda x: -len(x)) 

385 ) 

386 + ") " 

387) 

388 

389# If an unknown sequence starts with one of these, it will continue as an 

390# unknown sequence until the end, unless it turns out to have a replacement. 

391allowed_unknown_starts: set[str] = set( 

392 [ 

393 "Relating", 

394 "accompanied", 

395 "added", 

396 "after", 

397 "answering", 

398 "as", 

399 "based", 

400 "before", 

401 "conjugated", 

402 "conjunction", 

403 "construed", 

404 "especially", 

405 "expression:", 

406 "figurative:", 

407 "followed", 

408 "for", 

409 "forms", 

410 "from", 

411 "governs", 

412 "in", 

413 "indicating", 

414 "modifying", 

415 "normally", 

416 "not", 

417 "of", 

418 "preceding", 

419 "prefixed", 

420 "referring", 

421 "relating", 

422 "revived", 

423 "said", 

424 "since", 

425 "takes", 

426 "used", 

427 "with", 

428 "With", 

429 "without", 

430 ] 

431) 

432# Allow the ignored unknown starts without complaining 

433allowed_unknown_starts.update(ignored_unknown_starts) 

434 

435# Full unknown tags that will be ignored in decode_tags() 

436# XXX this is unused, ask Tatu where the contents is now 

437ignored_unknown_tags: set[str] = set([]) 

438 

439# Head endings that are mapped to tags 

440head_end_map = { 

441 " 1st conj.": "conjugation-1", 

442 " 2nd conj.": "conjugation-2", 

443 " 3rd conj.": "conjugation-3", 

444 " 4th conj.": "conjugation-4", 

445 " 5th conj.": "conjugation-5", 

446 " 6th conj.": "conjugation-6", 

447 " 7th conj.": "conjugation-7", 

448} 

449head_end_re = re.compile( 

450 r"(" + "|".join(re.escape(x) for x in head_end_map.keys()) + r")$" 

451) 

452 

453# Transitivity and inflection class labels at the end of Japanese heads, 

454# e.g. 走る "intransitive godan", 給ふ "transitive or intransitive ^†yodan" 

455ja_head_end_re = re.compile( 

456 r" (?:\^†)?(transitive|intransitive|godan|ichidan|suru|kuru|yodan|nidan" 

457 r"|irregular|uninflectable" 

458 r"|-i|-na|-nari|-tari|-ku|-shiku|-ka|-ri|-se|-ko|-n)(?: or)?$" 

459) 

460 

461 

462# Dictionary of language-specific parenthesized head part starts that 

463# either introduce new tags or modify previous tags. The value for each 

464# language is a dictionary that maps the first word of the head part to 

465# (rem_tags, add_tags), where ``rem_tags`` can be True to remove all previous 

466# tags or a space-separated string of tags to remove, and ``add_tags`` should 

467# be a string of tags to add. 

468lang_specific_head_map: dict[ 

469 str, dict[str, Union[tuple[str, str], tuple[Literal[True], str]]] 

470] = { 

471 "Danish": { 

472 # prefix: (rem_tags space separate string/True, add_tags s-sep str) 

473 "c": ("neuter", "common-gender"), 

474 "n": ("common-gender", "neuter"), 

475 "pl": ("singular neuter common-gender", "plural"), 

476 "sg": ("plural neuter common-gender", "singular"), 

477 }, 

478} 

479 

480 

481# Regular expression used to strip additional stuff from the end of alt_of and 

482# form_of. 

483alt_of_form_of_clean_re = re.compile( 

484 r"(?s)(" 

485 + "|".join( 

486 [ 

487 r":", 

488 r'[“"]', 

489 r";", 

490 r" \(", 

491 r" - ", 

492 r" ־ ", 

493 r" ᠆ ", 

494 r" ‐ ", 

495 r" ‑ ", 

496 r" ‒ ", 

497 r" – ", 

498 r" — ", 

499 r" ― ", 

500 r" − ", 

501 r" ⸺ ", 

502 r" ⸻ ", 

503 r" ﹘ ", 

504 r" ﹣ ", 

505 r" - ", 

506 r" \+ ", 

507 r" \(with ", 

508 r" with -ra/-re", 

509 r"\. Used ", 

510 r"\. Also ", 

511 r"\. Since ", 

512 r"\. A ", 

513 r"\.\. A ", 

514 r"\. An ", 

515 r"\.\. An ", 

516 r"\. an ", 

517 r"\. The ", 

518 r"\. Spanish ", 

519 r"\. Language ", 

520 r"\. former name of ", 

521 r"\. AIM", 

522 r"\. OT", 

523 r"\. Not ", 

524 r"\. Now ", 

525 r"\. Nowadays ", 

526 r"\. Early ", 

527 r"\. ASEAN", 

528 r"\. UN", 

529 r"\. IMF", 

530 r"\. WHO", 

531 r"\. WIPO", 

532 r"\. AC", 

533 r"\. DC", 

534 r"\. DNA", 

535 r"\. RNA", 

536 r"\. SOB", 

537 r"\. IMO", 

538 r"\. Behavior", 

539 r"\. Income ", 

540 r"\. More ", 

541 r"\. Most ", 

542 r"\. Only ", 

543 r"\. Also ", 

544 r"\. From ", 

545 r"\. Of ", 

546 r"\.\. Of ", 

547 r"\. To ", 

548 r"\. For ", 

549 r"\. If ", 

550 r"\. Praenominal ", 

551 r"\. This ", 

552 r"\. Replaced ", 

553 r"\. CHCS is the ", 

554 r"\. Equivalent ", 

555 r"\. Initialism ", 

556 r"\. Note ", 

557 r"\. Alternative ", 

558 r"\. Compare ", 

559 r"\. Cf\. ", 

560 r"\. Comparable ", 

561 r"\. Involves ", 

562 r"\. Sometimes ", 

563 r"\. Commonly ", 

564 r"\. Often ", 

565 r"\. Typically ", 

566 r"\. Possibly ", 

567 r"\. Although ", 

568 r"\. Rare ", 

569 r"\. Instead ", 

570 r"\. Integrated ", 

571 r"\. Distinguished ", 

572 r"\. Given ", 

573 r"\. Found ", 

574 r"\. Was ", 

575 r"\. In ", 

576 r"\. It ", 

577 r"\.\. It ", 

578 r"\. One ", 

579 r"\. Any ", 

580 r"\. They ", 

581 r"\. Members ", 

582 r"\. Each ", 

583 r"\. Original ", 

584 r"\. Especially ", 

585 r"\. Usually ", 

586 r"\. Known ", 

587 r"\.\. Known ", 

588 r"\. See ", 

589 r"\. see ", 

590 r"\. target was not ", 

591 r"\. Popular ", 

592 r"\. Pedantic ", 

593 r"\. Positive ", 

594 r"\. Society ", 

595 r"\. Plan ", 

596 r"\. Environmentally ", 

597 r"\. Affording ", 

598 r"\. Encompasses ", 

599 r"\. Expresses ", 

600 r"\. Indicates ", 

601 r"\. Text ", 

602 r"\. Large ", 

603 r"\. Sub-sorting ", 

604 r"\. Sax", 

605 r"\. First-person ", 

606 r"\. Second-person ", 

607 r"\. Third-person ", 

608 r"\. 1st ", 

609 r"\. 2nd ", 

610 r"\. 3rd ", 

611 r"\. Term ", 

612 r"\. Northeastern ", 

613 r"\. Northwestern ", 

614 r"\. Southeast ", 

615 r"\. Egyptian ", 

616 r"\. English ", 

617 r"\. Cape Province was split into ", 

618 r"\. Pañcat", 

619 r"\. of the ", 

620 r"\. is ", 

621 r"\. after ", 

622 r"\. or ", 

623 r"\. chromed", 

624 r"\. percussion", 

625 r"\. with his ", 

626 r"\. a\.k\.a\. ", 

627 r"\. comparative form ", 

628 r"\. singular ", 

629 r"\. plural ", 

630 r"\. present ", 

631 r"\. his ", 

632 r"\. her ", 

633 r"\. equivalent ", 

634 r"\. measuring ", 

635 r"\. used in ", 

636 r"\. cutely ", 

637 r"\. Protects", 

638 r'\. "', 

639 r"\.^", 

640 r"\. \+ ", 

641 r"\., ", 

642 r". — ", 

643 r", a ", 

644 r", an ", 

645 r", the ", 

646 r", obsolete ", 

647 r", possessed", # 'd/English 

648 r", imitating", # 1/English 

649 r", derived from", 

650 r", called ", 

651 r", especially ", 

652 r", slang for ", 

653 r", used to", # c/o /English 

654 r", commonly", # b/w /English 

655 r" corresponding to ", 

656 r" equivalent to ", 

657 r" popularized by ", 

658 r" denoting ", 

659 r" in its various senses\.", 

660 r" used by ", 

661 r" but not for ", 

662 r" since ", 

663 r" i\.e\. ", 

664 r" i\. e\. ", 

665 r" e\.g\. ", 

666 r" eg\. ", 

667 r" etc\. ", 

668 r"\[http", 

669 r" — used as ", 

670 r" by K\. Forsyth ", 

671 r" by J\. R\. Allen ", 

672 r" by S\. Ferguson ", 

673 r" by G\. Donaldson ", 

674 r" May refer to ", 

675 r" An area or region ", 

676 ] 

677 ) 

678 + r").*$" 

679) 

680 

681 

682class ValidNode: 

683 """Node in the valid_sequences tree. Each node is part of a chain 

684 or chains that form sequences built out of keys in key->tags 

685 maps like xlat_tags, etc. The ValidNode's 'word' is the key 

686 by which it is refered to in the root dict or a `children` dict, 

687 `end` marks that the node is the end-terminus of a sequence (but 

688 it can still continue if the sequence is shared by the start of 

689 other sequences: "nominative$" and "nominative plural$" for example), 

690 `tags` and `topics` are the dicts containing tag and topic strings 

691 for terminal nodes (end==True).""" 

692 

693 __slots__ = ( 

694 "end", 

695 "tags", 

696 "topics", 

697 "children", 

698 ) 

699 

700 def __init__( 

701 self, 

702 end=False, 

703 tags: Optional[list[str]] = None, 

704 topics: Optional[list[str]] = None, 

705 children: Optional[dict[str, "ValidNode"]] = None, 

706 ) -> None: 

707 self.end = end 

708 self.tags: list[str] = tags or [] 

709 self.topics: list[str] = topics or [] 

710 self.children: dict[str, "ValidNode"] = children or {} 

711 

712 

713def add_to_valid_tree(tree: ValidNode, desc: str, v: Optional[str]) -> None: 

714 """Helper function for building trees of valid tags/sequences during 

715 initialization.""" 

716 assert isinstance(tree, ValidNode) 

717 assert isinstance(desc, str) 

718 assert v is None or isinstance(v, str) 

719 node = tree 

720 

721 # Build the tree structure: each node has children nodes 

722 # whose names are denoted by their dict key. 

723 for w in desc.split(" "): 

724 if w in node.children: 

725 node = node.children[w] 

726 else: 

727 new_node = ValidNode() 

728 node.children[w] = new_node 

729 node = new_node 

730 if not node.end: 

731 node.end = True 

732 if not v: 

733 return None # Terminate early because there are no tags 

734 

735 tagslist = [] 

736 topicslist = [] 

737 for vv in v.split(): 

738 if vv in valid_tags: 

739 tagslist.append(vv) 

740 elif vv in valid_topics: 740 ↛ 743line 740 didn't jump to line 743 because the condition on line 740 was always true

741 topicslist.append(vv) 

742 else: 

743 print( 

744 "WARNING: tag/topic {!r} maps to unknown {!r}".format(desc, vv) 

745 ) 

746 topics = " ".join(topicslist) 

747 tags = " ".join(tagslist) 

748 # Changed to "_tags" and "_topics" to avoid possible key-collisions. 

749 if topics: 

750 node.topics.extend([topics]) 

751 if tags: 

752 node.tags.extend([tags]) 

753 

754 

755def add_to_valid_tree1( 

756 tree: ValidNode, 

757 k: str, 

758 v: Union[list[str], tuple[str, ...], str], 

759 valid_values: Union[set[str], dict[str, Any]], 

760) -> list[str]: 

761 assert isinstance(tree, ValidNode) 

762 assert isinstance(k, str) 

763 assert v is None or isinstance(v, (list, tuple, str)) 

764 assert isinstance(valid_values, (set, dict)) 

765 if not v: 765 ↛ 766line 765 didn't jump to line 766 because the condition on line 765 was never true

766 add_to_valid_tree(valid_sequences, k, None) 

767 return [] 

768 elif isinstance(v, str): 

769 v = [v] 

770 q = [] 

771 for vv in v: 

772 assert isinstance(vv, str) 

773 add_to_valid_tree(valid_sequences, k, vv) 

774 vvs = vv.split() 

775 for x in vvs: 

776 q.append(x) 

777 # return each individual tag 

778 return q 

779 

780 

781def add_to_valid_tree_mapping( 

782 tree: ValidNode, 

783 mapping: Union[dict[str, Union[list[str], str]], dict[str, str]], 

784 valid_values: Union[set[str], dict[str, Any]], 

785 recurse: bool, 

786) -> None: 

787 assert isinstance(tree, ValidNode) 

788 assert isinstance(mapping, dict) 

789 assert isinstance(valid_values, (set, dict)) 

790 assert recurse in (True, False) 

791 for k, v in mapping.items(): 

792 assert isinstance(k, str) 

793 assert isinstance(v, (list, str)) 

794 if isinstance(v, str): 

795 q = add_to_valid_tree1(tree, k, [v], valid_values) 

796 else: 

797 q = add_to_valid_tree1(tree, k, v, valid_values) 

798 if recurse: 

799 visited = set() 

800 while q: 

801 v = q.pop() 

802 if v in visited: 

803 continue 

804 visited.add(v) 

805 if v not in mapping: 

806 continue 

807 vv = mapping[v] 

808 qq = add_to_valid_tree1(tree, k, vv, valid_values) 

809 q.extend(qq) 

810 

811 

812# Tree of sequences considered to be tags (includes sequences that are 

813# mapped to something that becomes one or more valid tags) 

814valid_sequences = ValidNode() 

815sequences_with_slashes: set[str] = set() 

816for tag in valid_tags: 

817 # The basic tags used in our tag system; some are a bit weird, but easier 

818 # to implement this with 'false' positives than filter out stuff no one else 

819 # uses. 

820 if "/" in tag: 

821 sequences_with_slashes.add(tag) 

822 add_to_valid_tree(valid_sequences, tag, tag) 

823for tag in uppercase_tags: 

824 hyphenated = re.sub(r"\s+", "-", tag) 

825 if "/" in tag: 

826 sequences_with_slashes.add(tag) 

827 add_to_valid_tree(valid_sequences, tag, hyphenated) 

828 

829# xlat_tags_map! 

830add_to_valid_tree_mapping(valid_sequences, xlat_tags_map, valid_tags, False) 

831for k in xlat_tags_map: 

832 if "/" in k: 

833 sequences_with_slashes.add(k) 

834# Add topics to the same table, with all generalized topics also added 

835for topic in valid_topics: 

836 assert " " not in topic 

837 if "/" in topic: 837 ↛ 838line 837 didn't jump to line 838 because the condition on line 837 was never true

838 sequences_with_slashes.add(topic) 

839 add_to_valid_tree(valid_sequences, topic, topic) 

840# Let each original topic value stand alone. These are not generally on 

841# valid_topics. We add the original topics with spaces replaced by hyphens. 

842for topic in topic_generalize_map.keys(): 

843 hyphenated = re.sub(r"\s+", "-", topic) 

844 if "/" in topic: 844 ↛ 845line 844 didn't jump to line 845 because the condition on line 844 was never true

845 sequences_with_slashes.add(topic) 

846 add_to_valid_tree(valid_sequences, topic, hyphenated) 

847# Add canonicalized/generalized topic values 

848add_to_valid_tree_mapping( 

849 valid_sequences, topic_generalize_map, valid_topics, True 

850) 

851 

852# Regex used to divide a decode candidate into parts that shouldn't 

853# have their slashes turned into spaces 

854slashes_re = re.compile( 

855 r"(" + "|".join((re.escape(s) for s in sequences_with_slashes)) + r")" 

856) 

857 

858# Regexp used to find "words" from word heads and linguistic descriptions 

859word_pattern = ( 

860 r"[^ ,;()\u200e]+|" 

861 r"\([^ ,;()\u200e]+\)[^ ,;()\u200e]+|" 

862 r"[\u2800-\u28ff]|" # Braille characters 

863 r"\(([^()]|\([^()]*\))*\)" 

864) 

865 

866word_re_global = re.compile(word_pattern) 

867 

868 

869def distw(titleparts: Sequence[str], word: str) -> float: 

870 """Computes how distinct ``word`` is from the most similar word in 

871 ``titleparts``. Returns 1 if words completely distinct, 0 if 

872 identical, or otherwise something in between.""" 

873 assert isinstance(titleparts, (list, tuple)) 

874 assert isinstance(word, str) 

875 w = min( 

876 Levenshtein.distance(word, tw) / max(len(tw), len(word)) 

877 for tw in titleparts 

878 ) 

879 return w 

880 

881 

882def map_with( 

883 ht: dict[str, str | list[str]] | dict[str, str], 

884 lst: Sequence[str], 

885) -> list[str]: 

886 """Takes alternatives from ``lst``, maps them using ``ht`` to zero or 

887 more alternatives each, and returns a combined list of alternatives.""" 

888 assert isinstance(ht, dict) 

889 assert isinstance(lst, (list, tuple)) 

890 ret = [] 

891 for x in lst: 

892 assert isinstance(x, str) 

893 x = x.strip() 

894 x = ht.get(x, x) 

895 if isinstance(x, str): 895 ↛ 898line 895 didn't jump to line 898 because the condition on line 895 was always true

896 if x: 896 ↛ 891line 896 didn't jump to line 891 because the condition on line 896 was always true

897 ret.append(x) 

898 elif isinstance(x, (list, tuple)): 

899 ret.extend(x) 

900 else: 

901 raise RuntimeError("map_with unexpected value: {!r}".format(x)) 

902 return ret 

903 

904 

905TagList = list[str] 

906PosPathStep = tuple[int, TagList, TagList] 

907 

908 

909def check_unknown( 

910 from_i: int, 

911 to_i: int, 

912 i: int, 

913 wordlst: Sequence[str], 

914 allow_any: bool, 

915 no_unknown_starts: bool, 

916) -> list[PosPathStep]: 

917 """Check if the current section from_i->to_i is actually unknown 

918 or if it needs some special handling. We already presupposed that 

919 this is UNKNOWN; this is just called to see what *kind* of UNKNOWN.""" 

920 assert isinstance(to_i, int) 

921 assert isinstance(from_i, int) 

922 assert isinstance(i, int) 

923 # Adds unknown tag if needed. Returns new last_i 

924 # print("check_unknown to_i={} from_i={} i={}" 

925 # .format(to_i, from_i, i)) 

926 if from_i >= to_i: 

927 return [] 

928 words = wordlst[from_i:to_i] 

929 tag = " ".join(words) 

930 assert tag 

931 # print(f"{tag=}") 

932 if re.match(ignored_unknown_starts_re, tag): 

933 # Tags with this start are to be ignored 

934 return [(from_i, ["UNKNOWN"], [])] 

935 if tag in ignored_unknown_tags: 935 ↛ 936line 935 didn't jump to line 936 because the condition on line 935 was never true

936 return [] # One of the tags listed as to be ignored 

937 if tag in ("and", "or"): 

938 return [] 

939 if ( 

940 not allow_any 

941 and not words[0].startswith("~") 

942 and ( 

943 no_unknown_starts 

944 or words[0] not in allowed_unknown_starts 

945 or len(words) <= 1 

946 ) 

947 ): 

948 # print("ERR allow_any={} words={}" 

949 # .format(allow_any, words)) 

950 return [ 

951 (from_i, ["UNKNOWN"], ["error-unknown-tag"]) 

952 ] # Add ``tag`` here to include 

953 else: 

954 return [(from_i, ["UNKNOWN"], [tag])] 

955 

956 

957def add_new1( 

958 node: ValidNode, 

959 i: int, 

960 start_i: int, 

961 last_i: int, 

962 new_paths: list[list[PosPathStep]], 

963 new_nodes: list[tuple[ValidNode, int, int]], 

964 pos_paths: list[list[list[PosPathStep]]], 

965 wordlst: list[str], 

966 allow_any: bool, 

967 no_unknown_starts: bool, 

968 max_last_i: int, 

969) -> int: 

970 assert isinstance(new_paths, list) 

971 # print("add_new: start_i={} last_i={}".format(start_i, last_i)) 

972 # print("$ {} last_i={} start_i={}" 

973 # .format(w, last_i, start_i)) 

974 max_last_i = max(max_last_i, last_i) # if last_i has grown 

975 if (node, start_i, last_i) not in new_nodes: 

976 new_nodes.append((node, start_i, last_i)) 

977 if node.end: 

978 # We can see a terminal point in the search tree. 

979 u = check_unknown( 

980 last_i, start_i, i, wordlst, allow_any, no_unknown_starts 

981 ) 

982 # Create new paths candidates based on different past possible 

983 # paths; pos_path[last_i] contains possible paths, so add this 

984 # new one at the beginning(?) 

985 # The list comprehension inside the parens generates an iterable 

986 # of lists, so this is .extend( [(last_i...)], [(last_i...)], ... ) 

987 # XXX: this is becoming impossible to annotate, nodes might 

988 # need to become classed objects and not just dicts, or at least 

989 # a TypedDict with a "children" node 

990 new_paths.extend( 

991 [(last_i, node.tags, node.topics)] + u + x 

992 for x in pos_paths[last_i] 

993 ) 

994 max_last_i = i + 1 

995 return max_last_i 

996 

997 

998@functools.lru_cache(maxsize=65536) 

999def decode_tags( 

1000 src: str, 

1001 allow_any=False, 

1002 no_unknown_starts=False, 

1003) -> tuple[list[tuple[str, ...]], list[str]]: 

1004 tagsets, topics = decode_tags1(src, allow_any, no_unknown_starts) 

1005 # print(f"decode_tags: {src=}, {tagsets=}") 

1006 

1007 # Insert retry-code here that modifies the text source 

1008 if ( 

1009 any(s.startswith("error-") for tagset in tagsets for s in tagset) 

1010 # I hate Python's *nested* list comprehension syntax ^ 

1011 or any(s.startswith("error-") for s in topics) 

1012 ): 

1013 new_tagsets: list[tuple[str, ...]] = [] 

1014 new_topics: list[str] = [] 

1015 

1016 if "/" in src: 

1017 # slashes_re contains valid key entries with slashes; we're going 

1018 # to skip them by splitting the string and skipping handling every 

1019 # second entry, which contains the splitting group like "masculine/ 

1020 # feminine" style keys. 

1021 split_parts = re.split(slashes_re, src) 

1022 new_parts: list[str] = [] 

1023 if len(split_parts) > 1: 

1024 for i, s in enumerate(split_parts): 

1025 if i % 2 == 0: 

1026 new_parts.append(s.replace("/", " ")) 

1027 else: 

1028 new_parts.append(s) 

1029 new_src = "".join(new_parts) 

1030 else: 

1031 new_src = src 

1032 new_tagsets, new_topics = decode_tags1( 

1033 new_src, allow_any, no_unknown_starts 

1034 ) 

1035 elif " or " in src or " and " in src: 

1036 # Annoying kludge. 

1037 new_src = src.replace(" and ", " ") 

1038 new_src = new_src.replace(" or ", " ") 

1039 new_tagsets, new_topics = decode_tags1( 

1040 new_src, allow_any, no_unknown_starts 

1041 ) 

1042 # print(f"{new_tagsets=}") 

1043 

1044 if new_tagsets or new_topics: 

1045 old_errors = sum( 

1046 1 for tagset in tagsets for s in tagset if s.startswith("error") 

1047 ) 

1048 old_errors += sum(1 for s in topics if s.startswith("error")) 

1049 new_errors = sum( 

1050 1 

1051 for new_tagset in new_tagsets 

1052 for s in new_tagset 

1053 if s.startswith("error") 

1054 ) 

1055 new_errors += sum(1 for s in new_topics if s.startswith("error")) 

1056 

1057 if new_errors <= old_errors: 1057 ↛ 1060line 1057 didn't jump to line 1060 because the condition on line 1057 was always true

1058 return new_tagsets, new_topics 

1059 

1060 return tagsets, topics 

1061 

1062 

1063def decode_tags1( 

1064 src: str, 

1065 allow_any=False, 

1066 no_unknown_starts=False, 

1067) -> tuple[list[tuple[str, ...]], list[str]]: 

1068 """Decodes tags, doing some canonicalizations. This returns a list of 

1069 lists of tags and a list of topics.""" 

1070 assert isinstance(src, str) 

1071 

1072 # print("decode_tags: src={!r}".format(src)) 

1073 

1074 pos_paths: list[list[list[PosPathStep]]] = [[[]]] 

1075 wordlst: list[str] = [] 

1076 max_last_i = 0 # pre-initialized here so that it can be used as a ref 

1077 

1078 add_new = functools.partial( 

1079 add_new1, # pre-set parameters and references for function 

1080 pos_paths=pos_paths, 

1081 wordlst=wordlst, 

1082 allow_any=allow_any, 

1083 no_unknown_starts=no_unknown_starts, 

1084 max_last_i=max_last_i, 

1085 ) 

1086 # First split the tags at commas and semicolons. Their significance is that 

1087 # a multi-word sequence cannot continue across them. 

1088 parts = split_at_comma_semi(src, extra=[";", ":"]) 

1089 

1090 for part in parts: 

1091 max_last_i = len(wordlst) # "how far have we gone?" 

1092 lst1 = part.split() 

1093 if not lst1: 

1094 continue 

1095 wordlst.extend(lst1) 

1096 cur_nodes: list[tuple[ValidNode, int, int]] = [] # Currently seen 

1097 for w in lst1: 

1098 i = len(pos_paths) - 1 

1099 new_nodes: list[tuple[ValidNode, int, int]] = [] 

1100 # replacement nodes for next loop 

1101 new_paths: list[list[PosPathStep]] = [] 

1102 # print("ITER i={} w={} max_last_i={} wordlst={}" 

1103 # .format(i, w, max_last_i, wordlst)) 

1104 node: ValidNode 

1105 start_i: int 

1106 last_i: int 

1107 for node, start_i, last_i in cur_nodes: 

1108 # ValidNodes are part of a search tree that checks if a 

1109 # phrase is found in xlat_tags_map and other text->tags dicts. 

1110 if w in node.children: 

1111 # the phrase continues down the tree 

1112 # print("INC", w) 

1113 max_last_i = add_new( 

1114 node.children[w], 

1115 i, 

1116 start_i, 

1117 last_i, 

1118 new_paths, 

1119 new_nodes, 

1120 ) 

1121 if node.end: 

1122 # we've hit an end point, the tags and topics have already 

1123 # been gathered at some point, don't do anything with the 

1124 # old stuff 

1125 if w in valid_sequences.children: 

1126 # This starts a *new* possible section 

1127 max_last_i = add_new( 

1128 valid_sequences.children[w], # root-> 

1129 i, 

1130 i, 

1131 i, 

1132 new_paths, 

1133 new_nodes, 

1134 ) 

1135 if w not in node.children and not node.end: 

1136 # print("w not in node and $: i={} last_i={} wordlst={}" 

1137 # .format(i, last_i, wordlst)) 

1138 # If i == last_i == 0, for example (beginning) 

1139 if ( 

1140 i == last_i 

1141 or no_unknown_starts 

1142 or wordlst[last_i] not in allowed_unknown_starts 

1143 ): 

1144 # print("NEW", w) 

1145 if w in valid_sequences.children: 

1146 # Start new sequences here 

1147 max_last_i = add_new( 

1148 valid_sequences.children[w], 

1149 i, 

1150 i, 

1151 last_i, 

1152 new_paths, 

1153 new_nodes, 

1154 ) 

1155 if not new_nodes: 

1156 # This is run at the start when i == max_last_i == 0, 

1157 # which is what populates the first node in new_nodes. 

1158 # Some initial words cause the rest to be interpreted as unknown 

1159 # print("not new nodes: i={} last_i={} wordlst={}" 

1160 # .format(i, max_last_i, wordlst)) 

1161 if ( 

1162 i == max_last_i 

1163 or no_unknown_starts 

1164 or wordlst[max_last_i] not in allowed_unknown_starts 

1165 ): 

1166 # print("RECOVER w={} i={} max_last_i={} wordlst={}" 

1167 # .format(w, i, max_last_i, wordlst)) 

1168 if w in valid_sequences.children: 

1169 max_last_i = add_new( 

1170 # new sequence from root 

1171 valid_sequences.children[w], 

1172 i, 

1173 i, 

1174 max_last_i, 

1175 new_paths, 

1176 new_nodes, 

1177 ) 

1178 cur_nodes = new_nodes # Completely replace nodes! 

1179 # 2023-08-18, fix to improve performance 

1180 # Decode tags does a big search of the best-shortest matching 

1181 # sequences of tags, but the original algorithm didn't have 

1182 # any culling happen during operation, so in a case with 

1183 # a lot of tags (for example, big blocks of text inserted 

1184 # somewhere by mistake that is processed by decode_tags), 

1185 # it would lead to exponential growth of new_paths contents. 

1186 # This culling, using the same weighting algorithm code as 

1187 # in the original is just applied to new_paths before it is 

1188 # added to pos_paths. Basically it's "take the 10 best paths". 

1189 # This *can* cause bugs if it gets stuck in a local minimum 

1190 # or something, but this whole process is one-dimensional 

1191 # and not that complex, so hopefully it works out... 

1192 pw = [] 

1193 path: list[PosPathStep] 

1194 for path in new_paths: 

1195 weight = len(path) 

1196 if any(x[1] == ["UNKNOWN"] for x in path): 

1197 weight += 100 # Penalize unknown paths 

1198 pw.append((weight, path)) 

1199 new_paths = [weightpath[1] for weightpath in sorted(pw)[:10]] 

1200 pos_paths.append(new_paths) 

1201 

1202 # print("END max_last_i={} len(wordlst)={} len(pos_paths)={}" 

1203 # .format(max_last_i, len(wordlst), len(pos_paths))) 

1204 

1205 if cur_nodes: 

1206 # print("END HAVE_NODES") 

1207 for node, start_i, last_i in cur_nodes: 

1208 if node.end: 

1209 # print("$ END start_i={} last_i={}" 

1210 # .format(start_i, last_i)) 

1211 for path in pos_paths[start_i]: 

1212 pos_paths[-1].append( 

1213 [(last_i, node.tags, node.topics)] + path 

1214 ) 

1215 else: 

1216 # print("UNK END start_i={} last_i={} wordlst={}" 

1217 # .format(start_i, last_i, wordlst)) 

1218 u = check_unknown( 

1219 last_i, 

1220 len(wordlst), 

1221 len(wordlst), 

1222 wordlst, 

1223 allow_any, 

1224 no_unknown_starts, 

1225 ) 

1226 if pos_paths[start_i]: 

1227 for path in pos_paths[start_i]: 

1228 pos_paths[-1].append(u + path) 

1229 else: 

1230 pos_paths[-1].append(u) 

1231 else: 

1232 # Check for a final unknown tag 

1233 # print("NO END NODES max_last_i={}".format(max_last_i)) 

1234 paths = pos_paths[max_last_i] or [[]] 

1235 u = check_unknown( 

1236 max_last_i, 

1237 len(wordlst), 

1238 len(wordlst), 

1239 wordlst, 

1240 allow_any, 

1241 no_unknown_starts, 

1242 ) 

1243 if u: 

1244 # print("end max_last_i={}".format(max_last_i)) 

1245 for path in list(paths): # Copy in case it is the last pos 

1246 pos_paths[-1].append(u + path) 

1247 

1248 # import json 

1249 # print("POS_PATHS:", json.dumps(pos_paths, indent=2, sort_keys=True)) 

1250 

1251 if not pos_paths[-1]: 

1252 # print("decode_tags: {}: EMPTY POS_PATHS[-1]".format(src)) 

1253 return [], [] 

1254 

1255 # Find the best path 

1256 pw = [] 

1257 for path in pos_paths[-1]: 

1258 weight = len(path) 

1259 if any(x[1] == ["UNKNOWN"] for x in path): 

1260 weight += 100 # Penalize unknown paths 

1261 pw.append((weight, path)) 

1262 path = min(pw)[1] 

1263 

1264 # Convert the best path to tagsets and topics 

1265 tagsets: list[list[str]] = [[]] 

1266 topics: list[str] = [] 

1267 for i, tagspec, topicspec in path: 

1268 if len(tagsets or "") > 16: 

1269 # ctx.error("Too many tagsets! This is probably exponential", 

1270 # sortid="form_descriptions/20230818") 

1271 return [("error-unknown-tag", "error-exponential-tagsets")], [] 

1272 if tagspec == ["UNKNOWN"]: 

1273 new_tagsets = [] 

1274 for x in tagsets: 

1275 new_tagsets.append(x + topicspec) 

1276 tagsets = new_tagsets 

1277 continue 

1278 if tagspec: 

1279 new_tagsets = [] 

1280 for x in tagsets: 

1281 for t in tagspec: 

1282 if t: 1282 ↛ 1289line 1282 didn't jump to line 1289 because the condition on line 1282 was always true

1283 new_tags = list(x) 

1284 for tag in t.split(): 

1285 if tag not in new_tags: 

1286 new_tags.append(tag) 

1287 new_tagsets.append(new_tags) 

1288 else: 

1289 new_tagsets.append(x) 

1290 tagsets = new_tagsets 

1291 if topicspec: 

1292 for t in topicspec: 

1293 for topic in t.split(): 

1294 if topic not in topics: 

1295 topics.append(topic) 

1296 

1297 # print("unsorted tagsets:", tagsets) 

1298 ret_tagsets = sorted(set(tuple(sorted(set(tags))) for tags in tagsets)) 

1299 # topics = list(sorted(set(topics))) XXX tests expect not sorted 

1300 # print("decode_tags: {} -> {} topics {}".format(src, tagsets, topics)) 

1301 # Yes, ret_tagsets is a list of tags in tuples, while topics is a LIST 

1302 # of tags. Turning topics into a tuple breaks tests, turning the tuples 

1303 # inside tagsets into lists breaks tests, I'm leaving them mismatched 

1304 # for now. XXX 

1305 return ret_tagsets, topics 

1306 

1307 

1308def parse_head_final_tags( 

1309 wxr: WiktextractContext, lang: str, form: str 

1310) -> tuple[str, list[str]]: 

1311 """Parses tags that are allowed at the end of a form head from the end 

1312 of the form. This can also be used for parsing the final gender etc tags 

1313 from translations and linkages.""" 

1314 assert isinstance(wxr, WiktextractContext) 

1315 assert isinstance(lang, str) # Should be language that "form" is for 

1316 assert isinstance(form, str) 

1317 

1318 # print("parse_head_final_tags: lang={} form={!r}".format(lang, form)) 

1319 

1320 # Make sure there are no double spaces in the form as this code does not 

1321 # handle them otherwise. 

1322 form = re.sub(r"\s+", " ", form.strip()) 

1323 if not form: 

1324 return form, [] 

1325 

1326 origform = form 

1327 

1328 tags = [] 

1329 

1330 # If parsing for certain Bantu languages (e.g., Swahili), handle 

1331 # some extra head-final tags first 

1332 if lang in head_final_bantu_langs: 

1333 m = re.search(head_final_bantu_re, form) 

1334 if m is not None: 

1335 tagkeys = m.group(1) 

1336 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1336 ↛ 1351line 1336 didn't jump to line 1351 because the condition on line 1336 was always true

1337 form = form[: m.start()] 

1338 v = head_final_bantu_map[tagkeys] 

1339 if v.startswith("?"): 1339 ↛ 1340line 1339 didn't jump to line 1340 because the condition on line 1339 was never true

1340 v = v[1:] 

1341 wxr.wtp.debug( 

1342 "suspicious suffix {!r} in language {}: {}".format( 

1343 tagkeys, lang, origform 

1344 ), 

1345 sortid="form_descriptions/1028", 

1346 ) 

1347 tags.extend(v.split()) 

1348 

1349 # If parsing for certain Semitic languages (e.g., Arabic), handle 

1350 # some extra head-final tags first 

1351 if lang in head_final_semitic_langs: 

1352 m = re.search(head_final_semitic_re, form) 

1353 if m is not None: 

1354 tagkeys = m.group(1) 

1355 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1355 ↛ 1370line 1355 didn't jump to line 1370 because the condition on line 1355 was always true

1356 form = form[: m.start()] 

1357 v = head_final_semitic_map[tagkeys] 

1358 if v.startswith("?"): 1358 ↛ 1359line 1358 didn't jump to line 1359 because the condition on line 1358 was never true

1359 v = v[1:] 

1360 wxr.wtp.debug( 

1361 "suspicious suffix {!r} in language {}: {}".format( 

1362 tagkeys, lang, origform 

1363 ), 

1364 sortid="form_descriptions/1043", 

1365 ) 

1366 tags.extend(v.split()) 

1367 

1368 # If parsing for certain other languages (e.g., Lithuanian, 

1369 # French, Finnish), handle some extra head-final tags first 

1370 if lang in head_final_other_langs: 

1371 m = re.search(head_final_other_re, form) 

1372 if m is not None: 

1373 tagkeys = m.group(1) 

1374 if not wxr.wtp.title.endswith(tagkeys): # type:ignore[union-attr] 1374 ↛ 1380line 1374 didn't jump to line 1380 because the condition on line 1374 was always true

1375 form = form[: m.start()] 

1376 tags.extend(head_final_other_map[tagkeys].split(" ")) 

1377 

1378 # Handle normal head-final tags 

1379 # Loop this until nothing is found 

1380 while True: 

1381 prev_form = form 

1382 m = re.search(head_final_re, form) 

1383 if m is not None: 

1384 # print(f"{m=}, {m.groups()=}") 

1385 tagkeys = m.group(3) 

1386 # Only replace tags ending with numbers in languages that have 

1387 # head-final numeric tags (e.g., Bantu classes); also, don't replace 

1388 # tags if the main title ends with them (then presume they are part 

1389 # of the word) 

1390 # print("head_final_tags form={!r} tagkeys={!r} lang={}" 

1391 # .format(form, tagkeys, lang)) 

1392 tagkeys_contains_digit = re.search(r"\d", tagkeys) 

1393 if ( 

1394 (not tagkeys_contains_digit or lang in head_final_numeric_langs) 

1395 and not wxr.wtp.title.endswith(" " + tagkeys) # type:ignore[union-attr] 

1396 and 

1397 # XXX the above test does not capture when the whole word is a 

1398 # xlat_head_map key, so I added the below test to complement 

1399 # it; does this break anything? 

1400 not wxr.wtp.title == tagkeys 

1401 ): # defunct/English, 

1402 # "more defunct" -> "more" ["archaic"] 

1403 if ( 1403 ↛ 1421line 1403 didn't jump to line 1421 because the condition on line 1403 was always true

1404 not tagkeys_contains_digit 

1405 or lang in head_final_numeric_langs 

1406 ): 

1407 # m.start(3) gets the start of what is in m.group(3), handy 

1408 form = form[: m.start(3)].strip() 

1409 v = xlat_head_map[tagkeys] 

1410 if v.startswith("?"): 1410 ↛ 1411line 1410 didn't jump to line 1411 because the condition on line 1410 was never true

1411 v = v[1:] 

1412 wxr.wtp.debug( 

1413 "suspicious suffix {!r} in language {}: {}".format( 

1414 tagkeys, lang, origform 

1415 ), 

1416 sortid="form_descriptions/1077", 

1417 ) 

1418 tags.extend(v.split()) 

1419 else: 

1420 break 

1421 if prev_form == form: 

1422 break 

1423 

1424 # Generate warnings about words ending in " or" after processing 

1425 if ( 

1426 (form.endswith(" or") and not origform.endswith(" or")) 

1427 or re.search( 

1428 r" (1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|" 

1429 r"1a|2a|9a|10a|m1|f1|f2|m2|f3|m3|f4|m4|f5|m5|or|\?)" 

1430 r"($|/| (f|m|sg|pl|anim|inan))", 

1431 form, 

1432 ) 

1433 or form.endswith(" du") 

1434 ): 

1435 if form not in ok_suspicious_forms: 

1436 wxr.wtp.debug( 

1437 "suspicious unhandled suffix in {}:" 

1438 " {!r}, originally {!r}".format(lang, form, origform), 

1439 sortid="form_descriptions/1089", 

1440 ) 

1441 

1442 # print("parse_head_final_tags: form={!r} tags={}".format(form, tags)) 

1443 return form, tags 

1444 

1445 

1446def quote_kept_parens(s: str) -> str: 

1447 """Changes certain parenthesized expressions so that they won't be 

1448 interpreted as parentheses. This is used for parts that are kept as 

1449 part of the word, such as "read admiral (upper half)".""" 

1450 return re.sub( 

1451 r"\((lower half|upper half|k|s|n|II|III|A|C|G|U|Y|" 

1452 r"vinyl|p-phenylene vinylene|\(\(\s*\)\))\)", 

1453 r"__lpar__\1__rpar__", 

1454 s, 

1455 ) 

1456 

1457 

1458def quote_kept_ruby( 

1459 wxr: WiktextractContext, 

1460 ruby_tuples: list[ 

1461 tuple[ 

1462 str, 

1463 str, 

1464 ] 

1465 ], 

1466 s: str, 

1467) -> str: 

1468 if len(ruby_tuples) < 1: 1468 ↛ 1469line 1468 didn't jump to line 1469 because the condition on line 1468 was never true

1469 wxr.wtp.debug( 

1470 "quote_kept_ruby called with no ruby", 

1471 sortid="form_description/1114/20230517", 

1472 ) 

1473 return s 

1474 ks = [] 

1475 rs = [] 

1476 for k, r in ruby_tuples: 

1477 ks.append(re.escape(k)) 

1478 rs.append(re.escape(r)) 

1479 if not (ks and rs): 1479 ↛ 1480line 1479 didn't jump to line 1480 because the condition on line 1479 was never true

1480 wxr.wtp.debug( 

1481 f"empty column in ruby_tuples: {ruby_tuples}", 

1482 sortid="form_description/1124/20230606", 

1483 ) 

1484 return s 

1485 newm = re.compile( 

1486 r"({})\s*\(\s*({})\s*\)".format("|".join(ks), "|".join(rs)) 

1487 ) 

1488 rub_re = re.compile( 

1489 r"({})".format( 

1490 r"|".join( 

1491 r"{}\(*{}\)*".format( 

1492 re.escape(k), 

1493 re.escape(r), 

1494 ) 

1495 for k, r in ruby_tuples 

1496 ) 

1497 ) 

1498 ) 

1499 

1500 def paren_replace(m: re.Match) -> str: 

1501 return re.sub(newm, r"\1__lrub__\2__rrub__", m.group(0)) 

1502 

1503 return re.sub(rub_re, paren_replace, s) 

1504 

1505 

1506def unquote_kept_parens(s: str) -> str: 

1507 """Converts the quoted parentheses back to normal parentheses.""" 

1508 return re.sub(r"__lpar__(.*?)__rpar__", r"(\1)", s) 

1509 

1510 

1511def add_romanization( 

1512 wxr: WiktextractContext, 

1513 data: WordData, 

1514 roman: str, 

1515 text: str, 

1516 is_reconstruction: bool, 

1517 head_group: Optional[int], 

1518 ruby: Sequence[tuple[str, str]], 

1519) -> None: 

1520 tags_lst = ["romanization"] 

1521 m = re.match(r"([^:]+):(.+)", roman) 

1522 # This function's purpose is to intercept broken romanizations, 

1523 # like "Yale: hēnpyeng" style tags. Most romanization styles 

1524 # are already present as tags, so we can use decode_tags to find 

1525 # them. 

1526 if m: 1526 ↛ 1527line 1526 didn't jump to line 1527 because the condition on line 1526 was never true

1527 tagsets, topics = decode_tags(m.group(1)) 

1528 if tagsets: 

1529 for tags in tagsets: 

1530 tags_lst.extend(tags) 

1531 roman = m.group(2) 

1532 add_related( 

1533 wxr, 

1534 data, 

1535 tags_lst, 

1536 [roman], 

1537 text, 

1538 True, 

1539 is_reconstruction, 

1540 head_group, 

1541 ruby, 

1542 ) 

1543 

1544 

1545def add_related( 

1546 wxr: WiktextractContext, 

1547 data: WordData, 

1548 tags_lst: Union[list[str], tuple[str, ...]], 

1549 related_list: list[str], 

1550 origtext: str, 

1551 add_all_canonicals: bool, 

1552 is_reconstruction: bool, 

1553 head_group: Optional[int], 

1554 ruby_data: Optional[Sequence[tuple[str, str]]] = None, 

1555 links: list[tuple[str, str]] | None = None, 

1556 link_dict: dict[str, list[str]] | None = None, 

1557) -> Optional[list[tuple[str, ...]]]: 

1558 """Internal helper function for some post-processing entries for related 

1559 forms (e.g., in word head). This returns a list of list of tags to be 

1560 added to following related forms or None (cf. walrus/English word head, 

1561 parenthesized part starting with "both").""" 

1562 assert isinstance(wxr, WiktextractContext) 

1563 assert isinstance(tags_lst, (list, tuple)) 

1564 for x in tags_lst: 

1565 assert isinstance(x, str) 

1566 assert isinstance(related_list, (list, tuple)) 

1567 assert isinstance(origtext, str) 

1568 assert add_all_canonicals in (True, False) 

1569 assert isinstance(ruby_data, (list, tuple)) or ruby_data is None 

1570 if ruby_data is None: 1570 ↛ 1571line 1570 didn't jump to line 1571 because the condition on line 1570 was never true

1571 ruby_data = [] 

1572 related = " ".join(related_list) 

1573 # print("add_related: tags_lst={} related={}".format(tags_lst, related)) 

1574 if related == "[please provide]": 1574 ↛ 1575line 1574 didn't jump to line 1575 because the condition on line 1574 was never true

1575 return None 

1576 if related in IGNORED_RELATED: 1576 ↛ 1577line 1576 didn't jump to line 1577 because the condition on line 1576 was never true

1577 return None 

1578 if is_reconstruction and related.startswith("*") and len(related) > 1: 

1579 related = related[1:] 

1580 

1581 # print(f"{links=}, {link_dict=}") 

1582 # Get title word, with any reconstruction prefix removed 

1583 titleword = re.sub(r"^Reconstruction:[^/]*/", "", wxr.wtp.title) # type:ignore[arg-type] 

1584 

1585 def check_related(related: str) -> None: 

1586 # Warn about some suspicious related forms 

1587 m = re.search(suspicious_related_re, related) 

1588 if (m and m.group(0) not in titleword) or ( 

1589 related in ("f", "m", "n", "c") and len(titleword) >= 3 

1590 ): 

1591 if "eumhun" in tags_lst: 1591 ↛ 1592line 1591 didn't jump to line 1592 because the condition on line 1591 was never true

1592 return 

1593 if "cangjie-input" in tags_lst: 1593 ↛ 1594line 1593 didn't jump to line 1594 because the condition on line 1593 was never true

1594 return 

1595 if "class" in tags_lst: 1595 ↛ 1596line 1595 didn't jump to line 1596 because the condition on line 1595 was never true

1596 return 

1597 if wxr.wtp.section == "Korean" and re.search( 1597 ↛ 1601line 1597 didn't jump to line 1601 because the condition on line 1597 was never true

1598 r"^\s*\w*>\w*\s*$", related 

1599 ): 

1600 # ignore Korean "i>ni" / "라>나" values 

1601 return 

1602 if ( 1602 ↛ 1609line 1602 didn't jump to line 1609 because the condition on line 1602 was never true

1603 wxr.wtp.section == "Burmese" 

1604 and "romanization" in tags_lst 

1605 and re.search(r":", related) 

1606 ): 

1607 # ignore Burmese with ":", that is used in Burmese 

1608 # translitteration of "း", the high-tone visarga. 

1609 return 

1610 wxr.wtp.debug( 

1611 "suspicious related form tags {}: {!r} in {!r}".format( 

1612 tags_lst, related, origtext 

1613 ), 

1614 sortid="form_descriptions/1147", 

1615 ) 

1616 

1617 following_tagsets = None # Tagsets to add to following related forms 

1618 roman = None 

1619 tagsets1: list[tuple[str, ...]] = [tuple()] 

1620 topics1: list[str] = [] 

1621 

1622 m = re.match(r"\((([^()]|\([^()]*\))*)\)\s+", related) 

1623 if m: 

1624 paren = m.group(1) 

1625 related = related[m.end() :] 

1626 m = re.match(r"^(all|both) (.*)", paren) 

1627 if m: 1627 ↛ 1628line 1627 didn't jump to line 1628 because the condition on line 1627 was never true

1628 tagsets1, topics1 = decode_tags(m.group(2)) 

1629 following_tagsets = tagsets1 

1630 else: 

1631 tagsets1, topics1 = decode_tags(paren) 

1632 else: 

1633 m = re.search(r"\s+\((([^()]|\([^()]*\))*)\)$", related) 

1634 if m: 

1635 paren = m.group(1) 

1636 if paren.startswith("U+"): 1636 ↛ 1637line 1636 didn't jump to line 1637 because the condition on line 1636 was never true

1637 related = related[: m.start()] 

1638 else: 

1639 cls = classify_desc(paren) 

1640 if ( 1640 ↛ 1647line 1640 didn't jump to line 1647 because the condition on line 1640 was always true

1641 cls in ("romanization", "english") 

1642 and classify_desc(related[: m.start()]) == "other" 

1643 ): 

1644 roman = paren 

1645 related = related[: m.start()] 

1646 else: 

1647 related = related[: m.start()] 

1648 tagsets1, topics1 = decode_tags(paren) 

1649 if related and related.startswith("{{"): 1649 ↛ 1650line 1649 didn't jump to line 1650 because the condition on line 1649 was never true

1650 wxr.wtp.debug( 

1651 "`{{` in word head form - possible Wiktionary error: {!r}".format( 

1652 related 

1653 ), 

1654 sortid="form_descriptions/1177", 

1655 ) 

1656 return None # Likely Wiktionary coding error 

1657 related = unquote_kept_parens(related) 

1658 # Split related by "/" (e.g., grande/Spanish) superlative in head 

1659 # Do not split if / in word title, see π//Japanese 

1660 if len(related) > 5 and "/" not in wxr.wtp.title: # type:ignore[operator] 

1661 alts = split_at_comma_semi(related, separators=["/"]) 

1662 else: 

1663 alts = [related] 

1664 if ruby_data: 

1665 # prepare some regex stuff in advance 

1666 ks, rs = [], [] 

1667 for k, r in ruby_data: 

1668 ks.append(re.escape(k)) 

1669 rs.append(re.escape(r)) 

1670 splitter = r"((?:{})__lrub__(?:{})__rrub__)".format( 

1671 "|".join(ks), "|".join(rs) 

1672 ) 

1673 for related in alts: 

1674 ruby: list[tuple[str, str]] = [] 

1675 if ruby_data: 

1676 new_related = [] 

1677 rub_split = re.split(splitter, related) 

1678 for s in rub_split: 

1679 m = re.match(r"(.+)__lrub__(.+)__rrub__", s) 

1680 if m: 

1681 # add ruby with (\1, \2) 

1682 ruby.append((m.group(1), m.group(2))) 

1683 new_related.append(m.group(1)) 

1684 else: 

1685 new_related.append(s) 

1686 related = "".join(new_related) 

1687 tagsets2, topics2 = decode_tags(" ".join(tags_lst)) 

1688 for tags1 in tagsets1: 

1689 assert isinstance(tags1, (list, tuple)) 

1690 for tags2 in tagsets2: 

1691 assert isinstance(tags1, (list, tuple)) 

1692 dt: LinkageData = {"word": related} 

1693 if roman: 

1694 dt["roman"] = roman 

1695 if ruby: 

1696 dt["ruby"] = ruby 

1697 if "alt-of" in tags2: 1697 ↛ 1698line 1697 didn't jump to line 1698 because the condition on line 1697 was never true

1698 check_related(related) 

1699 data_extend(data, "tags", tags1) 

1700 data_extend(data, "tags", tags2) 

1701 data_extend(data, "topics", topics1) 

1702 data_extend(data, "topics", topics2) 

1703 data_append(data, "alt_of", dt) 

1704 elif "form-of" in tags2: 1704 ↛ 1705line 1704 didn't jump to line 1705 because the condition on line 1704 was never true

1705 check_related(related) 

1706 data_extend(data, "tags", tags1) 

1707 data_extend(data, "tags", tags2) 

1708 data_extend(data, "topics", topics1) 

1709 data_extend(data, "topics", topics2) 

1710 data_append(data, "form_of", dt) 

1711 elif "compound-of" in tags2: 1711 ↛ 1712line 1711 didn't jump to line 1712 because the condition on line 1711 was never true

1712 check_related(related) 

1713 data_extend(data, "tags", tags1) 

1714 data_extend(data, "tags", tags2) 

1715 data_extend(data, "topics", topics1) 

1716 data_extend(data, "topics", topics2) 

1717 data_append(data, "compound", related) 

1718 else: 

1719 lang = wxr.wtp.section or "LANG_MISSING" 

1720 related, final_tags = parse_head_final_tags( 

1721 wxr, lang, related 

1722 ) 

1723 # print("add_related: related={!r} tags1={!r} tags2={!r} " 

1724 # "final_tags={!r}" 

1725 # .format(related, tags1, tags2, final_tags)) 

1726 tags = list(tags1) + list(tags2) + list(final_tags) 

1727 check_related(related) 

1728 form: FormData = {"form": related} 

1729 if ( 

1730 links 

1731 and link_dict 

1732 and ( 

1733 form_links := match_links_to_form( 

1734 wxr, related, links, link_dict 

1735 ) 

1736 ) 

1737 ): 

1738 form["links"] = form_links 

1739 if head_group: 

1740 form["head_nr"] = head_group 

1741 if roman: 

1742 form["roman"] = roman 

1743 if ruby: 

1744 form["ruby"] = ruby 

1745 data_extend(form, "topics", topics1) 

1746 data_extend(form, "topics", topics2) 

1747 if topics1 or topics2: 1747 ↛ 1748line 1747 didn't jump to line 1748 because the condition on line 1747 was never true

1748 wxr.wtp.debug( 

1749 "word head form has topics: {}".format(form), 

1750 sortid="form_descriptions/1233", 

1751 ) 

1752 # Add tags from canonical form into the main entry 

1753 if "canonical" in tags: 

1754 if related in ("m", "f") and len(titleword) > 1: 1754 ↛ 1755line 1754 didn't jump to line 1755 because the condition on line 1754 was never true

1755 wxr.wtp.debug( 

1756 "probably incorrect canonical form " 

1757 "{!r} ignored (probably tag combination " 

1758 "missing from xlat_head_map)".format(related), 

1759 sortid="form_descriptions/1241", 

1760 ) 

1761 continue 

1762 if ( 

1763 related != titleword 

1764 or add_all_canonicals 

1765 or topics1 

1766 or topics2 

1767 or ruby 

1768 ): 

1769 data_extend(form, "tags", sorted(set(tags))) 

1770 else: 

1771 # We won't add canonical form here 

1772 filtered_tags = list( 

1773 x for x in tags if x != "canonical" 

1774 ) 

1775 data_extend(data, "tags", filtered_tags) 

1776 continue 

1777 else: 

1778 data_extend(form, "tags", sorted(set(tags))) 

1779 # Only insert if the form is not already there 

1780 for old in data.get("forms", ()): 

1781 if form == old: 1781 ↛ 1782line 1781 didn't jump to line 1782 because the condition on line 1781 was never true

1782 break 

1783 else: 

1784 data_append(data, "forms", form) 

1785 

1786 # If this form had pre-tags that started with "both" or "all", add those 

1787 # tags also to following related forms that don't have their own tags 

1788 # specified. 

1789 return following_tagsets 

1790 

1791 

1792def match_links_to_form( 

1793 wxr: WiktextractContext, 

1794 form: str, 

1795 links: list[tuple[str, str]], 

1796 link_dict: dict[str, list[str]] | None, 

1797) -> list[tuple[str, str]] | None: 

1798 if not links: 1798 ↛ 1799line 1798 didn't jump to line 1799 because the condition on line 1798 was never true

1799 return None 

1800 if link_dict is None: 

1801 link_dict = {} 

1802 for ltxt, ltrg in links: 

1803 if ltxt not in link_dict: 

1804 link_dict[ltxt] = [ 

1805 ltrg, 

1806 ] 

1807 else: 

1808 link_dict[ltxt].append(ltrg) 

1809 ret: list[tuple[str, str]] = [] 

1810 if form in link_dict: 

1811 if len(link_dict[form]) > 1 and any( 1811 ↛ 1814line 1811 didn't jump to line 1814 because the condition on line 1811 was never true

1812 x != link_dict[form][0] for x in link_dict[form] 

1813 ): 

1814 wxr.wtp.warning( 

1815 f"{form=} has many different " 

1816 f"link candidates `{link_dict[form]}`, " 

1817 f"which can't be disambiguated.", 

1818 sortid="form_descriptions/match_links_to_form", 

1819 ) 

1820 for ltarg in link_dict[form]: 

1821 ret.append((form, ltarg)) 

1822 elif " " in form: 

1823 # split and search for a sequence of links... 

1824 split_forms = form.split() 

1825 found = False 

1826 for i, (ltext, ltarg) in enumerate(links): 

1827 if ltext == split_forms[0]: 

1828 for j, f in enumerate(split_forms): 

1829 if i + j >= len(links): 

1830 break 

1831 if f.strip(",;() ") != links[i + j][0].strip(",;() "): 

1832 break 

1833 if i + j == len(links): 1833 ↛ 1834line 1833 didn't jump to line 1834 because the condition on line 1833 was never true

1834 break 

1835 else: 

1836 found = True 

1837 if found: 

1838 ret = links[i : i + len(split_forms)] 

1839 break 

1840 # We only care about weird links 

1841 # print(f"{len(ret)=}, {ret}") 

1842 for txt, tar in ret: 

1843 if txt != tar and txt != tar[: tar.find("#")]: 

1844 break 

1845 else: 

1846 return None 

1847 return ret or None 

1848 

1849 

1850# Issue #967, in English word forms sometimes forms are skipped because 

1851# they are taggable words and their distw() is too big, like clipping from clip 

1852WORDS_WITH_FALSE_POSITIVE_TAGS: dict[str, list[str]] = { 

1853 "clip": ["clipping"], # XXX remember to change me back to clipping after 

1854 "English": ["English", "Englishes"], 

1855 "common": ["common", "commoner"], 

1856 # tests. 

1857} 

1858 

1859WORDS_WITH_FALSE_POSITIVE_FORMS: dict[str, list[str]] = { 

1860 "unaccountability": ["countable", "uncountable"], 

1861 "uncountability": ["countable", "uncountable"], 

1862} 

1863 

1864FALSE_POSITIVE_MISSING_FORMS: dict[str, list[str]] = {} 

1865 

1866FORM_ASSOCIATED_TAG_WORDS: set[str] = { 

1867 "participle", 

1868 "past", 

1869 "present", 

1870 "singular", 

1871 "plural", 

1872 "first-person", 

1873 "second-person", 

1874 "third-person", 

1875 "gerund", 

1876} 

1877 

1878SEMICOLON_REPLACEMENT = "__SEMICOLON__" 

1879 

1880 

1881def parse_word_head( 

1882 wxr: WiktextractContext, 

1883 word: str, 

1884 pos: str, 

1885 text: str, 

1886 data: WordData, 

1887 is_reconstruction: bool, 

1888 head_group: Optional[int], 

1889 original_header_nodes: list[WikiNode | str] | None = None, 

1890 ruby=None, 

1891 links: list[ 

1892 tuple[ 

1893 str, 

1894 str, 

1895 ] 

1896 ] 

1897 | None = None, 

1898 label_templates: list[TemplateData] | None = None, 

1899) -> None: 

1900 """Parses the head line for a word for in a particular language and 

1901 part-of-speech, extracting tags and related forms.""" 

1902 assert isinstance(wxr, WiktextractContext) 

1903 assert isinstance(pos, str) 

1904 assert isinstance(text, str) 

1905 assert isinstance(data, dict) 

1906 assert isinstance(ruby, (list, tuple)) or ruby is None 

1907 if ruby is None: 

1908 ruby = [] 

1909 assert is_reconstruction in (True, False) 

1910 # print("PARSE_WORD_HEAD: {}: {!r}".format(wxr.wtp.section, text)) 

1911 # print(f"PARSE_WORD_HEAD: {data=}") 

1912 # print(f"PARSE_WORD_HEAD: {links=}") 

1913 

1914 # Save original text for if we want to look for mismatched form-links 

1915 

1916 link_dict: dict[str, list[str]] | None 

1917 if links is not None: 

1918 link_dict = {} 

1919 for ltxt, ltrg in links: 

1920 if ltxt not in link_dict: 

1921 link_dict[ltxt] = [ 

1922 ltrg, 

1923 ] 

1924 else: 

1925 link_dict[ltxt].append(ltrg) 

1926 else: 

1927 link_dict = None 

1928 

1929 # print(f"MAIN: {links=}") 

1930 link_words_not_alnum = [] 

1931 if not word.isalnum() and not word.replace("-", "").isalnum(): 

1932 # `-` is kosher, add more of these if needed. 

1933 # if the word contains non-letter or -number characters, it 

1934 # might have something that messes with split-at-semi-comma; we 

1935 # collect links so that we can skip splitting them. 

1936 if links is None and original_header_nodes is not None: 

1937 links, _ = extract_links_from_node( 

1938 wxr, 

1939 original_header_nodes, 

1940 remove_anchor_tags=True, 

1941 expand_nodes=True, 

1942 ) 

1943 if links is not None: 1943 ↛ 1947line 1943 didn't jump to line 1947 because the condition on line 1943 was always true

1944 for ltext, ltar in links: 

1945 if not ltext.isalnum(): 

1946 link_words_not_alnum.append(ltext) 

1947 if word not in link_words_not_alnum: 1947 ↛ 1950line 1947 didn't jump to line 1950 because the condition on line 1947 was always true

1948 link_words_not_alnum.append(word) 

1949 

1950 if link_words_not_alnum is None: 1950 ↛ 1951line 1950 didn't jump to line 1951 because the condition on line 1950 was never true

1951 link_words_not_alnum = [] 

1952 

1953 if len(link_words_not_alnum) > 0: 

1954 # if we have link data (that is, links with stuff like commas and 

1955 # spaces, replace word_re with a modified local scope pattern 

1956 # print(f"links {list((c, ord(c)) for link in links for c in link)=}") 

1957 word_re = re.compile( 

1958 r"\b" # In case we have forms that are longer and contain links 

1959 + 

1960 # or words as a substring... 

1961 r"\b|\b".join( 

1962 sorted( 

1963 (re.escape(s) for s in link_words_not_alnum), 

1964 key=lambda x: -len(x), 

1965 ) 

1966 ) 

1967 + r"\b|" 

1968 + word_pattern 

1969 ) 

1970 else: 

1971 word_re = word_re_global 

1972 

1973 if "Lua execution error" in text or "Lua timeout error" in text: 1973 ↛ 1974line 1973 didn't jump to line 1974 because the condition on line 1973 was never true

1974 return 

1975 

1976 # Fix words with "superlative:" or "comparative:" at end of head 

1977 # e.g. grande/Spanish/Adj 

1978 text = re.sub(r" (superlative|comparative): (.*)", r" (\1 \2)", text) 

1979 

1980 # Parse Arabic non-past forms, e.g. أبلع/Arabic/Verb 

1981 m = re.search(r", non-past ([^)]+ \([^)]+\))", text) 

1982 if m: 

1983 add_related( 

1984 wxr, 

1985 data, 

1986 ["non-past"], 

1987 [m.group(1)], 

1988 text, 

1989 True, 

1990 is_reconstruction, 

1991 head_group, 

1992 ruby, 

1993 links, 

1994 link_dict, 

1995 ) 

1996 text = text[: m.start()] + text[m.end() :] 

1997 

1998 language = wxr.wtp.section 

1999 titleword = re.sub( 

2000 r"^Reconstruction:[^/]*/", "", wxr.wtp.title or "MISSING_TITLE" 

2001 ) 

2002 titleparts = list( 

2003 m.group(0) 

2004 for m in re.finditer(word_re, wxr.wtp.title or "MISSING_TITLE") 

2005 ) 

2006 if not titleparts: 2006 ↛ 2007line 2006 didn't jump to line 2007 because the condition on line 2006 was never true

2007 return 

2008 

2009 # Remove " or" from the end to prevent weird canonical forms 

2010 if text.endswith(" or"): 

2011 for tp in titleparts: 

2012 if text.endswith(tp): 2012 ↛ 2013line 2012 didn't jump to line 2013 because the condition on line 2012 was never true

2013 break 

2014 else: 

2015 text = text.removesuffix(" or").rstrip() 

2016 

2017 # Handle the part of the head that is not in parentheses. However, certain 

2018 # parenthesized parts are part of word, and those must be handled 

2019 # specially here. 

2020 if ruby: 

2021 text = quote_kept_ruby(wxr, ruby, text) 

2022 base = text 

2023 base = quote_kept_parens(base) 

2024 base = remove_text_in_parentheses(base) 

2025 base = base.replace("?", "") # Removes uncertain articles etc 

2026 base = re.sub(r"\s+", " ", base) 

2027 base = re.sub(r" ([,;])", r"\1", base) 

2028 base = re.sub(r" • ", r" ", base) 

2029 # Many languages use • as a punctuation mark separating the base 

2030 # from the rest of the head. στάδιος/Ancient Greek, issue #176 

2031 base = base.strip() 

2032 # print(f"{base=}, {text=}") 

2033 

2034 # Check for certain endings in head (mostly for compatibility with weird 

2035 # heads, e.g. rata/Romanian "1st conj." at end) 

2036 m = re.search(head_end_re, base) 

2037 tags: Union[tuple[str, ...], list[str]] = [] 

2038 if m: 2038 ↛ 2039line 2038 didn't jump to line 2039 because the condition on line 2038 was never true

2039 tags = head_end_map[m.group(1).lower()].split() 

2040 data_extend(data, "tags", tags) 

2041 base = base[: m.start()] 

2042 

2043 if language == "Japanese": 

2044 # Remove the leftover "^" of the historical kana superscript, 

2045 # e.g. 静か "^(←しづか)-na" 

2046 base = re.sub(r" \^(?!†)", " ", base).strip() 

2047 while m := ja_head_end_re.search(base): 

2048 tag = m.group(1) 

2049 if tag == "uninflectable": 2049 ↛ 2050line 2049 didn't jump to line 2050 because the condition on line 2049 was never true

2050 tag = "uninflected" 

2051 data_extend(data, "tags", [tag]) 

2052 base = base[: m.start()] 

2053 

2054 # Special case: handle Hán Nôm readings for Vietnamese characters 

2055 m = re.match( 

2056 r"{}: (Hán Nôm) readings: (.*)".format(re.escape(titleword)), base 

2057 ) 

2058 if m: 2058 ↛ 2059line 2058 didn't jump to line 2059 because the condition on line 2058 was never true

2059 tag, readings = m.groups() 

2060 tag = re.sub(r"\s+", "-", tag) 

2061 for reading in split_at_comma_semi( 

2062 readings, skipped=link_words_not_alnum 

2063 ): 

2064 add_related( 

2065 wxr, 

2066 data, 

2067 [tag], 

2068 [reading], 

2069 text, 

2070 True, 

2071 is_reconstruction, 

2072 head_group, 

2073 ruby, 

2074 links, 

2075 link_dict, 

2076 ) 

2077 return 

2078 

2079 # Special case: Hebrew " [pattern: nnn]" ending 

2080 m = re.search(r"\s+\[pattern: ([^]]+)\]", base) 

2081 if m: 2081 ↛ 2082line 2081 didn't jump to line 2082 because the condition on line 2081 was never true

2082 add_related( 

2083 wxr, 

2084 data, 

2085 ["class"], 

2086 [m.group(1)], 

2087 text, 

2088 True, 

2089 is_reconstruction, 

2090 head_group, 

2091 ruby, 

2092 links, 

2093 link_dict, 

2094 ) 

2095 base = base[: m.start()] + base[m.end() :] 

2096 

2097 # Clean away some messy "Upload an image" template text used in 

2098 # American Sign Language: 

2099 # S@NearBaseForearm-PalmUp Frontandback S@BaseForearm-PalmUp 

2100 m = re.search(r"Upload .+ gif image.", base) 

2101 if m: 2101 ↛ 2102line 2101 didn't jump to line 2102 because the condition on line 2101 was never true

2102 base = base[: m.start()] + base[m.end() :] 

2103 

2104 semicolon_present = False 

2105 # Split the head into alternatives. This is a complicated task, as 

2106 # we do not want so split on "or" or "," when immediately followed by more 

2107 # head-final tags, but otherwise do want to split by them. 

2108 # 20230907 added "or" to this to handle 'true or false', titles with 'or' 

2109 if wxr.wtp.title and ( 

2110 "," in wxr.wtp.title or ";" in wxr.wtp.title or " or " in wxr.wtp.title 

2111 ): 

2112 # If the title has ";", we don't want to split on that and can remove 

2113 # the ; from the splitting regex pretty easily because it's uncommon. 

2114 # However, commas are so common that not splitting on them is just 

2115 # not feasible, and we have to just deal with that if there are 

2116 # alternative forms or variations with stray commas that shouldn't 

2117 # be split. 

2118 if ";" in wxr.wtp.title: 

2119 semicolon_present = True 

2120 base = base.replace(";", SEMICOLON_REPLACEMENT) 

2121 default_splitter = head_split_no_semicolon_re 

2122 else: 

2123 default_splitter = head_split_re 

2124 # A kludge to handle article titles/phrases with commas. 

2125 # Preprocess splits to first capture the title, then handle 

2126 # all the others as usual. 

2127 presplits = re.split(r"({})".format(wxr.wtp.title), base) 

2128 splits = [] 

2129 for psplit in presplits: 

2130 if psplit == wxr.wtp.title: 

2131 splits.append(psplit) 

2132 else: 

2133 splits.extend(re.split(default_splitter, psplit)) 

2134 else: 

2135 # Do the normal split; previous behavior. 

2136 splits = re.split(head_split_re, base) 

2137 # print("BASE: ", repr(base)) 

2138 # print("SPLITS:", splits) 

2139 alts: list[str] = [] 

2140 # print("parse_word_head: splits:", splits, 

2141 # "head_split_re_parens:", head_split_re_parens) 

2142 for i in range( 

2143 0, len(splits) - head_split_re_parens, head_split_re_parens + 1 

2144 ): 

2145 v = splits[i] 

2146 ending = splits[i + 1] or "" # XXX is this correct??? 

2147 # print("parse_word_head alts v={!r} ending={!r} alts={}" 

2148 # .format(v, ending, alts)) 

2149 if alts and (v == "" and ending): 

2150 assert ending[0] == " " 

2151 alts[-1] += " or" + ending # endings starts with space 

2152 elif v or ending: 

2153 alts.append((v or "") + (ending or "")) 

2154 last = splits[-1].strip() 

2155 conn = "" if len(splits) < 3 else splits[-2] 

2156 # print("parse_word_head alts last={!r} conn={!r} alts={}" 

2157 # .format(last, conn, alts)) 

2158 if ( 2158 ↛ 2169line 2158 didn't jump to line 2169 because the condition on line 2158 was never true

2159 alts 

2160 and last 

2161 and ( 

2162 last.split()[0] in xlat_head_map 

2163 or ( 

2164 conn == " or " 

2165 and (alts[-1] + " or " + last).strip() in xlat_head_map 

2166 ) 

2167 ) 

2168 ): 

2169 alts[-1] += " or " + last 

2170 elif last: 2170 ↛ 2171line 2170 didn't jump to line 2171 because the condition on line 2170 was never true

2171 alts.append(last) 

2172 

2173 # print("parse_word_head alts: {}".format(alts)) 

2174 # print(f"{base=}") 

2175 

2176 # Process the head alternatives 

2177 canonicals: list[tuple[list[str], list[str]]] = [] 

2178 mode: Optional[str] = None 

2179 for alt_i, alt in enumerate(alts): 

2180 alt = alt.strip() 

2181 if alt.startswith("compound form:"): 2181 ↛ 2182line 2181 didn't jump to line 2182 because the condition on line 2181 was never true

2182 mode = "compound-form" 

2183 alt = alt[14:].strip() 

2184 if ((dash_i := alt.find(" -")) > 0) and ( 2184 ↛ 2190line 2184 didn't jump to line 2190 because the condition on line 2184 was never true

2185 dash_i > (wxr.wtp.title or "").find(" -") 

2186 ): 

2187 # test_en_head / test_suffixes_at_end_of_form1 

2188 # Some heads have suffixes that end up attached to the form 

2189 # like in https://en.wiktionary.org/wiki/%E6%A5%BD%E3%81%97%E3%81%84 

2190 alt = alt[:dash_i] 

2191 if mode == "compound-form": 2191 ↛ 2192line 2191 didn't jump to line 2192 because the condition on line 2191 was never true

2192 add_related( 

2193 wxr, 

2194 data, 

2195 ["in-compounds"], 

2196 [alt], 

2197 text, 

2198 True, 

2199 is_reconstruction, 

2200 head_group, 

2201 ruby, 

2202 links, 

2203 link_dict, 

2204 ) 

2205 continue 

2206 # For non-first parts, see if it can be treated as tags-only 

2207 if alt_i == 0: 

2208 expanded_alts = [alt] 

2209 else: 

2210 expanded_alts = map_with(xlat_descs_map, [alt]) 

2211 # print("EXPANDED_ALTS:", expanded_alts) 

2212 tagsets: Optional[list[tuple[str, ...]]] 

2213 for alt in expanded_alts: 

2214 baseparts = list(m.group(0) for m in word_re.finditer(alt)) 

2215 if alt_i > 0: 

2216 tagsets, topics = decode_tags(" ".join(baseparts)) 

2217 if not any("error-unknown-tag" in x for x in tagsets): 

2218 data_extend(data, "topics", topics) 

2219 for tags1 in tagsets: 

2220 data_extend(data, "tags", tags1) 

2221 continue 

2222 

2223 alt, tags = parse_head_final_tags( 

2224 wxr, language or "MISSING_LANG", alt 

2225 ) 

2226 tags = list(tags) # Make sure we don't modify anything cached 

2227 tags.append("canonical") 

2228 if alt_i == 0 and "," in wxr.wtp.title or ";" in wxr.wtp.title: # type:ignore[operator] 

2229 # Kludge to handle article titles/phrases with commas. 

2230 # basepart's regex strips commas, which leads to a 

2231 # canonical form that is the title phrase without a comma. 

2232 # basepart in add_related is almost immediately joined with 

2233 # spaces anyhow. XXX not exactly sure why it's 

2234 # canonicals.append((tags, baseparts)) and not (tags, [alt]) 

2235 baseparts = [alt] 

2236 canonicals.append((tags, baseparts)) 

2237 

2238 # If more of this kind of replace-and-return-original kind of stuff is 

2239 # needed, make semicolon_present into a flag enum, something like `modified` 

2240 if semicolon_present: 

2241 new_cans = [] 

2242 for tags, baseparts in canonicals: 

2243 new_cans.append( 

2244 ( 

2245 tags, 

2246 [s.replace(SEMICOLON_REPLACEMENT, ";") for s in baseparts], 

2247 ) 

2248 ) 

2249 canonicals = new_cans 

2250 for tags, baseparts in canonicals: 

2251 add_related( 

2252 wxr, 

2253 data, 

2254 tags, 

2255 baseparts, 

2256 text, 

2257 len(canonicals) > 1, 

2258 is_reconstruction, 

2259 head_group, 

2260 ruby, 

2261 links, 

2262 link_dict, 

2263 ) 

2264 

2265 # Handle parenthesized descriptors for the word form and links to 

2266 # related words 

2267 text = quote_kept_parens(text) 

2268 parens = list( 

2269 m.group(2) 

2270 for m in re.finditer(r"(^|\s)\((([^()]|\([^()]*\))*)\)", text) 

2271 ) 

2272 parens.extend( 

2273 m.group(1) 

2274 for m in re.finditer(r"[^\s]\((([^()]|\([^()]*\))*)\)($|\s)", text) 

2275 ) 

2276 have_romanization = False 

2277 have_ruby = False 

2278 hiragana = "" 

2279 katakana = "" 

2280 for paren in parens: 

2281 paren = paren.strip() 

2282 if not paren: 2282 ↛ 2283line 2282 didn't jump to line 2283 because the condition on line 2282 was never true

2283 continue 

2284 can_be_form = True 

2285 if label_templates is not None and paren.startswith( 

2286 "__LABEL_TEMPLATE_" 

2287 ): 

2288 # wxr.wtp.warning("Found label template in head") 

2289 # continue 

2290 can_be_form = False 

2291 m = re.match(r"__LABEL_TEMPLATE_(\d+)__", paren) 

2292 if m is None: 2292 ↛ 2293line 2292 didn't jump to line 2293 because the condition on line 2292 was never true

2293 wxr.wtp.warning( 

2294 f"Label template list magic phrase is broken: `{paren}`", 

2295 sortid="20260508/label list index broken", 

2296 ) 

2297 continue 

2298 ht = label_templates[int(m.group(1))] 

2299 desc = ht.get("expansion", "").strip() 

2300 if desc: 2300 ↛ 2303line 2300 didn't jump to line 2303 because the condition on line 2300 was always true

2301 paren = desc 

2302 else: 

2303 wxr.wtp.warning( 

2304 f"Label template seems to have no text contents: {ht=}", 

2305 sortid="20260508/label_templates", 

2306 ) 

2307 continue 

2308 if paren.startswith("see "): 

2309 continue 

2310 if paren.startswith("U+"): 2310 ↛ 2311line 2310 didn't jump to line 2311 because the condition on line 2310 was never true

2311 continue 

2312 # In some rare cases, strip word that inflects form the form 

2313 # description, e.g. "look through rose-tinted glasses"/English. 

2314 # `([looks])` 

2315 paren = re.sub(r"\s*\(\[[^])]*\]\)", "", paren) 

2316 

2317 # If it starts with hiragana or katakana, treat as such form. Note 

2318 # that each hiragana/katakana character is in separate parentheses, 

2319 # so we must concatenate them. 

2320 try: 

2321 un = unicodedata.name(paren[0]).split()[0] 

2322 except ValueError: 

2323 un = "INVALID" 

2324 if un == "KATAKANA": 2324 ↛ 2325line 2324 didn't jump to line 2325 because the condition on line 2324 was never true

2325 katakana += paren 

2326 have_ruby = True 

2327 continue 

2328 if un == "HIRAGANA": 2328 ↛ 2329line 2328 didn't jump to line 2329 because the condition on line 2328 was never true

2329 hiragana += paren 

2330 have_ruby = True 

2331 continue 

2332 

2333 # Parse format ", 16 (Japan, Mainland), 17 (Hong Kong, Taiwan) strokes," 

2334 # in the middle of the parenthesized expression, e.g. 薄 

2335 def strokes_repl(m: re.Match) -> str: 

2336 strokes1, tags1, strokes2, tags2 = m.groups() 

2337 for strokes, tags in [[strokes1, tags1], [strokes2, tags2]]: 

2338 tags = tags.split(", ") 

2339 tags = list( 

2340 "Mainland China" if t == "Mainland" else t for t in tags 

2341 ) 

2342 tags.append("strokes") 

2343 add_related( 

2344 wxr, 

2345 data, 

2346 tags, 

2347 [strokes], 

2348 text, 

2349 True, 

2350 is_reconstruction, 

2351 head_group, 

2352 ruby, 

2353 links, 

2354 link_dict, 

2355 ) 

2356 return ", " 

2357 

2358 if can_be_form is True: 

2359 paren = re.sub( 

2360 r", (\d+) \(([^()]+)\), (\d+) \(([^()]+)\) strokes, ", 

2361 strokes_repl, 

2362 paren, 

2363 ) 

2364 

2365 descriptors = map_with(xlat_descs_map, [paren]) 

2366 new_desc = [] 

2367 for desc in descriptors: 

2368 new_desc.extend( 

2369 map_with( 

2370 xlat_tags_map, 

2371 split_at_comma_semi( 

2372 desc, extra=[", or "], skipped=link_words_not_alnum 

2373 ), 

2374 ) 

2375 ) 

2376 prev_tags: Union[list[list[str]], list[tuple[str, ...]], None] = None 

2377 following_tags = None # Added to prev_tags from previous parenthesized 

2378 # part, e.g. walrus/English 

2379 # "(both nonstandard, proscribed, uncommon)" 

2380 for desc_i, desc in enumerate(new_desc): 

2381 # print("HEAD DESC: {!r}".format(desc)) 

2382 

2383 # Abort on certain descriptors (assume remaining values are 

2384 # examples or uninteresting, cf. gaan/Navajo, horior/Latin) 

2385 if re.match(r"^(per |e\.g\.$)", desc): 2385 ↛ 2386line 2385 didn't jump to line 2386 because the condition on line 2385 was never true

2386 break 

2387 

2388 # If it all consists of CJK characters, add it with the 

2389 # CJK tag. This is used at least for some Vietnamese 

2390 # words (e.g., ba/Vietnamese) 

2391 try: 

2392 if ( 2392 ↛ 2396line 2392 didn't jump to line 2396 because the condition on line 2392 was never true

2393 all(unicodedata.name(x).startswith("CJK ") for x in desc) 

2394 and can_be_form 

2395 ): 

2396 add_related( 

2397 wxr, 

2398 data, 

2399 ["CJK"], 

2400 [desc], 

2401 text, 

2402 True, 

2403 is_reconstruction, 

2404 head_group, 

2405 ruby, 

2406 links, 

2407 link_dict, 

2408 ) 

2409 continue 

2410 except ValueError: 

2411 pass 

2412 

2413 # Handle some special cases 

2414 splitdesc = desc.split() 

2415 if ( 2415 ↛ 2425line 2415 didn't jump to line 2425 because the condition on line 2415 was never true

2416 len(splitdesc) >= 3 

2417 and splitdesc[1] == "superlative" 

2418 and classify_desc(splitdesc[0]) != "tags" 

2419 and prev_tags 

2420 and can_be_form 

2421 ): 

2422 # Handle the special case of second comparative after comma, 

2423 # followed by superlative without comma. E.g. 

2424 # mal/Portuguese/Adv 

2425 for ts in prev_tags: 

2426 add_related( 

2427 wxr, 

2428 data, 

2429 ts, 

2430 [splitdesc[0]], 

2431 text, 

2432 True, 

2433 is_reconstruction, 

2434 head_group, 

2435 ruby, 

2436 links, 

2437 link_dict, 

2438 ) 

2439 desc = " ".join(splitdesc[1:]) 

2440 elif ( 2440 ↛ 2449line 2440 didn't jump to line 2449 because the condition on line 2440 was never true

2441 len(splitdesc) == 2 

2442 and splitdesc[0] in ("also", "and") 

2443 and prev_tags 

2444 and classify_desc(splitdesc[1]) != "tags" 

2445 and can_be_form 

2446 ): 

2447 # Sometimes alternative forms are prefixed with "also" or 

2448 # "and" 

2449 for ts in prev_tags: 

2450 add_related( 

2451 wxr, 

2452 data, 

2453 ts, 

2454 [splitdesc[1]], 

2455 text, 

2456 True, 

2457 is_reconstruction, 

2458 head_group, 

2459 ruby, 

2460 links, 

2461 link_dict, 

2462 ) 

2463 continue 

2464 elif len(splitdesc) >= 2 and splitdesc[0] in ("including",): 2464 ↛ 2465line 2464 didn't jump to line 2465 because the condition on line 2464 was never true

2465 continue 

2466 

2467 # If only one word, assume it is comma-separated alternative 

2468 # to the previous one 

2469 if len(splitdesc) == 1: 

2470 cls = classify_desc(desc) 

2471 if cls != "tags" and can_be_form: 

2472 if prev_tags: 2472 ↛ 2474line 2472 didn't jump to line 2474 because the condition on line 2472 was never true

2473 # Assume comma-separated alternative to previous one 

2474 for ts in prev_tags: 

2475 add_related( 

2476 wxr, 

2477 data, 

2478 ts, 

2479 [desc], 

2480 text, 

2481 True, 

2482 is_reconstruction, 

2483 head_group, 

2484 ruby, 

2485 links, 

2486 link_dict, 

2487 ) 

2488 continue 

2489 elif distw(titleparts, desc) <= 0.5: 2489 ↛ 2492line 2489 didn't jump to line 2492 because the condition on line 2489 was never true

2490 # Similar to head word, assume a dialectal variation to 

2491 # the base form. Cf. go/Alemannic German/Verb 

2492 add_related( 

2493 wxr, 

2494 data, 

2495 ["alternative"], 

2496 [desc], 

2497 text, 

2498 True, 

2499 is_reconstruction, 

2500 head_group, 

2501 ruby, 

2502 links, 

2503 link_dict, 

2504 ) 

2505 continue 

2506 elif ( 

2507 cls in ("romanization", "english") 

2508 and not have_romanization 

2509 and classify_desc(titleword) == "other" 

2510 and not ( 

2511 "categories" in data and desc in data["categories"] 

2512 ) 

2513 ): 

2514 # Assume it to be a romanization 

2515 add_romanization( 

2516 wxr, 

2517 data, 

2518 desc, 

2519 text, 

2520 is_reconstruction, 

2521 head_group, 

2522 ruby, 

2523 ) 

2524 have_romanization = True 

2525 continue 

2526 

2527 m = re.match(r"^(\d+) strokes?$", desc) 

2528 if m and can_be_form: 

2529 # Special case, used to give #strokes for Han characters 

2530 add_related( 

2531 wxr, 

2532 data, 

2533 ["strokes"], 

2534 [m.group(1)], 

2535 text, 

2536 True, 

2537 is_reconstruction, 

2538 head_group, 

2539 ruby, 

2540 links, 

2541 link_dict, 

2542 ) 

2543 continue 

2544 

2545 # See if it is radical+strokes 

2546 m = re.match( 

2547 r"^([\u2F00-\u2FDF\u2E80-\u2EFF\U00018800-\U00018AFF" 

2548 r"\uA490-\uA4CF\u4E00-\u9FFF]\+\d+)" 

2549 r"( in (Japanese|Chinese|traditional Chinese|" 

2550 r"simplified Chinese))?$", 

2551 desc, 

2552 ) 

2553 if m and can_be_form: 2553 ↛ 2556line 2553 didn't jump to line 2556 because the condition on line 2553 was never true

2554 # Special case, used to give radical + strokes for Han 

2555 # characters 

2556 radical_strokes = m.group(1) 

2557 lang = m.group(3) 

2558 t = ["radical+strokes"] 

2559 if lang: 

2560 t.extend(lang.split()) 

2561 add_related( 

2562 wxr, 

2563 data, 

2564 t, 

2565 [radical_strokes], 

2566 text, 

2567 True, 

2568 is_reconstruction, 

2569 head_group, 

2570 ruby, 

2571 links, 

2572 link_dict, 

2573 ) 

2574 prev_tags = None 

2575 following_tags = None 

2576 continue 

2577 

2578 # See if it indicates historical Katakana ortography (←) or 

2579 # just otherwise katakana/hiragana form 

2580 m = re.match(r"←\s*|kana\s+", desc) 

2581 if m: 2581 ↛ 2582line 2581 didn't jump to line 2582 because the condition on line 2581 was never true

2582 if desc.startswith("←"): 

2583 t1 = "historical " 

2584 else: 

2585 t1 = "" 

2586 x = desc[m.end() :] 

2587 if x.endswith("?"): 

2588 x = x[:-1] 

2589 # XXX should we add a tag indicating uncertainty? 

2590 if x: 

2591 name = unicodedata.name(x[0]) 

2592 if name.startswith("HIRAGANA "): 

2593 desc = t1 + "hiragana " + x 

2594 elif name.startswith("KATAKANA "): 

2595 desc = t1 + "katakana " + x 

2596 

2597 # See if it is "n strokes in Chinese" or similar 

2598 m = re.match( 

2599 r"(\d+) strokes in (Chinese|Japanese|" 

2600 r"traditional Chinese|simplified Chinese)$", 

2601 desc, 

2602 ) 

2603 if m and can_be_form: 2603 ↛ 2605line 2603 didn't jump to line 2605 because the condition on line 2603 was never true

2604 # Special case, used to give just strokes for some Han chars 

2605 strokes = m.group(1) 

2606 lang = m.group(2) 

2607 t = ["strokes"] 

2608 t.extend(lang.split()) 

2609 add_related( 

2610 wxr, 

2611 data, 

2612 t, 

2613 [strokes], 

2614 text, 

2615 True, 

2616 is_reconstruction, 

2617 head_group, 

2618 ruby, 

2619 links, 

2620 link_dict, 

2621 ) 

2622 prev_tags = None 

2623 following_tags = None 

2624 continue 

2625 

2626 # American Sign Language has images (or requests for image) 

2627 # as heads, + this ASL gloss after. 

2628 m2 = re.search(r"\(ASL gloss:\s+(.*)\)", text) 

2629 if m2 and can_be_form: 2629 ↛ 2630line 2629 didn't jump to line 2630 because the condition on line 2629 was never true

2630 add_related( 

2631 wxr, 

2632 data, 

2633 ["ASL-gloss"], 

2634 [m2.group(1)], 

2635 text, 

2636 True, 

2637 is_reconstruction, 

2638 head_group, 

2639 ruby, 

2640 links, 

2641 link_dict, 

2642 ) 

2643 continue 

2644 

2645 parts = list(m.group(0) for m in re.finditer(word_re, desc)) 

2646 if not parts: 2646 ↛ 2647line 2646 didn't jump to line 2647 because the condition on line 2646 was never true

2647 prev_tags = None 

2648 following_tags = None 

2649 continue 

2650 

2651 # Check for certain language-specific header part starts that 

2652 # modify 

2653 if ( 2653 ↛ 2658line 2653 didn't jump to line 2658 because the condition on line 2653 was never true

2654 len(parts) == 2 

2655 and language in lang_specific_head_map 

2656 and can_be_form 

2657 ): 

2658 ht2 = lang_specific_head_map[language] 

2659 if parts[0] in ht2: 

2660 rem_tags, add_tags = ht2[parts[0]] 

2661 new_prev_tags1: list[list[str]] = [] 

2662 tags2: Union[tuple[str, ...], list[str]] 

2663 for tags2 in prev_tags or [()]: 

2664 if rem_tags is True: # Remove all old tags 

2665 tsets = set() 

2666 else: 

2667 tsets = set(tags2) - set(rem_tags.split()) 

2668 tsets = tsets | set(add_tags.split()) 

2669 tags = list(sorted(tsets)) 

2670 add_related( 

2671 wxr, 

2672 data, 

2673 tags, 

2674 [parts[1]], 

2675 text, 

2676 True, 

2677 is_reconstruction, 

2678 head_group, 

2679 ruby, 

2680 links, 

2681 link_dict, 

2682 ) 

2683 new_prev_tags1.append(tags) 

2684 prev_tags = new_prev_tags1 

2685 following_tags = None 

2686 continue 

2687 

2688 # Handle the special case of descriptors that are parenthesized, 

2689 # e.g., (archaic or Scotland) 

2690 m = re.match(r"\(([^)]+)\)\s+(.*)$", desc) 

2691 if m is not None and classify_desc(m.group(1)) == "tags": 2691 ↛ 2692line 2691 didn't jump to line 2692 because the condition on line 2691 was never true

2692 tagpart = m.group(1) 

2693 related = [m.group(2)] 

2694 tagsets, topics = decode_tags(tagpart, no_unknown_starts=True) 

2695 if topics: 

2696 wxr.wtp.debug( 

2697 "parenthized head part {!r} contains topics: {}".format( 

2698 tagpart, topics 

2699 ), 

2700 sortid="form_descriptions/1647", 

2701 ) 

2702 elif m is not None and re.match(r"in the sense ", m.group(1)): 2702 ↛ 2705line 2702 didn't jump to line 2705 because the condition on line 2702 was never true

2703 # Handle certain ignored cases 

2704 # e.g. bord/Danish: in the sense "plank" 

2705 related = [m.group(2)] 

2706 tagsets = [()] 

2707 else: 

2708 # Normal parsing of the descriptor 

2709 alt_related = None 

2710 alt_tagsets = None 

2711 tagsets = None 

2712 for i in range(len(parts), 0, -1): 

2713 related = parts[i:] 

2714 tagparts = parts[:i] 

2715 # print(" i={} related={} tagparts={}" 

2716 # .format(i, related, tagparts)) 

2717 tagsets, topics = decode_tags( 

2718 " ".join(tagparts), no_unknown_starts=True 

2719 ) 

2720 # print("tagparts={!r} tagsets={} topics={} related={} " 

2721 # "alt_related={} distw={:.2f}" 

2722 # .format(tagparts, tagsets, topics, related, 

2723 # alt_related, 

2724 # distw(titleparts, parts[i - 1]))) 

2725 if ( 

2726 topics 

2727 or not tagsets 

2728 or any("error-unknown-tag" in x for x in tagsets) 

2729 ): 

2730 if alt_related is not None: 2730 ↛ 2732line 2730 didn't jump to line 2732 because the condition on line 2730 was never true

2731 # We already had a good division, so let's stop. 

2732 break 

2733 # Bad division, try deeper 

2734 continue 

2735 # print(f"{parts[i-1]=}, {parts=}") 

2736 if ( 

2737 i > 1 

2738 and len(parts[i - 1]) >= 4 

2739 and ( 

2740 distw(titleparts, parts[i - 1]) <= 0.4 

2741 or ( 

2742 wxr.wtp.section == "English" 

2743 and wxr.wtp.title 

2744 in WORDS_WITH_FALSE_POSITIVE_TAGS 

2745 and parts[i - 1] 

2746 in WORDS_WITH_FALSE_POSITIVE_TAGS[wxr.wtp.title] 

2747 ) 

2748 ) 

2749 # Fixes 'unaccountability' wiktext #1196 

2750 and not ( 

2751 wxr.wtp.section == "English" 

2752 and wxr.wtp.title in WORDS_WITH_FALSE_POSITIVE_FORMS 

2753 and parts[i - 1] 

2754 in WORDS_WITH_FALSE_POSITIVE_FORMS[wxr.wtp.title] 

2755 ) 

2756 # Fixes wiktextract #983, where "participle" 

2757 # was too close to "Martinize" and so this accepted 

2758 # ["participle", "Martinize"] as matching; this 

2759 # kludge prevents this from happening if titleparts 

2760 # is shorter than what would be 'related'. 

2761 # This breaks if we want to detect stuff that 

2762 # actually gets an extra space-separated word when 

2763 # 'inflected'. 

2764 and ( 

2765 len(titleparts) >= len(parts[i - 1 :]) 

2766 or "or" in parts[i - 1 :] 

2767 ) 

2768 ): 

2769 # print(f"Reached; {parts=}, {parts[i-1]=}") 

2770 alt_related = related 

2771 alt_tagsets = tagsets 

2772 continue 

2773 alt_related = None 

2774 alt_tagsets = None 

2775 break 

2776 # for-else 

2777 else: 

2778 if alt_related is None and can_be_form: 

2779 # Check if the parenthesized part is likely a 

2780 # romanization 

2781 if ( 2781 ↛ 2789line 2781 didn't jump to line 2789 because the condition on line 2781 was never true

2782 (have_ruby or classify_desc(base) == "other") 

2783 and classify_desc(paren) == "romanization" 

2784 and not ( 

2785 "categories" in data 

2786 and desc in data["categories"] 

2787 ) 

2788 ): 

2789 for r in split_at_comma_semi( 

2790 paren, 

2791 extra=[" or "], 

2792 skipped=link_words_not_alnum, 

2793 ): 

2794 add_romanization( 

2795 wxr, 

2796 data, 

2797 r, 

2798 text, 

2799 is_reconstruction, 

2800 head_group, 

2801 ruby, 

2802 ) 

2803 have_romanization = True 

2804 continue 

2805 tagsets = [("error-unrecognized-head-form",)] 

2806 wxr.wtp.debug( 

2807 "unrecognized head form: {}".format(desc), 

2808 sortid="form_descriptions/1698", 

2809 ) 

2810 continue 

2811 

2812 if alt_related is not None: 2812 ↛ 2813line 2812 didn't jump to line 2813 because the condition on line 2812 was never true

2813 related = alt_related 

2814 tagsets = alt_tagsets 

2815 

2816 # print("FORM END: tagsets={} related={}".format(tagsets, related)) 

2817 # print("==================") 

2818 

2819 if ( 2819 ↛ 2840line 2819 didn't jump to line 2840 because the condition on line 2819 was never true

2820 len(related) <= 0 

2821 and wxr.wtp.section == "English" 

2822 and tagsets is not None 

2823 and len(tagsets) > 0 

2824 and not any( 

2825 s.startswith("error-") for tagset in tagsets for s in tagset 

2826 ) 

2827 and any( 

2828 s in FORM_ASSOCIATED_TAG_WORDS 

2829 for tagset in tagsets 

2830 for s in tagset 

2831 ) 

2832 and ( 

2833 wxr.wtp.title not in FALSE_POSITIVE_MISSING_FORMS 

2834 and not any( 

2835 rel in FALSE_POSITIVE_MISSING_FORMS[wxr.wtp.title or ""] 

2836 for rel in related 

2837 ) 

2838 ) 

2839 ): 

2840 wxr.wtp.debug( 

2841 f"Form tags without form: {desc=}, {tagsets=}", 

2842 sortid="form_description/20250107", 

2843 ) 

2844 if not tagsets: 2844 ↛ 2845line 2844 didn't jump to line 2845 because the condition on line 2844 was never true

2845 continue 

2846 

2847 # print(f"{alts=}, {related=}") 

2848 

2849 assert isinstance(related, (list, tuple)) 

2850 related_str = " ".join(related) 

2851 if "or" in titleparts: 

2852 alts = [related_str] 

2853 else: 

2854 alts = split_at_comma_semi( 

2855 related_str, 

2856 separators=[r"\bor\b"], 

2857 skipped=link_words_not_alnum, 

2858 ) 

2859 # print(f"{related_str=}, {alts=}") 

2860 if not alts: 

2861 alts = [""] 

2862 for related_str in alts: 

2863 if related_str and can_be_form: 

2864 if prev_tags and ( 

2865 all( 

2866 all( 

2867 t in ["nonstandard", "dialectal"] 

2868 or valid_tags[t] == "dialect" 

2869 for t in tags 

2870 ) 

2871 for ts in tagsets 

2872 ) 

2873 or ( 

2874 any("participle" in ts for ts in prev_tags) 

2875 and all( 

2876 "attributive" in ts 

2877 or any(valid_tags[t] == "gender" for t in ts) 

2878 for ts in tagsets 

2879 ) 

2880 ) 

2881 ): 

2882 # Merged with previous tags. Don't update previous 

2883 # tags here; cf. burn/English/Verb 

2884 for tags_l in tagsets: 

2885 for ts in prev_tags: 

2886 tags_l1 = sorted(set(tags_l) | set(ts)) 

2887 add_related( 

2888 wxr, 

2889 data, 

2890 tags_l1, 

2891 [related_str], 

2892 text, 

2893 True, 

2894 is_reconstruction, 

2895 head_group, 

2896 ruby, 

2897 links, 

2898 link_dict, 

2899 ) 

2900 else: 

2901 # Not merged with previous tags 

2902 for tags_l in tagsets: 

2903 if following_tags is not None: 2903 ↛ 2904line 2903 didn't jump to line 2904 because the condition on line 2903 was never true

2904 for ts in following_tags: 

2905 tags_l1 = list( 

2906 sorted(set(tags_l) | set(ts)) 

2907 ) 

2908 add_related( 

2909 wxr, 

2910 data, 

2911 tags_l1, 

2912 [related_str], 

2913 text, 

2914 True, 

2915 is_reconstruction, 

2916 head_group, 

2917 ruby, 

2918 links, 

2919 link_dict, 

2920 ) 

2921 else: 

2922 ret = add_related( 

2923 wxr, 

2924 data, 

2925 tags_l, 

2926 [related_str], 

2927 text, 

2928 True, 

2929 is_reconstruction, 

2930 head_group, 

2931 ruby, 

2932 links, 

2933 link_dict, 

2934 ) 

2935 if ret is not None: 2935 ↛ 2936line 2935 didn't jump to line 2936 because the condition on line 2935 was never true

2936 following_tags = ret 

2937 prev_tags = tagsets 

2938 else: 

2939 if desc_i < len(new_desc) - 1 and all( 2939 ↛ 2946line 2939 didn't jump to line 2946 because the condition on line 2939 was never true

2940 "participle" in ts or "infinitive" in ts 

2941 for ts in tagsets 

2942 ): 

2943 # Interpret it as a standalone form description 

2944 # in the middle, probably followed by forms or 

2945 # language-specific descriptors. cf. drikke/Danish 

2946 new_prev_tags2 = [] 

2947 for ts1 in prev_tags or [()]: 

2948 for ts2 in tagsets: 

2949 ts = tuple(sorted(set(ts1) | set(ts2))) 

2950 new_prev_tags2.append(ts) 

2951 prev_tags = new_prev_tags2 

2952 continue 

2953 for tags in tagsets: 

2954 data_extend(data, "tags", tags) 

2955 prev_tags = tagsets 

2956 following_tags = None 

2957 

2958 # Finally, if we collected hirakana/katakana, add them now 

2959 if hiragana: 2959 ↛ 2960line 2959 didn't jump to line 2960 because the condition on line 2959 was never true

2960 add_related( 

2961 wxr, 

2962 data, 

2963 ["hiragana"], 

2964 [hiragana], 

2965 text, 

2966 True, 

2967 is_reconstruction, 

2968 head_group, 

2969 ruby, 

2970 ) 

2971 if katakana: 2971 ↛ 2972line 2971 didn't jump to line 2972 because the condition on line 2971 was never true

2972 add_related( 

2973 wxr, 

2974 data, 

2975 ["katakana"], 

2976 [katakana], 

2977 text, 

2978 True, 

2979 is_reconstruction, 

2980 head_group, 

2981 ruby, 

2982 ) 

2983 

2984 # XXX check if this is actually relevant, tags in word root data 

2985 # is extremely rare (not sure where they slip through). 

2986 tags = data.get("tags", []) # type:ignore 

2987 if len(tags) > 0: 

2988 # wxr.wtp.debug( 

2989 # f"Tags appear in word root data: {data['tags']=}", # type:ignore 

2990 # sortid="form_descriptions/2620/20240606", 

2991 # ) # Messes up tests. 

2992 data["tags"] = sorted(set(tags)) # type:ignore 

2993 

2994 

2995def parse_sense_qualifier( 

2996 wxr: WiktextractContext, text: str, data: Union[SenseData, LinkageData] 

2997) -> None: 

2998 """Parses tags or topics for a sense or some other data. The values are 

2999 added into the dictionary ``data``.""" 

3000 assert isinstance(wxr, WiktextractContext) 

3001 assert isinstance(text, str) 

3002 assert isinstance(data, dict) 

3003 # print("parse_sense_qualifier:", text) 

3004 if re.match(r"\([^()]+\)$", text): 3004 ↛ 3005line 3004 didn't jump to line 3005 because the condition on line 3004 was never true

3005 text = text[1:-1] 

3006 if re.match(r'"[^"]+"$', text): 3006 ↛ 3007line 3006 didn't jump to line 3007 because the condition on line 3006 was never true

3007 text = text[1:-1] 

3008 lst = map_with(xlat_descs_map, [text]) 

3009 sense_tags: list[str] = [] 

3010 for text in lst: 

3011 for semi in split_at_comma_semi(text): 

3012 if not semi: 3012 ↛ 3013line 3012 didn't jump to line 3013 because the condition on line 3012 was never true

3013 continue 

3014 orig_semi = semi 

3015 idx = semi.find(":") 

3016 if idx >= 0: 3016 ↛ 3017line 3016 didn't jump to line 3017 because the condition on line 3016 was never true

3017 semi = semi[:idx] 

3018 cls = classify_desc(semi, allow_unknown_tags=True) 

3019 # print("parse_sense_qualifier: classify_desc: {} -> {}" 

3020 # .format(semi, cls)) 

3021 if cls == "tags": 

3022 tagsets, topics = decode_tags(semi) 

3023 data_extend(data, "topics", topics) 

3024 # XXX should think how to handle distinct options better, 

3025 # e.g., "singular and plural genitive"; that can't really be 

3026 # done with changing the calling convention of this function. 

3027 # Should split sense if more than one category of tags differs. 

3028 for tags in tagsets: 

3029 sense_tags.extend(tags) 

3030 elif cls == "taxonomic": 3030 ↛ 3031line 3030 didn't jump to line 3031 because the condition on line 3030 was never true

3031 if re.match(r"×[A-Z]", semi): 

3032 sense_tags.append("extinct") 

3033 semi = semi[1:] 

3034 data["taxonomic"] = semi 

3035 elif cls == "english": 

3036 if "qualifier" in data and data["qualifier"] != orig_semi: 3036 ↛ 3037line 3036 didn't jump to line 3037 because the condition on line 3036 was never true

3037 data["qualifier"] += "; " + orig_semi 

3038 else: 

3039 data["qualifier"] = orig_semi 

3040 else: 

3041 wxr.wtp.debug( 

3042 "unrecognized sense qualifier: {}".format(text), 

3043 sortid="form_descriptions/1831", 

3044 ) 

3045 sense_tags = sorted(set(sense_tags)) 

3046 data_extend(data, "tags", sense_tags) 

3047 

3048 

3049def parse_pronunciation_tags( 

3050 wxr: WiktextractContext, text: str, data: SoundData 

3051) -> None: 

3052 assert isinstance(wxr, WiktextractContext) 

3053 assert isinstance(text, str) 

3054 assert isinstance(data, dict) 

3055 text = text.strip() 

3056 if not text: 

3057 return 

3058 cls = classify_desc(text) 

3059 found_tags = False 

3060 got_notes_text = False 

3061 if cls == "tags": 

3062 tagsets, topics = decode_tags(text) 

3063 data_extend(data, "topics", topics) 

3064 for tagset in tagsets: 

3065 for t in tagset: 

3066 if " " in t: 

3067 got_notes_text = True 

3068 else: 

3069 found_tags = True 

3070 data_append(data, "tags", t) 

3071 

3072 if not found_tags: 

3073 # Try again to find tags, but keep notes text as above for context; 

3074 # tags will be in the note text itself, but if we remove them, it'll 

3075 # just be confusing. 

3076 got_notes_text = False 

3077 for s in split_at_comma_semi(text): 3077 ↛ 3091line 3077 didn't jump to line 3091 because the loop on line 3077 didn't complete

3078 if not (s := s.strip()): 3078 ↛ 3079line 3078 didn't jump to line 3079 because the condition on line 3078 was never true

3079 continue 

3080 if classify_desc(s) != "tags": 

3081 got_notes_text = True 

3082 break 

3083 tagsets, topics = decode_tags(s) 

3084 data_extend(data, "topics", topics) 

3085 for tagset in tagsets: 

3086 for t in tagset: 

3087 if " " not in t: 3087 ↛ 3090line 3087 didn't jump to line 3090 because the condition on line 3087 was always true

3088 data_append(data, "tags", t) 

3089 else: 

3090 got_notes_text = True 

3091 if got_notes_text: 3091 ↛ 3096line 3091 didn't jump to line 3096 because the condition on line 3091 was always true

3092 # If we're mixing and matching tags and notes text, keep whole note 

3093 # text 

3094 data["note"] = text 

3095 

3096 if got_notes_text is True: 

3097 data["note"] = text 

3098 

3099 

3100def parse_translation_desc( 

3101 wxr: WiktextractContext, lang: str, text: str, tr: TranslationData 

3102) -> None: 

3103 assert isinstance(wxr, WiktextractContext) 

3104 assert isinstance(lang, str) # The language of ``text`` 

3105 assert isinstance(text, str) 

3106 assert isinstance(tr, dict) 

3107 # print("parse_translation_desc:", text) 

3108 

3109 # Process all parenthesized parts from the translation item 

3110 note = None 

3111 restore_beginning = "" 

3112 restore_end = "" 

3113 while True: 

3114 beginning = False 

3115 # See if we can find a parenthesized expression at the end 

3116 m = re.search(r"\s*\((([^()]|\([^()]+\))+)\)\.?$", text) 

3117 if m: 

3118 par = m.group(1) 

3119 text = text[: m.start()] 

3120 if par.startswith(("literally ", "lit.")): 

3121 continue # Not useful for disambiguation in many idioms 

3122 else: 

3123 # See if we can find a parenthesized expression at the start 

3124 m = re.match(r"^\^?\((([^()]|\([^()]+\))+)\):?(\s+|$)", text) 

3125 if m: 

3126 par = m.group(1) 

3127 text = text[m.end() :] 

3128 beginning = True 

3129 if re.match(r"^(\d|\s|,| or | and )+$", par): 3129 ↛ 3134line 3129 didn't jump to line 3134 because the condition on line 3129 was never true

3130 # Looks like this beginning parenthesized expression only 

3131 # contains digits or their combinations. We assume such 

3132 # to be sense descriptions if no sense has been selected, 

3133 # or otherwise just ignore them. 

3134 if not tr.get("sense"): 

3135 tr["sense"] = par 

3136 continue 

3137 else: 

3138 # See if we can find a parenthesized expression in the middle. 

3139 # Romanizations are sometimes between word and gender marker, 

3140 # e.g. wife/English/Tr/Yiddish. 

3141 m = re.search(r"\s+\((([^()]|\([^()]+\))+)\)", text) 

3142 if m: 

3143 par = m.group(1) 

3144 text = text[: m.start()] + text[m.end() :] 

3145 else: 

3146 # No more parenthesized expressions - break out of the loop 

3147 break 

3148 

3149 # Some cleanup of artifacts that may result from skipping some templates 

3150 # in earlier stages 

3151 if par.startswith(": "): 3151 ↛ 3152line 3151 didn't jump to line 3152 because the condition on line 3151 was never true

3152 par = par[2:] 

3153 if par.endswith(","): 3153 ↛ 3154line 3153 didn't jump to line 3154 because the condition on line 3153 was never true

3154 par = par[:-1] 

3155 if re.match(r'^[“"]([^“”"]*)[“”"]$', par): 3155 ↛ 3156line 3155 didn't jump to line 3156 because the condition on line 3155 was never true

3156 par = par[1:-1] 

3157 par = par.strip() 

3158 

3159 # Check for special script pronunciation followed by romanization, 

3160 # used in many Asian languages. 

3161 lst = par.split(", ") 

3162 if len(lst) == 2: 

3163 a, r = lst 

3164 if classify_desc(a) == "other": 

3165 cls = classify_desc(r) 

3166 # print("parse_translation_desc: r={} cls={}".format(r, cls)) 

3167 if cls == "romanization" or ( 

3168 cls == "english" and len(r.split()) == 1 and r[0].islower() 

3169 ): 

3170 if tr.get("alt") and tr.get("alt") != a: 3170 ↛ 3171line 3170 didn't jump to line 3171 because the condition on line 3170 was never true

3171 wxr.wtp.debug( 

3172 'more than one value in "alt": {} vs. {}'.format( 

3173 tr["alt"], a 

3174 ), 

3175 sortid="form_descriptions/1930", 

3176 ) 

3177 tr["alt"] = a 

3178 if tr.get("roman") and tr.get("roman") != r: 3178 ↛ 3179line 3178 didn't jump to line 3179 because the condition on line 3178 was never true

3179 wxr.wtp.debug( 

3180 'more than one value in "roman": {} vs. {}'.format( 

3181 tr["roman"], r 

3182 ), 

3183 sortid="form_descriptions/1936", 

3184 ) 

3185 tr["roman"] = r 

3186 continue 

3187 

3188 # Check for certain comma-separated tags combined with English text 

3189 # at the beginning or end of a comma-separated parenthesized list 

3190 while len(lst) > 1: 

3191 cls = classify_desc(lst[0]) 

3192 if cls == "tags": 3192 ↛ 3193line 3192 didn't jump to line 3193 because the condition on line 3192 was never true

3193 tagsets, topics = decode_tags(lst[0]) 

3194 for t in tagsets: 

3195 data_extend(tr, "tags", t) 

3196 data_extend(tr, "topics", topics) 

3197 lst = lst[1:] 

3198 continue 

3199 cls = classify_desc(lst[-1]) 

3200 if cls == "tags": 

3201 tagsets, topics = decode_tags(lst[-1]) 

3202 for t in tagsets: 

3203 data_extend(tr, "tags", t) 

3204 data_extend(tr, "topics", topics) 

3205 lst = lst[:-1] 

3206 continue 

3207 break 

3208 par = ", ".join(lst) 

3209 

3210 if not par: 3210 ↛ 3211line 3210 didn't jump to line 3211 because the condition on line 3210 was never true

3211 continue 

3212 if re.search(tr_ignored_parens_re, par): 3212 ↛ 3213line 3212 didn't jump to line 3213 because the condition on line 3212 was never true

3213 continue 

3214 if par.startswith("numeral:"): 

3215 par = par[8:].strip() 

3216 

3217 # Classify the part in parenthesis and process accordingly 

3218 cls = classify_desc(par) 

3219 # print("parse_translation_desc classify: {!r} -> {}" 

3220 # .format(par, cls)) 

3221 if par == text: 

3222 pass 

3223 if par == "f": 3223 ↛ 3224line 3223 didn't jump to line 3224 because the condition on line 3223 was never true

3224 data_append(tr, "tags", "feminine") 

3225 elif par == "m": 3225 ↛ 3226line 3225 didn't jump to line 3226 because the condition on line 3225 was never true

3226 data_append(tr, "tags", "masculine") 

3227 elif cls == "tags": 

3228 tagsets, topics = decode_tags(par) 

3229 for tags in tagsets: 

3230 data_extend(tr, "tags", tags) 

3231 data_extend(tr, "topics", topics) 

3232 elif cls == "english": 

3233 # If the text contains any of certain grammatical words, treat it 

3234 # as a "note" instead of "english" 

3235 if re.search(tr_note_re, par): 

3236 if par.endswith(":"): 3236 ↛ 3237line 3236 didn't jump to line 3237 because the condition on line 3236 was never true

3237 par = par[:-1] 

3238 if par not in ("see entry for forms",): 3238 ↛ 3113line 3238 didn't jump to line 3113 because the condition on line 3238 was always true

3239 if note: 3239 ↛ 3240line 3239 didn't jump to line 3240 because the condition on line 3239 was never true

3240 note = note + ";" + par 

3241 else: 

3242 note = par 

3243 else: 

3244 # There can be more than one parenthesized english item, see 

3245 # e.g. Aunt/English/Translations/Tamil 

3246 if "translation" in tr and "english" in tr: 

3247 tr["english"] += "; " + par # DEPRECATED for "translation" 

3248 tr["translation"] += "; " + par 

3249 else: 

3250 tr["english"] = par # DEPRECATED for "translation" 

3251 tr["translation"] = par 

3252 elif cls == "romanization": 

3253 # print("roman text={!r} text cls={}" 

3254 # .format(text, classify_desc(text))) 

3255 if classify_desc(text) in ( 

3256 "english", 

3257 "romanization", 

3258 ) and lang not in ("Egyptian",): 

3259 if beginning: 

3260 restore_beginning += "({}) ".format(par) 

3261 else: 

3262 restore_end = " ({})".format(par) + restore_end 

3263 else: 

3264 if tr.get("roman"): 3264 ↛ 3265line 3264 didn't jump to line 3265 because the condition on line 3264 was never true

3265 wxr.wtp.debug( 

3266 'more than one value in "roman": {} vs. {}'.format( 

3267 tr["roman"], par 

3268 ), 

3269 sortid="form_descriptions/2013", 

3270 ) 

3271 tr["roman"] = par 

3272 elif cls == "taxonomic": 3272 ↛ 3273line 3272 didn't jump to line 3273 because the condition on line 3272 was never true

3273 if tr.get("taxonomic"): 

3274 wxr.wtp.debug( 

3275 'more than one value in "taxonomic": {} vs. {}'.format( 

3276 tr["taxonomic"], par 

3277 ), 

3278 sortid="form_descriptions/2019", 

3279 ) 

3280 if re.match(r"×[A-Z]", par): 

3281 data_append(tr, "tags", "extinct") 

3282 par = par[1:] 

3283 tr["taxonomic"] = par 

3284 elif cls == "other": 3284 ↛ 3294line 3284 didn't jump to line 3294 because the condition on line 3284 was always true

3285 if tr.get("alt"): 3285 ↛ 3286line 3285 didn't jump to line 3286 because the condition on line 3285 was never true

3286 wxr.wtp.debug( 

3287 'more than one value in "alt": {} vs. {}'.format( 

3288 tr["alt"], par 

3289 ), 

3290 sortid="form_descriptions/2028", 

3291 ) 

3292 tr["alt"] = par 

3293 else: 

3294 wxr.wtp.debug( 

3295 "parse_translation_desc unimplemented cls {}: {}".format( 

3296 cls, par 

3297 ), 

3298 sortid="form_descriptions/2033", 

3299 ) 

3300 

3301 # Check for gender indications in suffix 

3302 text, final_tags = parse_head_final_tags(wxr, lang, text) 

3303 data_extend(tr, "tags", final_tags) 

3304 

3305 # Restore those parts that we did not want to remove (they are often 

3306 # optional words or words that are always used with the given translation) 

3307 text = restore_beginning + text + restore_end 

3308 

3309 if note: 

3310 tr["note"] = note.strip() 

3311 if text and text not in ignored_translations: 

3312 tr["word"] = text.strip() 

3313 

3314 # Sometimes gender seems to be at the end of "roman" field, see e.g. 

3315 # fire/English/Noun/Translations/Egyptian (for "oxidation reaction") 

3316 roman = tr.get("roman") 

3317 if roman: 

3318 if roman.endswith(" f"): 3318 ↛ 3319line 3318 didn't jump to line 3319 because the condition on line 3318 was never true

3319 data_append(tr, "tags", "feminine") 

3320 tr["roman"] = roman[:-2].strip() 

3321 elif roman.endswith(" m"): 3321 ↛ 3322line 3321 didn't jump to line 3322 because the condition on line 3321 was never true

3322 data_append(tr, "tags", "masculine") 

3323 tr["roman"] = roman[:-2].strip() 

3324 

3325 # If the word now has "translation" field but no "roman" field, and 

3326 # the word would be classified "other" (generally non-latin 

3327 # characters), and the value in "translation" is only one lowercase 

3328 # word, move it to "roman". This happens semi-frequently when the 

3329 # translation is transliterated the same as some English word. 

3330 roman = tr.get("roman") 

3331 english = tr.get("translation") 

3332 if english and not roman and "word" in tr: 

3333 cls = classify_desc(tr["word"]) 

3334 if cls == "other" and " " not in english and english[0].islower(): 

3335 del tr["translation"] 

3336 if "english" in tr: # DEPRECATED for "translation" 3336 ↛ 3338line 3336 didn't jump to line 3338 because the condition on line 3336 was always true

3337 del tr["english"] 

3338 tr["roman"] = english 

3339 

3340 # If the entry now has both tr["roman"] and tr["word"] and they have 

3341 # the same value, delete tr["roman"] (e.g., man/English/Translations 

3342 # Evenki) 

3343 if tr.get("word") and tr.get("roman") == tr.get("word"): 3343 ↛ 3344line 3343 didn't jump to line 3344 because the condition on line 3343 was never true

3344 del tr["roman"] 

3345 

3346 

3347def parse_alt_or_inflection_of( 

3348 wxr: WiktextractContext, gloss: str, gloss_template_args: set[str] 

3349) -> Optional[tuple[list[str], Optional[list[AltOf]]]]: 

3350 """Tries to parse an inflection-of or alt-of description. If successful, 

3351 this returns (tags, alt-of/inflection-of-dict). If the description cannot 

3352 be parsed, this returns None. This may also return (tags, None) when the 

3353 gloss describes a form (or some other tags were extracted from it), but 

3354 there was no alt-of/form-of/synonym-of word.""" 

3355 # print("parse_alt_or_inflection_of: {!r}".format(gloss)) 

3356 # Occasionally inflection_of/alt_of have "A(n) " etc. at the beginning. 

3357 

3358 # Never interpret a gloss that is equal to the word itself as a tag 

3359 # (e.g., instrumental/Romanian, instrumental/Spanish). 

3360 if gloss.lower() == wxr.wtp.title.lower() or ( # type:ignore[union-attr] 

3361 len(gloss) >= 5 and distw([gloss.lower()], wxr.wtp.title.lower()) < 0.2 # type:ignore[union-attr] 

3362 ): 

3363 return None 

3364 

3365 # First try parsing it as-is 

3366 parsed = parse_alt_or_inflection_of1(wxr, gloss, gloss_template_args) 

3367 if parsed is not None: 

3368 return parsed 

3369 

3370 # Next try parsing it with the first character converted to lowercase if 

3371 # it was previously uppercase. 

3372 if gloss and gloss[0].isupper(): 

3373 gloss = gloss[0].lower() + gloss[1:] 

3374 parsed = parse_alt_or_inflection_of1(wxr, gloss, gloss_template_args) 

3375 if parsed is not None: 

3376 return parsed 

3377 

3378 return None 

3379 

3380 

3381# These tags are not allowed in alt-or-inflection-of parsing 

3382alt_infl_disallowed: set[str] = set( 

3383 [ 

3384 "error-unknown-tag", 

3385 "place", # Not in inflected forms and causes problems e.g. house/ 

3386 # English 

3387 ] 

3388) 

3389 

3390 

3391def parse_alt_or_inflection_of1( 

3392 wxr: WiktextractContext, gloss: str, gloss_template_args: set[str] 

3393) -> Optional[tuple[list[str], Optional[list[AltOf]]]]: 

3394 """Helper function for parse_alt_or_inflection_of. This handles a single 

3395 capitalization.""" 

3396 if not gloss or not gloss.strip(): 3396 ↛ 3397line 3396 didn't jump to line 3397 because the condition on line 3396 was never true

3397 return None 

3398 

3399 # Prevent some common errors where we would parse something we shouldn't 

3400 if re.search(r"(?i)form of address ", gloss): 3400 ↛ 3401line 3400 didn't jump to line 3401 because the condition on line 3400 was never true

3401 return None 

3402 

3403 gloss = re.sub(r"only used in [^,]+, ", "", gloss) 

3404 

3405 # First try all formats ending with "of" (or other known last words that 

3406 # can end a form description) 

3407 matches = list(re.finditer(r"\b(of|for|by|as|letter|number) ", gloss)) 

3408 m: Optional[re.Match] 

3409 for m in reversed(matches): 

3410 desc = gloss[: m.end()].strip() 

3411 base = gloss[m.end() :].strip() 

3412 tagsets, topics = decode_tags(desc, no_unknown_starts=True) 

3413 if not topics and any( 

3414 not (alt_infl_disallowed & set(ts)) for ts in tagsets 

3415 ): 

3416 # Successfully parsed, including "of" etc. 

3417 tags: list[str] = [] 

3418 # If you have ("Western-Armenian", ..., "form-of") as your 

3419 # tag set, it's most probable that it's something like 

3420 # "Western Armenian form of խոսել (xosel)", which should 

3421 # get "alt-of" instead of "form-of" (inflection). 

3422 # խօսիլ/Armenian 

3423 for ts_t in tagsets: 

3424 if "form-of" in ts_t and any( 

3425 valid_tags.get(tk) == "dialect" for tk in ts_t 

3426 ): 

3427 ts_s = (set(ts_t) - {"form-of"}) | {"alt-of"} 

3428 else: 

3429 ts_s = set(ts_t) 

3430 if not (alt_infl_disallowed & ts_s): 3430 ↛ 3423line 3430 didn't jump to line 3423 because the condition on line 3430 was always true

3431 tags.extend(ts_s) 

3432 if ( 

3433 "alt-of" in tags 

3434 or "form-of" in tags 

3435 or "synonym-of" in tags 

3436 or "compound-of" in tags 

3437 ): 

3438 break 

3439 if m.group(1) == "of": 

3440 # Try parsing without the final "of". This is commonly used in 

3441 # various form-of expressions. 

3442 desc = gloss[: m.start()] 

3443 base = gloss[m.end() :] 

3444 tagsets, topics = decode_tags(desc, no_unknown_starts=True) 

3445 # print("ALT_OR_INFL: desc={!r} base={!r} tagsets={} topics={}" 

3446 # .format(desc, base, tagsets, topics)) 

3447 if not topics and any( 

3448 not (alt_infl_disallowed & set(t)) for t in tagsets 

3449 ): 

3450 tags = [] 

3451 for t in tagsets: 

3452 if not (alt_infl_disallowed & set(t)): 3452 ↛ 3451line 3452 didn't jump to line 3451 because the condition on line 3452 was always true

3453 tags.extend(t) 

3454 # It must have at least one tag from form_of_tags 

3455 if set(tags) & form_of_tags: 

3456 # Accept this as form-of 

3457 tags.append("form-of") 

3458 break 

3459 if set(tags) & alt_of_tags: 

3460 # Accept this as alt-of 

3461 tags.append("alt-of") 

3462 break 

3463 

3464 else: 

3465 # Did not find a form description based on last word; see if the 

3466 # whole description is tags 

3467 tagsets, topics = decode_tags(gloss, no_unknown_starts=True) 

3468 if not topics and any( 

3469 not (alt_infl_disallowed & set(ts)) and form_of_tags & set(ts) 

3470 for ts in tagsets 

3471 ): 

3472 tags = [] 

3473 for ts in tagsets: 

3474 if not (alt_infl_disallowed & set(ts)) and form_of_tags & set( 3474 ↛ 3473line 3474 didn't jump to line 3473 because the condition on line 3474 was always true

3475 ts 

3476 ): 

3477 tags.extend(ts) 

3478 base = "" 

3479 else: 

3480 return None 

3481 

3482 # kludge for Spanish (again): 'x of [word] combined with [clitic]' 

3483 m = re.search(r"combined with \w+$", base) 

3484 if m: 3484 ↛ 3485line 3484 didn't jump to line 3485 because the condition on line 3484 was never true

3485 tagsets, topics = decode_tags(m.group(0), no_unknown_starts=True) 

3486 if not topics: 

3487 for ts in tagsets: 

3488 tags.extend(ts) 

3489 base = base[: m.start()] 

3490 

3491 # It is fairly common for form_of glosses to end with something like 

3492 # "ablative case" or "in instructive case". Parse that ending. 

3493 base = base.strip() 

3494 lst = base.split() 

3495 # print("parse_alt_or_inflection_of: lst={}".format(lst)) 

3496 if len(lst) >= 3 and lst[-1] in ("case", "case."): 3496 ↛ 3497line 3496 didn't jump to line 3497 because the condition on line 3496 was never true

3497 node = valid_sequences.children.get(lst[-2]) 

3498 if node and node.end: 

3499 for s in node.tags: 

3500 tags.extend(s.split(" ")) 

3501 lst = lst[:-2] 

3502 if lst[-1] == "in" and len(lst) > 1: 

3503 lst = lst[:-1] 

3504 

3505 # Eliminate empty and duplicate tags 

3506 tags = sorted(set(t for t in tags if t)) 

3507 

3508 # Clean up some extra stuff from the linked word, separating the text 

3509 # into ``base`` (the linked word) and ``extra`` (additional information, 

3510 # such as English translation or clarifying word sense information). 

3511 orig_base = base 

3512 base = re.sub(alt_of_form_of_clean_re, "", orig_base) 

3513 base = re.sub(r" [(⟨][^()]*[)⟩]", "", base) # Remove all (...) groups 

3514 extra = orig_base[len(base) :] 

3515 extra = re.sub(r"^[- :;.,,—]+", "", extra) 

3516 if extra.endswith(".") and extra.count(".") == 1: 

3517 extra = extra[:-1].strip() 

3518 m = re.match(r"^\(([^()]*)\)$", extra) 

3519 if m: 3519 ↛ 3520line 3519 didn't jump to line 3520 because the condition on line 3519 was never true

3520 extra = m.group(1) 

3521 else: 

3522 # These weird backets used in "slash mark" 

3523 m = re.match(r"^⟨([^()]*)⟩$", extra) 

3524 if m: 3524 ↛ 3525line 3524 didn't jump to line 3525 because the condition on line 3524 was never true

3525 extra = m.group(1) 

3526 m = re.match(r'^[“"]([^"“”]*)["”]$', extra) 

3527 if m: 3527 ↛ 3528line 3527 didn't jump to line 3528 because the condition on line 3527 was never true

3528 extra = m.group(1) 

3529 # Note: base might still contain comma-separated values and values 

3530 # separated by "and" 

3531 base = base.strip() 

3532 if base.endswith(",") and len(base) > 2: 3532 ↛ 3533line 3532 didn't jump to line 3533 because the condition on line 3532 was never true

3533 base = base[:-1].strip() 

3534 while ( 

3535 base.endswith(".") 

3536 and not wxr.wtp.page_exists(base) 

3537 and base not in gloss_template_args 

3538 ): 

3539 base = base[:-1].strip() 

3540 if base.endswith('(\u201cconjecture")'): 3540 ↛ 3541line 3540 didn't jump to line 3541 because the condition on line 3540 was never true

3541 base = base[:-14].strip() 

3542 tags.append("conjecture") 

3543 while ( 3543 ↛ 3548line 3543 didn't jump to line 3548 because the condition on line 3543 was never true

3544 base.endswith(".") 

3545 and not wxr.wtp.page_exists(base) 

3546 and base not in gloss_template_args 

3547 ): 

3548 base = base[:-1].strip() 

3549 if ( 3549 ↛ 3554line 3549 didn't jump to line 3554 because the condition on line 3549 was never true

3550 base.endswith(".") 

3551 and base not in gloss_template_args 

3552 and base[:-1] in gloss_template_args 

3553 ): 

3554 base = base[:-1] 

3555 base = base.strip() 

3556 if not base: 

3557 return tags, None 

3558 

3559 # Kludge: Spanish verb forms seem to have a dot added at the end. 

3560 # Remove it; we know of no Spanish verbs ending with a dot. 

3561 language = wxr.wtp.section 

3562 pos = wxr.wtp.subsection 

3563 # print("language={} pos={} base={}".format(language, pos, base)) 

3564 if ( 3564 ↛ 3570line 3564 didn't jump to line 3570 because the condition on line 3564 was never true

3565 base.endswith(".") 

3566 and len(base) > 1 

3567 and base[-2].isalpha() 

3568 and (language == "Spanish" and pos == "Verb") 

3569 ): 

3570 base = base[:-1] 

3571 

3572 # Split base to alternatives when multiple alternatives provided 

3573 parts = split_at_comma_semi(base, extra=[" / ", "/", r" \+ "]) 

3574 titleword = re.sub(r"^Reconstruction:[^/]*/", "", wxr.wtp.title or "") 

3575 if ( 

3576 len(parts) <= 1 

3577 or base.startswith("/") 

3578 or base.endswith("/") 

3579 or "/" in titleword 

3580 ): 

3581 parts = [base] 

3582 # Split base to alternatives when of form "a or b" and "a" and "b" are 

3583 # similar (generally spelling variants of the same word or similar words) 

3584 if len(parts) == 1: 

3585 pp = base.split() 

3586 if len(pp) == 3 and pp[1] == "or" and distw([pp[0]], pp[2]) < 0.4: 

3587 parts = [pp[0], pp[2]] 

3588 

3589 # Create form-of/alt-of entries based on the extracted data 

3590 dt_lst: list[AltOf] = [] 

3591 for p in parts: 

3592 # Check for some suspicious base forms 

3593 m = re.search(r"[.,] |[{}()]", p) 

3594 if m and not wxr.wtp.page_exists(p): 3594 ↛ 3595line 3594 didn't jump to line 3595 because the condition on line 3594 was never true

3595 wxr.wtp.debug( 

3596 "suspicious alt_of/form_of with {!r}: {}".format(m.group(0), p), 

3597 sortid="form_descriptions/2278", 

3598 ) 

3599 if p.startswith("*") and len(p) >= 3 and p[1].isalpha(): 3599 ↛ 3600line 3599 didn't jump to line 3600 because the condition on line 3599 was never true

3600 p = p[1:] 

3601 dt: AltOf = {"word": p} 

3602 if extra: 

3603 dt["extra"] = extra 

3604 dt_lst.append(dt) 

3605 # print("alt_or_infl_of returning tags={} lst={} base={!r}" 

3606 # .format(tags, lst, base)) 

3607 return tags, dt_lst 

3608 

3609 

3610@functools.lru_cache(maxsize=65536) 

3611def classify_desc( 

3612 desc: str, 

3613 allow_unknown_tags=False, 

3614 no_unknown_starts=False, 

3615 accepted: Union[tuple[str, ...], frozenset[str]] = tuple(), 

3616) -> str: 

3617 """Determines whether the given description is most likely tags, english, 

3618 a romanization, or something else. Returns one of: "tags", "english", 

3619 "romanization", or "other". If ``allow_unknown_tags`` is True, then 

3620 allow "tags" classification even when the only tags are those starting 

3621 with a word in allowed_unknown_starts.""" 

3622 assert isinstance(desc, str) 

3623 # Empty and whitespace-only strings are treated as "other" 

3624 desc = desc.strip() 

3625 if not desc: 

3626 return "other" 

3627 

3628 normalized_desc = unicodedata.normalize("NFKD", desc) 

3629 

3630 # If it can be fully decoded as tags without errors, treat as tags 

3631 tagsets, topics = decode_tags(desc, no_unknown_starts=no_unknown_starts) 

3632 for tagset in tagsets: 

3633 assert isinstance(tagset, (list, tuple, set)) 

3634 if "error-unknown-tag" not in tagset and ( 

3635 topics or allow_unknown_tags or any(" " not in x for x in tagset) 

3636 ): 

3637 return "tags" 

3638 

3639 # Check if it looks like the taxonomic name of a species 

3640 if desc in known_species: 

3641 return "taxonomic" 

3642 desc1 = re.sub(r"^×([A-Z])", r"\1", desc) 

3643 desc1 = re.sub(r"\s*×.*", "", desc1) 

3644 lst = desc1.split() 

3645 if len(lst) > 1 and len(lst) <= 5 and lst[0] in known_firsts: 

3646 have_non_english = 1 if lst[0].lower() not in english_words else 0 

3647 for x in lst[1:]: 

3648 if x in ("A", "B", "C", "D", "E", "F", "I", "II", "III", "IV", "V"): 

3649 continue 

3650 if x[0].isupper(): 

3651 break 

3652 if x not in english_words: 

3653 have_non_english += 1 

3654 else: 

3655 # Starts with known taxonomic term, does not contain uppercase 

3656 # words (except allowed letters) and at least one word is not 

3657 # English 

3658 if have_non_english >= len(lst) - 1 and have_non_english > 0: 3658 ↛ 3664line 3658 didn't jump to line 3664 because the condition on line 3658 was always true

3659 return "taxonomic" 

3660 

3661 # If all words are in our English dictionary, interpret as English. 

3662 # [ -~] is regex black magic, "ALL CHARACTERS from space to tilde" 

3663 # in ASCII. Took me a while to figure out. 

3664 if re.match(r"[ -~―—“”…'‘’ʹ€]+$", normalized_desc) and len(desc) > 1: 

3665 if desc in english_words and desc[0].isalpha(): 

3666 return "english" # Handles ones containing whitespace 

3667 desc1 = re.sub( 

3668 tokenizer_fixup_re, lambda m: tokenizer_fixup_map[m.group(0)], desc 

3669 ) 

3670 tokens = tokenizer.tokenize(desc1) 

3671 if not tokens: 3671 ↛ 3672line 3671 didn't jump to line 3672 because the condition on line 3671 was never true

3672 return "other" 

3673 lst_bool = list( 

3674 x not in not_english_words 

3675 and 

3676 # not x.isdecimal() and 

3677 ( 

3678 x in english_words 

3679 or x.lower() in english_words 

3680 or x in known_firsts 

3681 or x[0].isdigit() 

3682 or x in accepted 

3683 or 

3684 # (x[0].isupper() and x.find("-") < 0 and x.isascii()) or 

3685 ( 

3686 x.endswith("s") and len(x) >= 4 and x[:-1] in english_words 

3687 ) # Plural 

3688 or ( 

3689 x.endswith("ies") 

3690 and len(x) >= 5 

3691 and x[:-3] + "y" in english_words 

3692 ) # E.g. lily - lilies 

3693 or ( 

3694 x.endswith("ing") 

3695 and len(x) >= 5 

3696 and x[:-3] in english_words 

3697 ) # E.g. bring - bringing 

3698 or ( 

3699 x.endswith("ing") 

3700 and len(x) >= 5 

3701 and x[:-3] + "e" in english_words 

3702 ) # E.g., tone - toning 

3703 or ( 

3704 x.endswith("ed") and len(x) >= 5 and x[:-2] in english_words 

3705 ) # E.g. hang - hanged 

3706 or ( 

3707 x.endswith("ed") 

3708 and len(x) >= 5 

3709 and x[:-2] + "e" in english_words 

3710 ) # E.g. atone - atoned 

3711 or (x.endswith("'s") and x[:-2] in english_words) 

3712 or (x.endswith("s'") and x[:-2] in english_words) 

3713 or ( 

3714 x.endswith("ise") 

3715 and len(x) >= 5 

3716 and x[:-3] + "ize" in english_words 

3717 ) 

3718 or ( 

3719 x.endswith("ised") 

3720 and len(x) >= 6 

3721 and x[:-4] + "ized" in english_words 

3722 ) 

3723 or ( 

3724 x.endswith("ising") 

3725 and len(x) >= 7 

3726 and x[:-5] + "izing" in english_words 

3727 ) 

3728 or ( 

3729 re.search(r"[-/]", x) 

3730 and all( 

3731 ((y in english_words and len(y) > 2) or not y) 

3732 for y in re.split(r"[-/]", x) 

3733 ) 

3734 ) 

3735 ) 

3736 for x in tokens 

3737 ) 

3738 cnt = lst_bool.count(True) 

3739 rejected_words = tuple( 

3740 x for i, x in enumerate(tokens) if not lst_bool[i] 

3741 ) 

3742 if ( 

3743 any( 

3744 lst_bool[i] and x[0].isalpha() and len(x) > 1 

3745 for i, x in enumerate(tokens) 

3746 ) 

3747 and not desc.startswith("-") 

3748 and not desc.endswith("-") 

3749 and re.search(r"\w+", desc) 

3750 and ( 

3751 cnt == len(lst_bool) 

3752 or ( 

3753 any( 

3754 lst_bool[i] and len(x) > 3 for i, x in enumerate(tokens) 

3755 ) 

3756 and cnt >= len(lst_bool) - 1 

3757 ) 

3758 or cnt / len(lst_bool) >= 0.8 

3759 or ( 

3760 all(x in potentially_english_words for x in rejected_words) 

3761 and cnt / len(lst_bool) >= 0.50 

3762 ) 

3763 ) 

3764 ): 

3765 return "english" 

3766 # Some translations have apparent pronunciation descriptions in /.../ 

3767 # which we'll put in the romanization field (even though they probably are 

3768 # not exactly romanizations). 

3769 if desc.startswith("/") and desc.endswith("/"): 

3770 return "romanization" 

3771 # If all characters are in classes that could occur in romanizations, 

3772 # treat as romanization 

3773 classes = list( 

3774 unicodedata.category(x) if x not in ("-", ",", ":", "/", '"') else "OK" 

3775 for x in normalized_desc 

3776 ) 

3777 classes1 = [] 

3778 num_latin = 0 

3779 num_greek = 0 

3780 # part = "" 

3781 # for ch, cl in zip(normalized_desc, classes): 

3782 # part += f"{ch}({cl})" 

3783 # print(part) 

3784 for ch, cl in zip(normalized_desc, classes): 

3785 if ch in ( 

3786 "'", # ' in Arabic, / in IPA-like parenthesized forms 

3787 ".", # e.g., "..." in translations 

3788 ";", 

3789 ":", 

3790 "!", 

3791 "‘", 

3792 "’", 

3793 '"', 

3794 "“", 

3795 "”", 

3796 "/", 

3797 "?", 

3798 "…", # alternative to "..." 

3799 "⁉", # 見る/Japanese automatic transcriptions... 

3800 "?", 

3801 "!", 

3802 "⁻", # superscript -, used in some Cantonese roman, e.g. "we" 

3803 "ʔ", 

3804 "ʼ", 

3805 "ʾ", 

3806 "ʹ", 

3807 ): # ʹ e.g. in understand/English/verb Russian transl 

3808 classes1.append("OK") 

3809 continue 

3810 if cl not in ("Ll", "Lu"): 

3811 classes1.append(cl) 

3812 continue 

3813 try: 

3814 name = unicodedata.name(ch) 

3815 first = name.split()[0] 

3816 if first == "LATIN": 

3817 num_latin += 1 

3818 elif first == "GREEK": 

3819 num_greek += 1 

3820 elif first == "COMBINING": # Combining diacritic 3820 ↛ 3821line 3820 didn't jump to line 3821 because the condition on line 3820 was never true

3821 cl = "OK" 

3822 elif re.match(non_latin_scripts_re, name): 3822 ↛ 3826line 3822 didn't jump to line 3826 because the condition on line 3822 was always true

3823 cl = "NO" # Not acceptable in romanizations 

3824 except ValueError: 

3825 cl = "NO" # Not acceptable in romanizations 

3826 classes1.append(cl) 

3827 # print("classify_desc: {!r} classes1: {}".format(desc, classes1)) 

3828 # print(set(classes1) ) 

3829 if all( 

3830 x in ("Ll", "Lu", "Lt", "Lm", "Mn", "Mc", "Zs", "Nd", "OK") 

3831 for x in classes1 

3832 ): 

3833 if ( 

3834 (num_latin >= num_greek + 2 or num_greek == 0) 

3835 and classes1.count("OK") < len(classes1) 

3836 and classes1.count("Nd") < len(classes1) 

3837 ): 

3838 return "romanization" 

3839 # Otherwise it is something else, such as hanji version of the word 

3840 return "other" 

3841 

3842 

3843def remove_text_in_parentheses(text: str) -> str: 

3844 parentheses = 0 

3845 new_text = "" 

3846 for c in text: 

3847 if c == "(": 

3848 parentheses += 1 

3849 elif c == ")": 

3850 parentheses -= 1 

3851 elif parentheses == 0: 

3852 new_text += c 

3853 return new_text