Coverage for src/wiktextract/extractor/en/linkages.py: 72%
758 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-26 05:53 +0000
1# Code related to parsing linkages (synonyms, hypernyms, related terms, etc)
2#
3# Copyright (c) 2019-2021 Tatu Ylonen. See file LICENSE and https://ylonen.org
5import re
6import unicodedata
7from typing import Optional, Sequence
9from wikitextprocessor import (
10 LevelNode,
11 NodeKind,
12 TemplateNode,
13 WikiNode,
14)
15from wikitextprocessor.core import TemplateArgs
16from wikitextprocessor.parser import (
17 HTMLNode,
18 is_list,
19 is_list_item,
20)
22from ...datautils import (
23 data_append,
24 data_extend,
25 ns_title_prefix_tuple,
26 split_at_comma_semi,
27)
28from ...page import clean_node, is_panel_template
29from ...tags import linkage_beginning_tags, valid_tags
30from ...wxr_context import WiktextractContext
31from ..ruby import extract_ruby, parse_ruby # noqa: F401
32from .form_descriptions import (
33 classify_desc,
34 decode_tags,
35 head_final_bantu_langs,
36 head_final_bantu_re,
37 head_final_numeric_langs,
38 head_final_other_langs,
39 head_final_other_re,
40 head_final_re,
41 parse_head_final_tags,
42 parse_sense_qualifier,
43)
44from .section_titles import TRANSLATIONS_TITLE
45from .type_utils import FormData, LinkageData, SenseData, WordData
47# Linkage will be ignored if it matches this regexp before splitting
48linkage_pre_split_ignore_re = re.compile(
49 r"^("
50 + "|".join(
51 re.escape(x)
52 for x in [
53 "For more variations, see ",
54 "Signal flag:",
55 "Semaphore:",
56 ]
57 )
58 + r")"
59)
61# Linkage will be ignored if it has one of these prefixes
62linkage_ignore_prefixes = [
63 "Historical and regional synonyms of ",
64 "edit data",
65 "or these other third-person pronouns",
66 "introduced in Unicode ",
67 "Entries in the ",
68 "Wikipedia article ",
69 "Wiktionary's coverage of ",
70 "Ethnologue entry for ",
71 "Any of Thesaurus:",
72 "See contents of Category:",
73 "See also Thesaurus:",
74 "See also Appendix:",
75 "see also Appendix:",
76 "see also Thesaurus:",
77 "As SMS messaging ",
78 "For the reversed question mark used in some right-to-left-scripts",
79 "such as ",
80 "Appendix:",
81 "Category:",
82 ":Category:",
83]
85# Linkage will be ignored if it has any of these suffixes
86linkage_ignore_suffixes = [
87 " Wikipedia",
88 " Wikipedia.",
89 " edition of Wiktionary",
90]
92# Linkage will be ignored if it is one of these (with full match)
93linkage_ignore_whole = [
94 "etc.",
95 "other derived terms:",
96 "Formal terms",
97 "informal and slang terms",
98]
100# Linkage will be ignored if it matches this regexp
101linkage_ignore_re = re.compile(
102 r"^("
103 + "|".join(re.escape(x) for x in linkage_ignore_whole)
104 + r")$|^("
105 + "|".join(re.escape(x) for x in linkage_ignore_prefixes)
106 + r")|("
107 + "|".join(re.escape(x) for x in linkage_ignore_suffixes)
108 + r")$"
109)
111# These prefixes will be removed from linkages, leaving the rest. This is
112# considered separately for each linkage in a list.
113linkage_remove_prefixes_re = re.compile(
114 r"^("
115 + r"|".join(
116 re.escape(x)
117 for x in [
118 ":",
119 "see Thesaurus:",
120 "See Thesaurus:",
121 "see also Thesaurus:",
122 "See also Thesaurus:",
123 "see also ",
124 "See also ",
125 "see ",
126 "See ",
127 "from ",
128 "abbreviation of ",
129 "ISO 639-1 code ",
130 "ISO 639-3 code ",
131 "Thesaurus:",
132 ]
133 )
134 + ")"
135)
137# When removing prefix from linkage, this dictionary can be used to map
138# the removed prefix to a space-separated list of tags to add
139linkage_remove_prefixes_tags = {
140 "abbreviation of ": "abbreviation",
141}
143# These suffixes will be removed from linkages, leaving the rest. This is
144# considered separately for each linkage in a list.
145linkage_remove_suffixes_re = re.compile(
146 r"(\s+on (Wikispecies|Wikimedia Commons|"
147 r"[A-Z]\w+ Wiktionary|[A-Z]\w+ Wikipedia)\.?|"
148 r"\s*[-–] Pre-reform orthography.*)"
149 r"$"
150)
152# Ignore linkage parenthesized sections that contain one of these strings
153linkage_paren_ignore_contains_re = re.compile(
154 r"\b("
155 + "|".join(
156 re.escape(x)
157 for x in [
158 "from Etymology",
159 "used as",
160 "usage notes",
161 ]
162 )
163 + ")([, ]|$)"
164)
166taxonomic_ending_map = {
167 "superkingdoms": "superkingdom",
168 "kingdoms": "kingdom",
169 "subkingdoms": "subkingdom",
170 "infrakingdoms": "infrakingdom",
171 "phylums": "phylum",
172 "subphylums": "subphylum",
173 "infraphylums": "infraphylum",
174 "superclasses": "superclass",
175 "classes": "class",
176 "orders": "order",
177 "suborders": "suborder",
178 "families": "family",
179 "subfamilies": "subfamily",
180 "genera": "genus",
181}
182for k, v in list(taxonomic_ending_map.items()):
183 taxonomic_ending_map[v] = v # Also add singular -> singular
184taxonomic_ending_re = re.compile(
185 r"\s+[-‐‑‒–—]\s+({})$".format(
186 "|".join(re.escape(x) for x in taxonomic_ending_map)
187 )
188)
190# Exceptional splits for linkages. This can be used to fix particular linkages
191# that are not handled correctly by the default code. This can also be used
192# to create automatic aliases, e.g., for mapping "..." and "…" to both.
193linkage_split_exceptions = {
194 "∛ ∜": ["∛", "∜"],
195 "...": ["...", "…"],
196 "…": ["...", "…"],
197}
199# Truncate linkage word if it matches any of these strings
200linkage_truncate_re = re.compile(
201 "|".join(
202 re.escape(x)
203 for x in [
204 " and its derived terms",
205 " UTF-16 0x214C",
206 ]
207 )
208)
210# Regexp for identifying special linkages containing lists of letters, digits,
211# or characters
212script_chars_re = re.compile(
213 r"(script letters| script| letters|"
214 r"Dialectological|Puctuation|Symbols|"
215 r"Guillemets|Single guillemets|"
216 r" tetragrams|"
217 r" digits)(;|$)|"
218 r"(^|; )(Letters using |Letters of the |"
219 r"Variations of letter )|"
220 r"^(Hiragana|Katakana)$"
221)
223# Matches an unicode character including any combining diacritics (even if
224# separate characters)
225unicode_dc_re = re.compile(
226 r"\w[{}]|.".format(
227 "".join(
228 chr(x)
229 for x in range(0, 0x110000)
230 if unicodedata.category(chr(x)) == "Mn"
231 )
232 )
233)
236def extract_alt_form_section(
237 wxr: WiktextractContext, word_entry: WordData, level_node: LevelNode
238) -> None:
239 for list_node in level_node.find_child(NodeKind.LIST):
240 for list_item in list_node.find_child(NodeKind.LIST_ITEM):
241 for node in list_item.children:
242 if isinstance(node, TemplateNode) and node.template_name in [
243 "l",
244 "link",
245 "L",
246 "alt",
247 "alter",
248 ]:
249 extract_l_template(wxr, word_entry, node)
250 elif isinstance(node, WikiNode) and node.kind == NodeKind.LINK:
251 word = clean_node(wxr, None, node)
252 if word != "": 252 ↛ 241line 252 didn't jump to line 241 because the condition on line 252 was always true
253 form: FormData = {"form": word, "tags": ["alternative"]}
254 data_append(word_entry, "forms", form)
257def extract_l_template(
258 wxr: WiktextractContext, word_entry: WordData, t_node: TemplateNode
259) -> None:
260 forms: list[FormData] = []
261 expanded_node = wxr.wtp.parse(
262 wxr.wtp.node_to_wikitext(t_node), expand_all=True
263 )
264 lang_code = clean_node(wxr, None, t_node.template_parameters.get(1, ""))
265 for span_tag in expanded_node.find_html("span"):
266 span_lang = span_tag.attrs.get("lang", "")
267 span_class = span_tag.attrs.get("class", "")
268 if span_lang == lang_code: 268 ↛ 273line 268 didn't jump to line 273 because the condition on line 268 was always true
269 word = clean_node(wxr, None, span_tag)
270 if word != "": 270 ↛ 265line 270 didn't jump to line 265 because the condition on line 270 was always true
271 form: FormData = {"form": word, "tags": ["alternative"]}
272 forms.append(form)
273 elif span_lang.endswith("-Latn") and len(forms) > 0:
274 roman = clean_node(wxr, None, span_tag)
275 if roman != "":
276 forms[-1]["roman"] = roman
277 elif "label-content" in span_class and len(forms) > 0:
278 tag_text = clean_node(wxr, None, span_tag)
279 if classify_desc(tag_text) == "tags":
280 tagsets1, _ = decode_tags(tag_text)
281 tags: list[str] = []
282 for ts in tagsets1:
283 tags.extend(ts)
284 for form in forms:
285 form["tags"].extend(tags)
286 data_extend(word_entry, "forms", forms)
289ZH_DIAL_TAGS = {
290 "Classical Chinese": ["Classical-Chinese"],
291 "Formal": ["formal"],
292 "Written Standard Chinese": ["Written-vernacular-Chinese"],
293 "Northeastern Mandarin": ["Northeastern-Mandarin"],
294 "Jilu Mandarin": ["Jilu-Mandarin"],
295 "Jiaoliao Mandarin": ["Jiaoliao-Mandarin"],
296 "Central Plains Mandarin": ["Central-Plains-Mandarin"],
297 "Lanyin Mandarin": ["Lanyin-Mandarin"],
298 "Southwestern Mandarin": ["Southwestern-Mandarin"],
299 "Jianghuai Mandarin": ["Jianghuai-Mandarin"],
300 "Northern Min": ["Min-Bei"],
301 "Eastern Min": ["Min-Dong"],
302 "Southern Min": ["Min-Nan"],
303 "Zhongshan Min": ["Zhongshan-Min"],
304 "Southern Pinghua": ["Southern-Pinghua"],
305 "Puxian Min": ["Puxian-Min"],
306}
309def extract_zh_dial_template(
310 wxr: WiktextractContext,
311 word_entry: WordData,
312 t_node: TemplateNode,
313 sense: str,
314):
315 # https://en.wiktionary.org/wiki/Template:zh-dial
316 from .pronunciation import split_zh_pron_raw_tag
318 linkage_list: list[LinkageData] = []
319 expanded_node = wxr.wtp.parse(
320 wxr.wtp.node_to_wikitext(t_node), expand_all=True
321 )
322 for table_node in expanded_node.find_child_recursively(NodeKind.TABLE): 322 ↛ 323line 322 didn't jump to line 323 because the loop on line 322 never started
323 is_note_row = False
324 note_tags = {}
325 for row_node in table_node.find_child(NodeKind.TABLE_ROW):
326 for cell_node in row_node.find_child(
327 NodeKind.TABLE_HEADER_CELL | NodeKind.TABLE_CELL
328 ):
329 if cell_node.kind == NodeKind.TABLE_HEADER_CELL:
330 is_note_row = clean_node(wxr, None, cell_node) == "Note"
331 elif is_note_row:
332 for note_str in clean_node(wxr, None, cell_node).split(";"):
333 if "-" in note_str:
334 note_symbol, note = note_str.split("-", maxsplit=1)
335 note_symbol = note_symbol.strip()
336 note = note.strip()
337 if note_symbol != "" and note != "":
338 note_tags[note_symbol] = note
339 lang_tags = []
340 region_tags = []
341 for row_node in table_node.find_child(NodeKind.TABLE_ROW):
342 if not row_node.contain_node(NodeKind.TABLE_CELL):
343 continue # skip header row
344 for header_node in row_node.find_child(NodeKind.TABLE_HEADER_CELL):
345 lang_tags = split_zh_pron_raw_tag(
346 clean_node(wxr, None, header_node)
347 )
348 if lang_tags == ["Note"]: # skip last note row
349 continue
350 for cell_node in row_node.find_child(NodeKind.TABLE_CELL):
351 for link_node in cell_node.find_child(NodeKind.LINK):
352 region_tags = split_zh_pron_raw_tag(
353 clean_node(wxr, None, link_node)
354 )
355 for span_tag in cell_node.find_html("span"):
356 span_text = clean_node(wxr, None, span_tag)
357 if span_text == "":
358 continue
359 if (
360 span_tag.attrs.get("lang", "") == "zh"
361 and span_text != wxr.wtp.title
362 ):
363 l_data: LinkageData = {"word": span_text}
364 if sense != "":
365 l_data["sense"] = sense
366 if len(lang_tags) > 0:
367 data_extend(l_data, "raw_tags", lang_tags)
368 if len(region_tags) > 0:
369 data_extend(l_data, "raw_tags", region_tags)
370 linkage_list.append(l_data)
371 elif (
372 span_tag.attrs.get("style", "") == "font-size:60%"
373 and len(linkage_list) > 0
374 ):
375 for note_symbol in span_text.split(","):
376 note_symbol = note_symbol.strip()
377 raw_tag = note_symbol
378 if note_symbol in note_tags:
379 raw_tag = note_tags[note_symbol]
380 if raw_tag != "":
381 data_append(
382 linkage_list[-1], "raw_tags", raw_tag
383 )
385 for l_data in linkage_list:
386 raw_tags = []
387 for raw_tag in l_data.get("raw_tags", []):
388 if raw_tag in ZH_DIAL_TAGS:
389 data_extend(l_data, "tags", ZH_DIAL_TAGS[raw_tag])
390 elif raw_tag in valid_tags:
391 data_append(l_data, "tags", raw_tag)
392 else:
393 raw_tags.append(raw_tag)
394 if len(raw_tags) > 0:
395 l_data["raw_tags"] = raw_tags
396 elif "raw_tags" in l_data:
397 del l_data["raw_tags"]
398 data_extend(word_entry, "synonyms", linkage_list)
401def parse_linkage(
402 wxr: WiktextractContext,
403 data: WordData,
404 field: str,
405 linkagenode: LevelNode,
406 word: str,
407 sense_datas: list[SenseData],
408 is_reconstruction: bool,
409) -> None:
410 assert isinstance(data, dict)
411 assert isinstance(field, str)
412 assert isinstance(linkagenode, LevelNode)
413 # print("field", field)
414 # print("data", data)
415 # print("children:")
416 if not wxr.config.capture_linkages: 416 ↛ 417line 416 didn't jump to line 417 because the condition on line 416 was never true
417 return
418 have_panel_template = False
420 def linkage_template_fn1(name: str, ht: TemplateArgs) -> Optional[str]:
421 nonlocal have_panel_template
422 if is_panel_template(wxr, name):
423 have_panel_template = True
424 return ""
425 # Ignore auto-filled templates like Template:table:Solar System/en
426 if name.startswith(("table:", "list:")):
427 return ""
428 return None
430 # Main body of parse_linkage()
431 l_nodes: list[str | WikiNode] = []
432 l_sense = ""
433 for node in linkagenode.children:
434 if isinstance(node, TemplateNode) and node.template_name == "zh-dial":
435 extract_zh_dial_template(wxr, data, node, l_sense)
436 elif isinstance(node, WikiNode) and node.kind == NodeKind.LIST:
437 for list_item in node.find_child(NodeKind.LIST_ITEM):
438 for t_node in list_item.find_child(NodeKind.TEMPLATE):
439 if t_node.template_name in ["s", "sense"]:
440 l_sense = clean_node(wxr, None, t_node).strip("(): ")
441 l_nodes.append(node)
442 else:
443 l_nodes.append(node)
444 text = wxr.wtp.node_to_wikitext(l_nodes)
445 parsed = wxr.wtp.parse(
446 text, expand_all=True, template_fn=linkage_template_fn1
447 )
449 text_outside_list_items = parse_linkage_recurse(
450 wxr,
451 parsed.children,
452 field,
453 None,
454 None,
455 word,
456 data,
457 sense_datas,
458 is_reconstruction,
459 )
461 if not data.get(field) and not have_panel_template:
462 text = "".join(text_outside_list_items).strip()
463 if "\n" not in text and "," in text and text.count(",") > 3:
464 if not text.startswith("See "): 464 ↛ exitline 464 didn't return from function 'parse_linkage' because the condition on line 464 was always true
465 parse_linkage_item(
466 wxr,
467 [text],
468 field,
469 word,
470 data,
471 sense_datas,
472 is_reconstruction,
473 None,
474 )
477def parse_linkage_recurse(
478 wxr: WiktextractContext,
479 contents: list[WikiNode | str],
480 field: str,
481 sense: str | None,
482 block_header_sense: str | None,
483 word: str,
484 data,
485 sense_datas,
486 is_reconstruction,
487) -> list[str]:
488 assert isinstance(contents, (list, tuple))
489 assert sense is None or isinstance(sense, str)
490 # print("PARSE_LINKAGE_RECURSE: {}: {}".format(sense, contents))
492 # Return values
493 text_outside_list_items: list[str] = []
495 for node in contents:
496 if isinstance(node, str):
497 # Ignore top-level text, generally comments before the
498 # linkages list. However, if no linkages are found, then
499 # use this for linkages (not all words use bullet points
500 # for linkages).
501 text_outside_list_items.append(node)
502 continue
503 assert isinstance(node, WikiNode)
504 kind = node.kind
505 # print("PARSE_LINKAGE_RECURSE CHILD", kind)
506 if is_list(node) or kind in (NodeKind.TABLE, NodeKind.TABLE_ROW):
507 toli = parse_linkage_recurse(
508 wxr,
509 node.children,
510 field,
511 sense,
512 block_header_sense,
513 word,
514 data,
515 sense_datas,
516 is_reconstruction,
517 )
518 text_outside_list_items.extend(toli)
519 elif is_list_item(node) or kind == NodeKind.TABLE_CELL:
520 v = parse_linkage_item(
521 wxr,
522 node.children,
523 field,
524 word,
525 data,
526 sense_datas,
527 is_reconstruction,
528 sense,
529 )
530 if v is not None: 530 ↛ 495line 530 didn't jump to line 495 because the condition on line 530 was always true
531 # parse_linkage_item() can return a value that should
532 # be used as the sense for the follow-on linkages,
533 # which are typically provided in a table (see 滿)
534 block_header_sense = "".join(v)
535 elif kind in (
536 NodeKind.TABLE_CAPTION,
537 NodeKind.TABLE_HEADER_CELL,
538 NodeKind.PREFORMATTED,
539 NodeKind.BOLD,
540 ):
541 # Let's still ignore table extra stuff
542 continue
543 elif isinstance(node, HTMLNode): 543 ↛ 545line 543 didn't jump to line 545 because the condition on line 543 was never true
544 # Recurse to process inside the HTML for most tags
545 if node.sarg in ("gallery", "ref", "cite", "caption"):
546 continue
547 classes = (node.attrs.get("class") or "").replace("+", " ").split()
548 if "qualifier-content" in classes:
549 sense1 = clean_node(wxr, None, node.children)
550 if sense1.endswith(":"):
551 sense1 = sense1[:-1].strip()
552 if sense and sense1:
553 wxr.wtp.debug(
554 "linkage qualifier-content on multiple "
555 "levels: {!r} and {!r}".format(sense, sense1),
556 sortid="page/2170",
557 )
558 toli = parse_linkage_recurse(
559 wxr,
560 node.children,
561 field,
562 sense1,
563 block_header_sense,
564 word,
565 data,
566 sense_datas,
567 is_reconstruction,
568 )
569 text_outside_list_items.extend(toli)
570 elif "list-switcher-header" in classes:
571 block_header_sense = clean_node(wxr, None, node.children)
572 if block_header_sense.endswith(":"):
573 block_header_sense = block_header_sense[:-1].strip()
574 elif any(x in classes for x in ("NavFrame", "term-list")):
575 # NavFrame uses previously assigned block_header_sense
576 # (from a "(sense):" item) and clears it afterwards
577 # print(f"{sense=}, {block_header_sense=}")
578 toli = parse_linkage_recurse(
579 wxr,
580 node.children,
581 field,
582 sense or block_header_sense,
583 block_header_sense,
584 word,
585 data,
586 sense_datas,
587 is_reconstruction,
588 )
589 text_outside_list_items.extend(toli)
590 block_header_sense = None
591 else:
592 toli = parse_linkage_recurse(
593 wxr,
594 node.children,
595 field,
596 sense,
597 block_header_sense,
598 word,
599 data,
600 sense_datas,
601 is_reconstruction,
602 )
603 text_outside_list_items.extend(toli)
604 elif isinstance(node, LevelNode): 604 ↛ 606line 604 didn't jump to line 606 because the condition on line 604 was never true
605 # Just recurse to any possible subsections
606 toli = parse_linkage_recurse(
607 wxr,
608 node.children,
609 field,
610 sense,
611 block_header_sense,
612 word,
613 data,
614 sense_datas,
615 is_reconstruction,
616 )
617 text_outside_list_items.extend(toli)
618 elif kind in (NodeKind.BOLD, NodeKind.ITALIC):
619 # Skip these on top level; at least sometimes bold is
620 # used for indicating a subtitle
621 continue
622 elif kind == NodeKind.LINK: 622 ↛ 639line 622 didn't jump to line 639 because the condition on line 622 was always true
623 # Recurse into the last argument
624 # Apparently ":/" is used as a link to "/", so strip
625 # initial value
626 toli = parse_linkage_recurse(
627 wxr,
628 node.largs[-1],
629 field,
630 sense,
631 block_header_sense,
632 word,
633 data,
634 sense_datas,
635 is_reconstruction,
636 )
637 text_outside_list_items.extend(toli)
638 else:
639 wxr.wtp.debug(
640 "parse_linkage_recurse unhandled {}: {}".format(kind, node),
641 sortid="page/2196",
642 )
644 return text_outside_list_items
647def parse_linkage_item(
648 wxr: WiktextractContext,
649 contents: list[str | WikiNode],
650 field: str,
651 word: str,
652 data: WordData,
653 sense_datas: list[SenseData],
654 is_reconstruction: bool,
655 sense: str | None = None,
656) -> list[str]:
657 assert isinstance(contents, (list, tuple))
658 assert isinstance(field, str)
659 assert sense is None or isinstance(sense, str)
661 # print("PARSE_LINKAGE_ITEM: {} ({}): {}"
662 # .format(field, sense, contents))
664 parts: list[str] = []
665 ruby: list[tuple[str, str]] = []
666 urls: list[str] = []
667 # data about link text; this is used to skip splitting on
668 # linkage text items that contain stuff like commas; for
669 # example "Hunde, die bellen, beißen nicht" in article
670 # beißen is split into "Hunde", "die bellen" etc.
671 # We take that link text and use it, eventually,
672 # in split_at_comma_semi to skip splitting on those
673 # commas.
674 links_that_should_not_be_split: list[str] = []
676 def item_recurse(
677 contents: list[str | WikiNode], possible_sense: str | None = None
678 ) -> bool:
679 assert isinstance(contents, (list, tuple))
680 nonlocal sense
681 nonlocal ruby
682 nonlocal parts
683 is_sense = False
684 # print("ITEM_RECURSE:", contents)
685 for node in contents:
686 if isinstance(node, str):
687 parts.append(node)
688 continue
689 kind = node.kind
690 # print(
691 # "ITEM_RECURSE KIND:",
692 # kind,
693 # node.sarg if node.sarg else node.largs,
694 # )
696 #### parts into possible_sense
697 if (
698 is_list_item(node)
699 or is_list(node)
700 or kind
701 in (
702 NodeKind.TABLE,
703 NodeKind.TABLE_ROW,
704 NodeKind.TABLE_CELL,
705 )
706 and parts
707 ):
708 # print(f"{parts=}")
709 candidate_sense: str | None
710 candidate_sense = clean_node(wxr, None, parts)
711 is_sense = False
713 if candidate_sense.endswith(":"):
714 is_sense = True
715 candidate_sense = candidate_sense[:-1].strip()
716 if candidate_sense.startswith("(") and candidate_sense.endswith( 716 ↛ 719line 716 didn't jump to line 719 because the condition on line 716 was never true
717 ")"
718 ):
719 is_sense = True
720 candidate_sense = candidate_sense[1:-1].strip()
721 if (
722 candidate_sense.lower() == TRANSLATIONS_TITLE
723 or not is_sense
724 ):
725 candidate_sense = None
726 # print(f"{possible_sense=}, {is_sense=}")
727 if is_sense:
728 possible_sense = candidate_sense
729 parts = []
730 else:
731 candidate_sense = None
733 # Handle nodes
734 if is_list_item(node): 734 ↛ 735line 734 didn't jump to line 735 because the condition on line 734 was never true
735 parse_linkage_item(
736 wxr,
737 node.children,
738 field,
739 word,
740 data,
741 sense_datas,
742 is_reconstruction,
743 possible_sense or sense,
744 )
745 elif is_list(node) or kind in (
746 NodeKind.TABLE,
747 NodeKind.TABLE_ROW,
748 NodeKind.TABLE_CELL,
749 ):
750 parse_linkage_recurse(
751 wxr,
752 node.children,
753 field,
754 possible_sense or sense,
755 None,
756 word,
757 data,
758 sense_datas,
759 is_reconstruction,
760 )
761 elif kind in ( 761 ↛ 765line 761 didn't jump to line 765 because the condition on line 761 was never true
762 NodeKind.TABLE_HEADER_CELL,
763 NodeKind.TABLE_CAPTION,
764 ):
765 continue
766 elif kind == NodeKind.HTML: 766 ↛ 767line 766 didn't jump to line 767 because the condition on line 766 was never true
767 classes = (node.attrs.get("class") or "").split()
768 if node.sarg in ("gallery", "ref", "cite", "caption"):
769 continue
770 elif node.sarg == "ruby":
771 rb = parse_ruby(wxr, node)
772 if rb:
773 ruby.append(rb)
774 parts.append(rb[0])
775 continue
776 elif node.sarg == "math":
777 parts.append(clean_node(wxr, None, node))
778 continue
779 elif "interProject" in classes:
780 continue # These do not seem to be displayed
781 if "NavFrame" in classes:
782 parse_linkage_recurse(
783 wxr,
784 node.children,
785 field,
786 possible_sense or sense,
787 None,
788 word,
789 data,
790 sense_datas,
791 is_reconstruction,
792 )
793 else:
794 item_recurse(node.children, possible_sense)
795 elif kind == NodeKind.LINK:
796 ignore = False
797 if isinstance(node.largs[0][0], str): 797 ↛ 685line 797 didn't jump to line 685 because the condition on line 797 was always true
798 v1 = node.largs[0][0].strip().lower()
799 if v1.startswith( 799 ↛ 803line 799 didn't jump to line 803 because the condition on line 799 was never true
800 ns_title_prefix_tuple(wxr, "Category", True)
801 + ns_title_prefix_tuple(wxr, "File", True)
802 ):
803 ignore = True
804 if not ignore: 804 ↛ 685line 804 didn't jump to line 685 because the condition on line 804 was always true
805 v = node.largs[-1]
806 if (
807 len(node.largs) == 1
808 and len(v) > 0
809 and isinstance(v[0], str)
810 and v[0][0] == ":"
811 ):
812 v = [v[0][1:]] + list(v[1:]) # type:ignore
813 if (
814 len(v) > 0
815 and isinstance(v[0], str)
816 and not v[0].isalnum()
817 ):
818 links_that_should_not_be_split.append("".join(v[0])) # type: ignore
819 item_recurse(v, possible_sense)
820 elif kind == NodeKind.URL:
821 if len(node.largs) < 2 and node.largs:
822 # Naked url captured
823 urls.extend(node.largs[-1]) # type:ignore[arg-type]
824 continue
825 if len(node.largs) == 2: 825 ↛ 830line 825 didn't jump to line 830 because the condition on line 825 was always true
826 # Url from link with text
827 urls.append(node.largs[0][-1]) # type:ignore[arg-type]
828 # print(f"{node.largs=!r}")
829 # print("linkage recurse URL {}".format(node))
830 item_recurse(node.largs[-1], possible_sense)
831 elif kind in ( 831 ↛ 838line 831 didn't jump to line 838 because the condition on line 831 was always true
832 NodeKind.PREFORMATTED,
833 NodeKind.BOLD,
834 NodeKind.ITALIC,
835 ):
836 item_recurse(node.children)
837 else:
838 wxr.wtp.debug(
839 "linkage item_recurse unhandled {}: {}".format(
840 node.kind, node
841 ),
842 sortid="page/2073",
843 )
845 return is_sense
847 # print("LINKAGE CONTENTS BEFORE ITEM_RECURSE: {!r}"
848 # .format(contents))
850 is_sense = item_recurse(contents)
852 if not is_sense:
853 item = clean_node(wxr, None, parts)
854 # print("LINKAGE ITEM CONTENTS:", parts)
855 # print("CLEANED ITEM: {!r}".format(item))
856 # print(f"URLS {urls=!r}")
858 if v := parse_linkage_item_text( 858 ↛ 871line 858 didn't jump to line 871 because the condition on line 858 was never true
859 wxr,
860 word,
861 data,
862 field,
863 item,
864 sense,
865 ruby,
866 sense_datas,
867 is_reconstruction,
868 urls or None,
869 links_that_should_not_be_split or None,
870 ):
871 return [v]
873 return []
876def parse_linkage_item_text(
877 wxr: WiktextractContext,
878 word: str,
879 data: WordData,
880 field: str,
881 item: str,
882 sense: Optional[str],
883 ruby: list,
884 pos_datas: list,
885 is_reconstruction: bool,
886 urls: Optional[list[str]] = None,
887 links: Optional[list[str]] = None,
888) -> Optional[str]:
889 """Parses a linkage item once it has been converted to a string. This
890 may add one or more linkages to ``data`` under ``field``. This
891 returns None or a string that contains a sense that should be applied
892 to additional linkages (commonly used in tables for Asian characters)."""
893 assert isinstance(wxr, WiktextractContext)
894 assert isinstance(word, str) # Main word (derived from page title)
895 assert isinstance(data, dict) # Parsed linkages are stored here under field
896 assert isinstance(field, str) # The field under which to store linkage
897 assert isinstance(item, str) # The string to parse
898 assert sense is None or isinstance(sense, str)
899 assert isinstance(ruby, list) # Captured ruby (hiragana/katakana) or ""
900 assert isinstance(pos_datas, list) # List of senses (containing "glosses")
901 assert urls is None or isinstance(urls, list) # Captured urls
902 assert is_reconstruction in (True, False)
904 item = item.replace("()", "")
905 item = re.sub(r"\s+", " ", item)
906 item = item.strip()
908 base_roman = None
909 base_alt = None
910 base_english = None
911 script_chars = False
912 base_qualifier = None
913 lang = wxr.wtp.section
915 # If ``sense`` can be parsed as tags, treat it as tags instead
916 if sense:
917 cls = classify_desc(sense, no_unknown_starts=True)
918 if cls == "tags":
919 base_qualifier = sense
920 sense = None
922 # Check if this item is a stand-alone sense (or tag) specifier
923 # for following items (e.g., commonly in a table, see 滿)
924 m = re.match(r"\(([-a-zA-Z0-9 ]+)\):$", item)
925 if m:
926 return m.group(1)
928 # Check for pre-split ignored linkages using the appropriate regexp
929 if re.search(linkage_pre_split_ignore_re, item):
930 return None
932 # print(" LINKAGE ITEM: {}: {} (sense {})"
933 # .format(field, item, sense))
935 # Replace occurrences of ~ in the item by the page title
936 safetitle = wxr.wtp.title.replace("\\", "\\\\") # type: ignore[union-attr]
937 item = item.replace(" ~ ", " " + safetitle + " ")
938 item = re.sub(r"^~ ", safetitle + " ", item)
939 item = re.sub(r" ~$", " " + safetitle, item)
941 # Many taxonomic terms contain hyponym lists that end with the
942 # kind of the hyponym (a taxonomic level in plural). Recognize
943 # such and add the term in singular to all linkages in the list.
944 m = re.search(taxonomic_ending_re, item)
945 if m:
946 base_english = taxonomic_ending_map[m.group(1)]
947 item = item[: m.start()]
949 # Some Korean and Japanese words use "word (romanized): english" pattern
950 # Sometimes the parenthesized part contains comma-separated alt and roman.
951 m = re.match(r"(.+?) \(([^():]+)\): ([-a-zA-Z0-9,. ]+)$", item)
952 if m:
953 rom = m.group(2)
954 eng = m.group(3)
955 rest = m.group(1)
956 if (
957 classify_desc(rest, no_unknown_starts=True) == "other"
958 and classify_desc(eng, no_unknown_starts=True) == "english"
959 ):
960 item = rest
961 base_roman = rom
962 lst = base_roman.split(", ")
963 if (
964 len(lst) == 2
965 and classify_desc(lst[0], no_unknown_starts=True) == "other"
966 ):
967 base_alt = lst[0]
968 base_roman = lst[1]
969 if base_english:
970 base_english += "; " + eng
971 else:
972 base_english = eng
974 # Many words have tags or similar descriptions in the beginning
975 # followed by a colon and one or more linkages (e.g.,
976 # panetella/Finnish)
977 m = re.match(r"^\((([^():]|\([^()]*\))+)\): ([^:]*)$", item) or re.match(
978 r"^([a-zA-Z][-'a-zA-Z0-9 ]*" r"(\([^()]+\)[-'a-zA-Z0-9 ]*)*): ([^:]*)$",
979 item,
980 )
981 if m:
982 desc = m.group(1)
983 rest = m.group(len(m.groups()))
984 # Check for certain comma-separated tags combined
985 # with English text at the beginning or end of a
986 # comma-separated parenthesized list
987 lst = split_at_comma_semi(desc, skipped=links)
988 while len(lst) > 1:
989 # Check for tags at the beginning
990 cls = classify_desc(lst[0], no_unknown_starts=True)
991 if cls == "tags":
992 if base_qualifier:
993 base_qualifier += ", " + lst[0]
994 else:
995 base_qualifier = lst[0]
996 lst = lst[1:]
997 continue
998 # Check for tags at the end
999 cls = classify_desc(lst[-1], no_unknown_starts=True)
1000 if cls == "tags":
1001 if base_qualifier:
1002 base_qualifier += ", " + lst[-1]
1003 else:
1004 base_qualifier = lst[-1]
1005 lst = lst[:-1]
1006 continue
1007 break
1008 desc = ", ".join(lst)
1010 # Sometimes we have e.g. "chemistry (slang)" with are
1011 # both tags (see "stink"). Handle that case by
1012 # removing parentheses if the value is still tags. The part with
1013 # parentheses could be on either side of the colon.
1014 if "(" in desc:
1015 x = desc.replace("(", ",").replace(")", ",")
1016 if classify_desc(x, no_unknown_starts=True) == "tags":
1017 desc = x
1018 elif "(" in rest:
1019 x = rest.replace("(", ",").replace(")", ",")
1020 if classify_desc(x, no_unknown_starts=True) == "tags":
1021 rest = desc
1022 desc = x
1024 # See if the prefix should trigger special handling for script
1025 # character, letter, digit, etc. handling
1026 if re.search(script_chars_re, desc):
1027 script_chars = True
1029 # Try to determine which side is description and which is
1030 # the linked term (both orders are widely used in Wiktionary)
1031 cls = classify_desc(desc, no_unknown_starts=True)
1032 cls2 = classify_desc(rest, no_unknown_starts=True)
1033 # print("linkage prefix: desc={!r} cls={} rest={!r} cls2={}"
1034 # .format(desc, cls, rest, cls2))
1036 e1 = wxr.wtp.page_exists(desc)
1037 e2 = wxr.wtp.page_exists(rest)
1038 if cls != "tags":
1039 if (
1040 cls2 == "tags"
1041 or (e1 and not e1)
1042 or (
1043 e1
1044 and e2
1045 and cls2 == "english"
1046 and cls in ("other", "romanization")
1047 )
1048 or (
1049 not e1
1050 and not e2
1051 and cls2 == "english"
1052 and cls in ("other", "romanization")
1053 )
1054 ):
1055 desc, rest = rest, desc # Looks like swapped syntax
1056 cls = cls2
1057 if re.search(linkage_paren_ignore_contains_re, desc): 1057 ↛ 1058line 1057 didn't jump to line 1058 because the condition on line 1057 was never true
1058 desc = ""
1059 # print("linkage colon prefix desc={!r} rest={!r} cls={}"
1060 # .format(desc, rest, cls))
1062 # Handle the prefix according to its type
1063 if cls == "tags":
1064 if base_qualifier:
1065 base_qualifier += ", " + desc
1066 else:
1067 base_qualifier = desc
1068 item = rest
1069 elif desc in ("NATO phonetic", "Morse code", "Braille", "ASL Manual"):
1070 if base_english: 1070 ↛ 1071line 1070 didn't jump to line 1071 because the condition on line 1070 was never true
1071 base_english += "; " + base_english
1072 else:
1073 base_english = desc
1074 item = rest
1075 elif cls in ("english", "taxonomic"):
1076 if sense: 1076 ↛ 1077line 1076 didn't jump to line 1077 because the condition on line 1076 was never true
1077 sense += "; " + desc
1078 else:
1079 sense = desc
1080 item = rest
1081 elif desc.isdecimal():
1082 idx = int(desc) - 1
1083 if idx >= 0 and idx < len(pos_datas):
1084 d = pos_datas[idx]
1085 gl = "; ".join(d.get("glosses", ()))
1086 if not gl: 1086 ↛ 1087line 1086 didn't jump to line 1087 because the condition on line 1086 was never true
1087 wxr.wtp.debug(
1088 "parenthesized numeric linkage prefix, "
1089 "but the referenced sense has no gloss: "
1090 "{}".format(desc),
1091 sortid="linkages/355",
1092 )
1093 elif sense:
1094 sense += "; " + gl
1095 else:
1096 sense = gl
1097 item = rest
1098 else:
1099 wxr.wtp.debug(
1100 "parenthesized numeric linkage prefix, "
1101 "but there is no sense with such index: {}".format(desc),
1102 sortid="linkages/365",
1103 )
1104 item = rest
1105 else:
1106 wxr.wtp.debug(
1107 "unrecognized linkage prefix: {} desc={} rest={} "
1108 "cls={} cls2={} e1={} e2={}".format(
1109 item, desc, rest, cls, cls2, e1, e2
1110 ),
1111 sortid="linkages/371",
1112 )
1113 item = rest
1115 base_sense = sense
1117 # Check for certain plural tag forms at end of items list, and apply
1118 # them to all items if found
1119 m = re.search(
1120 r" [-‐‑‒–—―] (diminutives|Diminutives|letters|digits|"
1121 r"characters|symbols|tetragrams|letter names|names|"
1122 r"female names|male names|proper nouns|contractions|"
1123 r"nonstandard spellings|verbs|prepositions|postpositions|"
1124 r"interjections|Abbreviations|abbreviations|variants|"
1125 r"ordinals|nouns|phrases|adjectives|adverbs|"
1126 r"augmentatives|pejoratives|compound words|numerals|"
1127 r"Tally marks|surnames|modern nonstandard spellings)$",
1128 item,
1129 )
1130 if m:
1131 suffix = m.group(1)
1132 if base_qualifier:
1133 base_qualifier += ", " + suffix
1134 else:
1135 base_qualifier = suffix
1136 item = item[: m.start()]
1138 # Certain linkage items have space-separated valus. These are
1139 # generated by, e.g., certain templates
1140 if base_sense and base_sense.endswith(" paper sizes"):
1141 base_qualifier = None
1142 item = ", ".join(item.split())
1143 # XXX isn't this now handled by the generic digits/letters/etc code?
1144 # elif base_qualifier in ("Arabic digits",):
1145 # item = ", ".join(item.split())
1147 item = re.sub(r"\s*\^\(\s*\)|\s*\^\s+", "", item) # Now empty superscript
1148 item = item.strip()
1149 if not item:
1150 return None
1152 # Kludge: if the item contains ")/" (with possibly spaces in between),
1153 # replace it by a comma so it gets split.
1154 item = re.sub(r"\)\s*/", "), ", item)
1156 # The item may contain multiple comma-separated linkages
1157 if base_roman:
1158 subitems = [item]
1159 else:
1160 # Split at commas. Also, in most cases split by " or ", but this
1161 # is complicated - "or" may end certain words (e.g., "logical or")
1162 # and it may separate head-final tags (e.g. "foo f or m"). Also,
1163 # some words have parenthesizxed parts in between, e.g.,
1164 # wife/English/Translations/Yiddish:
1165 # "ווײַב n (vayb) or f, פֿרוי f (froy)"
1166 subitems = []
1167 for item1 in split_at_comma_semi(item, skipped=links):
1168 if " or " not in item1:
1169 subitems.append(item1)
1170 continue
1171 # Item1 contains " or "
1172 item2 = re.sub(r"\s*\([^)]*\)", "", item1)
1173 item2 = re.sub(r"\s+", " ", item2)
1174 if (
1175 (
1176 lang not in head_final_bantu_langs
1177 or not re.search(head_final_bantu_re, item2)
1178 )
1179 and (
1180 lang not in head_final_other_langs
1181 or not re.search(head_final_other_re, item2)
1182 )
1183 and (
1184 not re.search(head_final_re, item2)
1185 or (
1186 item2[-1].isdecimal()
1187 and lang not in head_final_numeric_langs
1188 )
1189 )
1190 and not re.search(r"\bor\b", wxr.wtp.title or "MISSING_TITLE")
1191 and all(
1192 wxr.wtp.title not in x.split(" or ")
1193 for x in split_at_comma_semi(item2, skipped=links)
1194 if " or " in x
1195 )
1196 ):
1197 # We can split this item. Split the non-cleaned version
1198 # that still has any intervening parenthesized parts.
1199 subitems.extend(
1200 split_at_comma_semi(item1, extra=[" or "], skipped=links)
1201 )
1202 else:
1203 subitems.append(item1)
1204 if len(subitems) > 1: # Would be merged from multiple subitems
1205 ruby = [] # XXX what is the purpose of this?
1206 for item1 in subitems:
1207 if len(subitems) > 1 and item1 in ("...", "…"):
1208 # Some lists have ellipsis in the middle - don't generate
1209 # linkages for the ellipsis
1210 continue
1211 item1 = item1.strip()
1212 qualifier = base_qualifier
1213 sense = base_sense
1214 parts = []
1215 roman = base_roman # Usually None
1216 alt = base_alt # Usually None
1217 taxonomic = None
1218 english = base_english
1220 # Some words have derived terms with parenthesized quoted English
1221 # descriptions, which can sometimes essentially be tags
1222 # Some word (bleki/Esperanto...) can have parentheses inside
1223 # the quotes, so let's make this regex even more unreadable.
1224 m = re.search(r"\s*\(“([^”]+)”\)", item1)
1225 if m: 1225 ↛ 1226line 1225 didn't jump to line 1226 because the condition on line 1225 was never true
1226 t = m.group(1)
1227 item1 = (item1[: m.start()] + item1[m.end() :]).strip()
1228 cls = classify_desc(t)
1229 if cls == "tags":
1230 if qualifier:
1231 qualifier += ", " + t
1232 else:
1233 qualifier = t
1234 else:
1235 english = t
1237 # Some Korean words use "word (alt, oman, “english”) pattern
1238 # See 滿/Korean
1239 m = re.match(
1240 r"([^(),;:]+) \(([^(),;:]+), ([^(),;:]+), "
1241 r'[“”"]([^”“"]+)[“”"]\)$',
1242 item1,
1243 )
1244 if (
1245 m
1246 and classify_desc(m.group(1), no_unknown_starts=True) == "other"
1247 and classify_desc(m.group(2), no_unknown_starts=True) == "other"
1248 ):
1249 alt = m.group(2)
1250 roman = m.group(3)
1251 english = m.group(4)
1252 item1 = m.group(1)
1254 words = item1.split(" ")
1255 if (
1256 len(words) > 1
1257 and words[0] in linkage_beginning_tags
1258 and words[0] != wxr.wtp.title
1259 ):
1260 t = linkage_beginning_tags[words[0]]
1261 item1 = " ".join(words[1:])
1262 if qualifier: 1262 ↛ 1263line 1262 didn't jump to line 1263 because the condition on line 1262 was never true
1263 qualifier += ", " + t
1264 else:
1265 qualifier = t
1267 # Extract quoted English translations (there are also other
1268 # kinds of English translations)
1269 def english_repl(m: re.Match) -> str:
1270 nonlocal english
1271 nonlocal qualifier
1272 v = m.group(1).strip()
1273 # If v is "tags: sense", handle the tags
1274 m1 = re.match(r"^([a-zA-Z ]+): (.*)$", v)
1275 if m1 is not None: 1275 ↛ 1276line 1275 didn't jump to line 1276 because the condition on line 1275 was never true
1276 desc, rest = m1.groups()
1277 if classify_desc(desc, no_unknown_starts=True) == "tags":
1278 if qualifier:
1279 qualifier += ", " + desc
1280 else:
1281 qualifier = desc
1282 v = rest
1283 if english:
1284 english += "; " + v
1285 else:
1286 english = v
1287 return ""
1289 item1 = re.sub(r'[“"]([^“”"]+)[“”"],?\s*', english_repl, item1).strip()
1291 # There could be multiple parenthesized parts, and
1292 # sometimes both at the beginning and at the end.
1293 # And sometimes even in the middle, as in e.g.
1294 # wife/English/Translations/Yiddish
1295 while not script_chars and (
1296 not sense or not re.search(script_chars_re, sense)
1297 ):
1298 par = None
1299 nonfirst_par = False
1300 if par is None: 1300 ↛ 1317line 1300 didn't jump to line 1317 because the condition on line 1300 was always true
1301 # Try to find a parenthesized part from the beginning.
1302 m = re.match(r"\((([^()]|\([^()]*\))*)\):?\s*", item1)
1303 if m:
1304 par = m.group(1)
1305 item1 = item1[m.end() :]
1306 else:
1307 # Try to find a parenthesized part at the end or from the
1308 # middle.
1309 m = re.search(
1310 r"\s+\((\d|\d\d|[^\d]([^()]|\([^()]*\))*)\)" r"(\.$)?",
1311 item1,
1312 )
1313 if m:
1314 par = m.group(1)
1315 item1 = item1[: m.start()] + item1[m.end() :]
1316 nonfirst_par = True
1317 if not par:
1318 break
1319 if re.search(linkage_paren_ignore_contains_re, par):
1320 continue # Skip these linkage descriptors
1321 par = par.strip()
1322 # Handle tags from beginning of par. We also handle "other"
1323 # here as Korean entries often have Hanja form in the
1324 # beginning of parenthesis, before romanization. Similar
1325 # for many Japanese entries.
1326 while par: 1326 ↛ 1347line 1326 didn't jump to line 1347 because the condition on line 1326 was always true
1327 idx = par.find(",")
1328 if idx <= 0:
1329 break
1330 cls = classify_desc(par[:idx], no_unknown_starts=True)
1331 if cls == "other" and not alt: 1331 ↛ 1332line 1331 didn't jump to line 1332 because the condition on line 1331 was never true
1332 alt = par[:idx]
1333 elif cls == "taxonomic": 1333 ↛ 1334line 1333 didn't jump to line 1334 because the condition on line 1333 was never true
1334 taxonomic = par[:idx]
1335 elif cls == "tags":
1336 if qualifier:
1337 qualifier += ", " + par[:idx]
1338 else:
1339 qualifier = par[:idx]
1340 else:
1341 break
1342 par = par[idx + 1 :].strip()
1344 # Check for certain comma-separated tags combined
1345 # with English text at the beginning or end of a
1346 # comma-separated parenthesized list
1347 lst = par.split(",") if len(par) > 1 else [par]
1348 lst = list(x.strip() for x in lst if x.strip())
1349 while len(lst) > 1:
1350 cls = classify_desc(lst[0], no_unknown_starts=True)
1351 if cls == "tags": 1351 ↛ 1352line 1351 didn't jump to line 1352 because the condition on line 1351 was never true
1352 if qualifier:
1353 qualifier += ", " + lst[0]
1354 else:
1355 qualifier = lst[0]
1356 lst = lst[1:]
1357 continue
1358 cls = classify_desc(lst[-1], no_unknown_starts=True)
1359 if cls == "tags":
1360 if qualifier:
1361 qualifier += ", " + lst[-1]
1362 else:
1363 qualifier = lst[-1]
1364 lst = lst[:-1]
1365 continue
1366 break
1367 par = ", ".join(lst)
1369 # Handle remaining types
1370 if not par: 1370 ↛ 1371line 1370 didn't jump to line 1371 because the condition on line 1370 was never true
1371 continue
1372 if re.search(script_chars_re, par):
1373 script_chars = True
1374 if classify_desc(par, no_unknown_starts=True) == "tags": 1374 ↛ 1384line 1374 didn't jump to line 1384 because the condition on line 1374 was always true
1375 if base_qualifier: 1375 ↛ 1376line 1375 didn't jump to line 1376 because the condition on line 1375 was never true
1376 base_qualifier += "; " + par
1377 else:
1378 base_qualifier = par
1379 if qualifier: 1379 ↛ 1380line 1379 didn't jump to line 1380 because the condition on line 1379 was never true
1380 qualifier += "; " + par
1381 else:
1382 qualifier = par
1383 else:
1384 if base_sense:
1385 base_sense += "; " + par
1386 else:
1387 base_sense = par
1388 if sense:
1389 sense += "; " + par
1390 else:
1391 sense = par
1392 elif par.endswith(" letter names"): 1392 ↛ 1393line 1392 didn't jump to line 1393 because the condition on line 1392 was never true
1393 if base_qualifier:
1394 base_qualifier += "; " + par
1395 else:
1396 base_qualifier = par
1397 if qualifier:
1398 qualifier += "; " + par
1399 else:
1400 qualifier = par
1401 else:
1402 cls = classify_desc(par)
1403 # print("classify_desc: {!r} -> {}".format(par, cls))
1404 if cls == "tags":
1405 if qualifier:
1406 qualifier += ", " + par
1407 else:
1408 qualifier = par
1409 elif cls == "english":
1410 if nonfirst_par:
1411 if english:
1412 english += "; " + par
1413 else:
1414 english = par
1415 else:
1416 if sense: 1416 ↛ 1417line 1416 didn't jump to line 1417 because the condition on line 1416 was never true
1417 sense += "; " + par
1418 else:
1419 sense = par
1420 elif cls == "romanization":
1421 roman = par
1422 elif cls == "taxonomic":
1423 taxonomic = par
1424 elif par.isdecimal():
1425 idx = int(par) - 1
1426 if idx >= 0 and idx < len(pos_datas):
1427 d = pos_datas[idx]
1428 gl = "; ".join(d.get("glosses", ()))
1429 if not gl: 1429 ↛ 1430line 1429 didn't jump to line 1430 because the condition on line 1429 was never true
1430 wxr.wtp.debug(
1431 "parenthesized number "
1432 "but the referenced sense has no "
1433 "gloss: {}".format(par),
1434 sortid="linkages/665",
1435 )
1436 elif sense: 1436 ↛ 1439line 1436 didn't jump to line 1439 because the condition on line 1436 was always true
1437 sense += "; " + gl
1438 else:
1439 sense = gl
1440 else:
1441 wxr.wtp.debug(
1442 "parenthesized number but there is "
1443 "no sense with such index: {}".format(par),
1444 sortid="linkages/674",
1445 )
1446 else:
1447 if alt: 1447 ↛ 1448line 1447 didn't jump to line 1448 because the condition on line 1447 was never true
1448 alt += "; " + par
1449 else:
1450 alt = par
1452 # Handle certain special cases, unless we are parsing
1453 # script characters.
1454 if not script_chars:
1455 # Ignore all linkages with certain prefixes, suffixes, or parts
1456 # (this is done after removing certain prefixes and suffixes)
1457 if re.search(linkage_ignore_re, item1):
1458 continue # Ignore linkages with certain prefixes
1460 # Remove certain prefixes from linkages
1461 m = re.match(linkage_remove_prefixes_re, item1)
1462 if m:
1463 prefix = item1[: m.end()]
1464 item1 = item1[m.end() :]
1465 if prefix in linkage_remove_prefixes_tags:
1466 if qualifier:
1467 qualifier += ", " + linkage_remove_prefixes_tags[prefix]
1468 else:
1469 qualifier = linkage_remove_prefixes_tags[prefix]
1470 # Recheck ignored linkages
1471 if re.search(linkage_ignore_re, item1):
1472 continue
1474 # Remove certain suffixes from linkages
1475 m = re.search(linkage_remove_suffixes_re, item1)
1476 if m:
1477 item1 = item1[: m.start()]
1479 # Parse linkages with "value = english" syntax (e.g.,
1480 # väittää/Finnish)
1481 idx = item1.find(" = ")
1482 if idx >= 0:
1483 eng = item1[idx + 3 :]
1484 if classify_desc(eng, no_unknown_starts=True) == "english":
1485 english = eng
1486 item1 = item1[:idx]
1487 else:
1488 # Some places seem to use it reversed
1489 # "english = value"
1490 eng = item1[:idx]
1491 if classify_desc(eng, no_unknown_starts=True) == "english":
1492 english = eng
1493 item1 = item1[idx + 3 :]
1495 # Parse linkages with "value - english" syntax (e.g.,
1496 # man/Faroese)
1497 m = re.search(r" [-‐‑‒–—―] ", item1)
1498 if m and "(" not in item1:
1499 suffix = item1[m.end() :]
1500 cls = classify_desc(suffix, no_unknown_starts=True)
1501 if cls == "english":
1502 # This case intentionally ignores old values from english
1503 # (otherwise taxonomic lists fail)
1504 english = suffix
1505 item1 = item1[: m.start()]
1506 elif cls == "tags":
1507 if qualifier: 1507 ↛ 1508line 1507 didn't jump to line 1508 because the condition on line 1507 was never true
1508 qualifier += ", " + suffix
1509 else:
1510 qualifier = suffix
1511 item1 = item1[: m.start()]
1513 # Parse certain tags at the end of the linked term (unless
1514 # we are in a letters list)
1515 item1, q = parse_head_final_tags(wxr, lang or "MISSING_LANG", item1)
1516 if q:
1517 if qualifier: 1517 ↛ 1518line 1517 didn't jump to line 1518 because the condition on line 1517 was never true
1518 qualifier += ", " + ", ".join(q)
1519 else:
1520 qualifier = ", ".join(q)
1522 m = re.search(linkage_truncate_re, item1)
1523 if m: 1523 ↛ 1525line 1523 didn't jump to line 1525 because the condition on line 1523 was never true
1524 # suffix = item1[m.start():] # Currently ignored
1525 item1 = item1[: m.start()]
1526 if not item1:
1527 continue # Ignore empty link targets
1528 if item1 == word:
1529 continue # Ignore self-links
1531 def add(w: str, r: Optional[str]) -> None:
1532 assert isinstance(w, str)
1533 assert r is None or isinstance(r, str)
1534 nonlocal alt
1535 nonlocal taxonomic
1537 # We remove "*" from the beginning of reconstruction linkages.
1538 # Such linkages should only occur in reconstruction senses, so
1539 # this should not cause ambiguity.
1540 if is_reconstruction and w.startswith("*"):
1541 w = w[1:]
1543 # Check if the word contains the Fullwith Solidus, and if
1544 # so, split by it and treat the the results as alternative
1545 # linkages. (This is very commonly used for alternative
1546 # written forms in Chinese compounds and other linkages.)
1547 # However, if the word contains a comma, then we wont't
1548 # split as this is used when we have a different number
1549 # of romanizations than written forms, and don't know
1550 # which is which.
1551 if (
1552 (not w or "," not in w)
1553 and (not r or "," not in r)
1554 and not wxr.wtp.page_exists(w)
1555 ):
1556 lst = w.split("／") if len(w) > 1 else [w]
1557 if len(lst) == 1:
1558 lst = w.split(" / ")
1559 if len(lst) == 1 and len(lst[0]) >= 6:
1560 lst = w.split("/")
1561 if len(lst) > 1:
1562 # Treat each alternative as separate linkage
1563 for w in lst:
1564 add(w, r)
1565 return None
1567 # Heuristically remove "." at the end of most linkages
1568 # (some linkage lists end in a period, but we also have
1569 # abbreviations that end with a period that should be kept)
1570 if (
1571 w.endswith(".")
1572 and not wxr.wtp.page_exists(w)
1573 and (
1574 wxr.wtp.page_exists(w[:-1])
1575 or (len(w) >= 5)
1576 and "." not in w[:-1]
1577 )
1578 ):
1579 w = w[:-1]
1581 # If we have roman but not alt and the word is ASCII,
1582 # move roman to alt.
1583 if r and not alt and w.isascii():
1584 alt = r
1585 r = None
1586 # Add the linkage
1587 dt: LinkageData = {}
1588 if qualifier:
1589 parse_sense_qualifier(wxr, qualifier, dt)
1590 if sense:
1591 dt["sense"] = sense.strip()
1592 if r:
1593 dt["roman"] = r.strip()
1594 if ruby:
1595 dt["ruby"] = ruby
1596 if english:
1597 dt["english"] = english.strip() # DEPRECATED for "translation"
1598 dt["translation"] = english.strip()
1599 if taxonomic:
1600 if re.match(r"×[A-Z]", taxonomic):
1601 data_append(dt, "tags", "extinct")
1602 taxonomic = taxonomic[1:]
1603 dt["taxonomic"] = taxonomic
1604 if re.match(r"×[A-Z]", w):
1605 data_append(dt, "tags", "extinct")
1606 w = w[1:] # Remove × before dead species names
1607 if alt and re.match(r"×[A-Z]", alt):
1608 data_append(dt, "tags", "extinct")
1609 alt = alt[1:] # Remove × before dead species names
1610 if alt and alt.strip() != w:
1611 dt["alt"] = alt.strip()
1612 if urls:
1613 dt["urls"] = [
1614 url.strip() for url in urls if url and isinstance(url, str)
1615 ]
1616 dt["word"] = w
1617 for old in data.get(field, ()): # type: ignore[attr-defined]
1618 if dt == old:
1619 break
1620 else:
1621 data_append(data, field, dt)
1623 # Handle exceptional linkage splits and other linkage
1624 # conversions (including expanding to variant forms)
1625 if item1 in linkage_split_exceptions: 1625 ↛ 1626line 1625 didn't jump to line 1626 because the condition on line 1625 was never true
1626 for item2 in linkage_split_exceptions[item1]:
1627 add(item2, roman)
1628 continue
1630 # Various templates for letters in scripts use spaces as
1631 # separators and also have multiple characters without
1632 # spaces consecutively.
1633 v = sense or qualifier
1634 # print("lang={} v={} script_chars={} item1={!r}"
1635 # .format(wxr.wtp.section, v, script_chars, item1))
1636 if v and script_chars:
1637 if (
1638 len(item1.split()) > 1
1639 or len(list(re.finditer(unicode_dc_re, item1))) == 2
1640 or (len(subitems) > 10 and v in ("Hiragana", "Katakana"))
1641 ):
1642 if v == qualifier:
1643 # if sense:
1644 # sense += "; " + qualifier
1645 # else:
1646 # sense = qualifier
1647 qualifier = None
1648 if re.search(r" (letters|digits|script)$", v):
1649 qualifier = v # Also parse as qualifier
1650 elif re.search( 1650 ↛ 1657line 1650 didn't jump to line 1657 because the condition on line 1650 was always true
1651 r"Variations of letter |"
1652 r"Letters using |"
1653 r"Letters of the ",
1654 v,
1655 ):
1656 qualifier = "letter"
1657 parts = item1.split(". ")
1658 extra: Sequence[str] = ()
1659 if len(parts) > 1: 1659 ↛ 1660line 1659 didn't jump to line 1660 because the condition on line 1659 was never true
1660 extra = parts[1:]
1661 item1 = parts[0]
1662 # Handle multi-character names for chars in language's
1663 # alphabet, e.g., "Ny ny" in P/Hungarian.
1664 if (
1665 len(subitems) > 20
1666 and len(item1.split()) == 2
1667 and all(len(x) <= 3 for x in item1.split())
1668 ):
1669 parts = list(
1670 m.group(0)
1671 for m in re.finditer(r"(\w[\u0300-\u036f]?)+|.", item1)
1672 if not m.group(0).isspace()
1673 and m.group(0) not in ("(", ")")
1674 )
1675 else:
1676 parts = list(
1677 m.group(0)
1678 for m in re.finditer(r".[\u0300-\u036f]?", item1)
1679 if not m.group(0).isspace()
1680 and m.group(0) not in ("(", ")")
1681 )
1682 for e in extra: 1682 ↛ 1683line 1682 didn't jump to line 1683 because the loop on line 1682 never started
1683 idx = e.find(":")
1684 if idx >= 0:
1685 e = e[idx + 1 :].strip()
1686 if e.endswith("."):
1687 e = e[:-1]
1688 parts.extend(e.split())
1690 # XXX this is not correct - see P/Vietnamese
1691 # While some sequences have multiple consecutive
1692 # characters, others use pairs and some have
1693 # 2/3 character names, e.g., "Ng ng".
1695 rparts: Optional[list[Optional[str]]] = None
1696 if roman: 1696 ↛ 1697line 1696 didn't jump to line 1697 because the condition on line 1696 was never true
1697 rparts = list(
1698 m.group(0)
1699 for m in re.finditer(r".[\u0300-\u036f]", roman)
1700 if not m.group(0).isspace()
1701 )
1702 if len(rparts) != len(parts):
1703 rparts = None
1704 if not rparts: 1704 ↛ 1707line 1704 didn't jump to line 1707 because the condition on line 1704 was always true
1705 rparts = [None] * len(parts)
1707 for w, r in zip(parts, rparts):
1708 add(w, r)
1709 continue
1711 add(item1, roman)
1712 return None