Coverage for src/wiktextract/extractor/zh/linkage.py: 91%
245 statements
« prev ^ index » next coverage.py v7.10.3, created at 2025-08-15 05:18 +0000
1import re
3from wikitextprocessor import (
4 HTMLNode,
5 LevelNode,
6 NodeKind,
7 TemplateNode,
8 WikiNode,
9)
11from ...page import clean_node
12from ...wxr_context import WiktextractContext
13from ..ruby import extract_ruby
14from .models import Form, Linkage, WordEntry
15from .tags import translate_raw_tags
18def extract_linkage_section(
19 wxr: WiktextractContext,
20 page_data: list[WordEntry],
21 level_node: LevelNode,
22 linkage_type: str,
23) -> None:
24 sense = ""
25 linkage_list = []
26 for node in level_node.find_child(NodeKind.TEMPLATE | NodeKind.LIST):
27 if node.kind == NodeKind.LIST:
28 for item_node in node.find_child(NodeKind.LIST_ITEM):
29 sense, new_linkage_list = process_linkage_list_item(
30 wxr, item_node, sense
31 )
32 linkage_list.extend(new_linkage_list)
33 elif isinstance(node, TemplateNode): 33 ↛ 26line 33 didn't jump to line 26 because the condition on line 33 was always true
34 if node.template_name in ["s", "sense"]:
35 sense = clean_node(wxr, None, node).strip("()： ")
36 elif node.template_name == "zh-dial":
37 linkage_list.extend(extract_zh_dial_template(wxr, node, sense))
38 elif re.fullmatch(
39 r"(?:col|der|rel)\d", node.template_name, re.I
40 ) or node.template_name.endswith("-saurus"):
41 linkage_list.extend(
42 process_linkage_col_template(wxr, node, sense)
43 )
44 elif node.template_name == "ja-r/multi":
45 linkage_list.extend(
46 extract_ja_r_multi_template(wxr, node, sense)
47 )
49 if linkage_type == "alt_forms":
50 forms = [
51 Form(
52 form=l_data.word,
53 sense=l_data.sense,
54 tags=l_data.tags + ["alternative"],
55 raw_tags=l_data.raw_tags,
56 roman=l_data.roman,
57 ruby=l_data.ruby,
58 attestations=l_data.attestations,
59 )
60 for l_data in linkage_list
61 ]
62 page_data[-1].forms.extend(forms)
63 else:
64 getattr(page_data[-1], linkage_type).extend(linkage_list)
65 for data in page_data[:-1]:
66 if ( 66 ↛ 72line 66 didn't jump to line 72 because the condition on line 66 was never true
67 data.lang_code == page_data[-1].lang_code
68 and data.sounds == page_data[-1].sounds
69 and data.etymology_text == page_data[-1].etymology_text
70 and data.pos_level == page_data[-1].pos_level == level_node.kind
71 ):
72 getattr(data, linkage_type).extend(linkage_list)
75def process_linkage_list_item(
76 wxr: WiktextractContext, list_item: WikiNode, sense: str
77) -> tuple[str, list[Linkage]]:
78 raw_tags = []
79 linkage_list = []
80 for item_child in list_item.children:
81 if isinstance(item_child, TemplateNode):
82 if item_child.template_name in ["s", "sense"]:
83 sense = clean_node(wxr, None, item_child).strip("()： ")
84 elif item_child.template_name in ["qualifier", "qual"]:
85 raw_tags.append(clean_node(wxr, None, item_child).strip("()"))
86 elif item_child.template_name == "zh-l":
87 linkage_list.extend(
88 process_zh_l_template(wxr, item_child, sense, raw_tags)
89 )
90 raw_tags.clear()
91 elif item_child.template_name == "ja-r":
92 linkage_list.append(
93 process_ja_r_template(wxr, item_child, sense, raw_tags)
94 )
95 raw_tags.clear()
96 elif item_child.template_name.lower() in [
97 "l",
98 "link",
99 "alter",
100 "alt",
101 ]:
102 linkage_list.extend(
103 process_l_template(wxr, item_child, sense, raw_tags)
104 )
105 raw_tags.clear()
106 elif ( 106 ↛ 80line 106 didn't jump to line 80 because the condition on line 106 was always true
107 item_child.template_name.lower() in ["defdate", "datedef"]
108 and len(linkage_list) > 0
109 ):
110 from .gloss import extract_defdate_template
112 extract_defdate_template(wxr, linkage_list[-1], item_child)
113 elif (
114 isinstance(item_child, WikiNode)
115 and item_child.kind == NodeKind.LINK
116 ):
117 word = clean_node(wxr, None, item_child)
118 if len(word) > 0: 118 ↛ 80line 118 didn't jump to line 80 because the condition on line 118 was always true
119 linkage_data = Linkage(
120 word=word, sense=sense, raw_tags=raw_tags
121 )
122 translate_raw_tags(linkage_data)
123 linkage_list.append(linkage_data)
124 raw_tags.clear()
125 elif ( 125 ↛ 129line 125 didn't jump to line 129 because the condition on line 125 was never true
126 isinstance(item_child, WikiNode)
127 and item_child.kind == NodeKind.LIST
128 ):
129 for child_list_item in item_child.find_child(NodeKind.LIST_ITEM):
130 _, new_list = process_linkage_list_item(
131 wxr, child_list_item, sense
132 )
133 linkage_list.extend(new_list)
135 return sense, linkage_list
138def extract_zh_dial_template(
139 wxr: WiktextractContext, template_node: TemplateNode, sense: str
140) -> list[Linkage]:
141 from .pronunciation import split_zh_pron_raw_tag
143 linkage_list = []
144 expanded_node = wxr.wtp.parse(
145 wxr.wtp.node_to_wikitext(template_node), expand_all=True
146 )
147 for table_node in expanded_node.find_child_recursively(NodeKind.TABLE):
148 is_note_row = False
149 note_tags = {}
150 for row_node in table_node.find_child(NodeKind.TABLE_ROW):
151 for cell_node in row_node.find_child(
152 NodeKind.TABLE_HEADER_CELL | NodeKind.TABLE_CELL
153 ):
154 if cell_node.kind == NodeKind.TABLE_HEADER_CELL:
155 is_note_row = clean_node(wxr, None, cell_node) == "註解"
156 elif is_note_row:
157 for note_str in clean_node(wxr, None, cell_node).split(";"):
158 if "-" in note_str: 158 ↛ 157line 158 didn't jump to line 157 because the condition on line 158 was always true
159 note_symbol, note = note_str.split("-", maxsplit=1)
160 note_symbol = note_symbol.strip()
161 note = note.strip()
162 if note_symbol != "" and note != "": 162 ↛ 157line 162 didn't jump to line 157 because the condition on line 162 was always true
163 note_tags[note_symbol] = note
164 lang_tags = []
165 region_tags = []
166 for row_node in table_node.find_child(NodeKind.TABLE_ROW):
167 if not row_node.contain_node(NodeKind.TABLE_CELL):
168 continue # skip header row
169 for header_node in row_node.find_child(NodeKind.TABLE_HEADER_CELL):
170 lang_tags = split_zh_pron_raw_tag(
171 clean_node(wxr, None, header_node)
172 )
173 if lang_tags == ["註解"]: # skip last note row
174 continue
175 for cell_node in row_node.find_child(NodeKind.TABLE_CELL):
176 for link_node in cell_node.find_child(NodeKind.LINK):
177 region_tags = split_zh_pron_raw_tag(
178 clean_node(wxr, None, link_node)
179 )
180 for span_tag in cell_node.find_html("span"):
181 span_text = clean_node(wxr, None, span_tag)
182 if span_text == "": 182 ↛ 183line 182 didn't jump to line 183 because the condition on line 182 was never true
183 continue
184 if (
185 span_tag.attrs.get("lang", "") == "zh"
186 and span_text != wxr.wtp.title
187 ):
188 l_data = Linkage(word=span_text)
189 if len(lang_tags) > 0: 189 ↛ 191line 189 didn't jump to line 191 because the condition on line 189 was always true
190 l_data.raw_tags.extend(lang_tags)
191 if len(region_tags) > 0:
192 l_data.raw_tags.extend(region_tags)
193 translate_raw_tags(l_data)
194 linkage_list.append(l_data)
195 elif (
196 span_tag.attrs.get("style", "") == "font-size:60%"
197 and len(linkage_list) > 0
198 ):
199 for note_symbol in span_text.split(","):
200 note_symbol = note_symbol.strip()
201 raw_tag = note_symbol
202 if note_symbol in note_tags:
203 raw_tag = note_tags[note_symbol]
204 if raw_tag != "": 204 ↛ 199line 204 didn't jump to line 199 because the condition on line 204 was always true
205 linkage_list[-1].raw_tags.append(raw_tag)
206 translate_raw_tags(linkage_list[-1])
208 return linkage_list
211def process_zh_l_template(
212 wxr: WiktextractContext,
213 template_node: TemplateNode,
214 sense: str,
215 raw_tags: list[str] = [],
216) -> list[Linkage]:
217 # https://zh.wiktionary.org/wiki/Template:Zh-l
218 expanded_node = wxr.wtp.parse(
219 wxr.wtp.node_to_wikitext(template_node), expand_all=True
220 )
221 roman = ""
222 linkage_list = []
223 for i_tag in expanded_node.find_html_recursively(
224 "span", attr_name="class", attr_value="Latn"
225 ):
226 roman = clean_node(wxr, None, i_tag)
227 for span_tag in expanded_node.find_html(
228 "span", attr_name="lang", attr_value="zh"
229 ):
230 linkage_data = Linkage(
231 sense=sense,
232 raw_tags=raw_tags,
233 roman=roman,
234 word=clean_node(wxr, None, span_tag),
235 )
236 lang_attr = span_tag.attrs.get("lang", "")
237 if lang_attr == "zh-Hant":
238 linkage_data.tags.append("Traditional-Chinese")
239 elif lang_attr == "zh-Hans":
240 linkage_data.tags.append("Simplified-Chinese")
241 if len(linkage_data.word) > 0 and linkage_data.word != "／":
242 translate_raw_tags(linkage_data)
243 linkage_list.append(linkage_data)
244 return linkage_list
247def process_ja_r_template(
248 wxr: WiktextractContext,
249 template_node: TemplateNode,
250 sense: str,
251 raw_tags: list[str] = [],
252) -> Linkage:
253 # https://zh.wiktionary.org/wiki/Template:Ja-r
254 expanded_node = wxr.wtp.parse(
255 wxr.wtp.node_to_wikitext(template_node), expand_all=True
256 )
257 return process_expanded_ja_r_node(wxr, expanded_node, sense, raw_tags)
260def process_expanded_ja_r_node(
261 wxr: WiktextractContext,
262 expanded_node: WikiNode,
263 sense: str,
264 raw_tags: list[str] = [],
265) -> Linkage:
266 linkage_data = Linkage(sense=sense, raw_tags=raw_tags)
267 for span_node in expanded_node.find_html("span"):
268 span_class = span_node.attrs.get("class", "")
269 if "lang" in span_node.attrs:
270 ruby_data, no_ruby_nodes = extract_ruby(wxr, span_node)
271 linkage_data.word = clean_node(wxr, None, no_ruby_nodes)
272 linkage_data.ruby = ruby_data
273 elif "tr" in span_class:
274 linkage_data.roman = clean_node(wxr, None, span_node)
275 elif "mention-gloss" == span_class: 275 ↛ 276line 275 didn't jump to line 276 because the condition on line 275 was never true
276 linkage_data.sense = clean_node(wxr, None, span_node)
278 translate_raw_tags(linkage_data)
279 return linkage_data
282def process_l_template(
283 wxr: WiktextractContext,
284 t_node: TemplateNode,
285 sense: str,
286 raw_tags: list[str] = [],
287) -> None:
288 # https://zh.wiktionary.org/wiki/Template:l
289 expanded_node = wxr.wtp.parse(
290 wxr.wtp.node_to_wikitext(t_node), expand_all=True
291 )
292 linkage_list = []
293 lang_code = clean_node(wxr, None, t_node.template_parameters.get(1, ""))
294 for span_tag in expanded_node.find_html("span"):
295 span_lang = span_tag.attrs.get("lang", "")
296 span_class = span_tag.attrs.get("class", "")
297 if span_lang == lang_code:
298 linkage_data = Linkage(
299 sense=sense,
300 raw_tags=raw_tags,
301 word=clean_node(wxr, None, span_tag),
302 )
303 if len(linkage_data.word) > 0: 303 ↛ 294line 303 didn't jump to line 294 because the condition on line 303 was always true
304 translate_raw_tags(linkage_data)
305 linkage_list.append(linkage_data)
306 elif span_lang.endswith("-Latn") and len(linkage_list) > 0:
307 linkage_list[-1].roman = clean_node(wxr, None, span_tag)
308 elif "mention-gloss" == span_class and len(linkage_list) > 0:
309 linkage_list[-1].sense = clean_node(wxr, None, span_tag)
311 return linkage_list
314def process_linkage_col_template(
315 wxr: WiktextractContext, template_node: TemplateNode, sense: str
316) -> list[Linkage]:
317 # https://zh.wiktionary.org/wiki/Template:Col3
318 linkage_list = []
319 expanded_template = wxr.wtp.parse(
320 wxr.wtp.node_to_wikitext(template_node), expand_all=True
321 )
322 for ui_tag in expanded_template.find_html_recursively("li"):
323 current_data = []
324 roman = ""
325 raw_tags = []
326 for span_tag in ui_tag.find_html("span"):
327 span_lang = span_tag.attrs.get("lang", "")
328 if span_lang.endswith("-Latn"): 328 ↛ 329line 328 didn't jump to line 329 because the condition on line 328 was never true
329 roman = clean_node(wxr, None, span_tag)
330 elif "qualifier-content" in span_tag.attrs.get("class", ""):
331 span_text = clean_node(wxr, None, span_tag)
332 for raw_tag in re.split(r"或|、", span_text):
333 raw_tag = raw_tag.strip()
334 if raw_tag != "": 334 ↛ 332line 334 didn't jump to line 332 because the condition on line 334 was always true
335 raw_tags.append(raw_tag)
336 elif span_lang != "":
337 l_data = Linkage(
338 word=clean_node(wxr, None, span_tag), sense=sense
339 )
340 class_names = span_tag.attrs.get("class", "")
341 if class_names == "Hant": 341 ↛ 342line 341 didn't jump to line 342 because the condition on line 341 was never true
342 l_data.tags.append("Traditional-Chinese")
343 elif class_names == "Hans": 343 ↛ 344line 343 didn't jump to line 344 because the condition on line 343 was never true
344 l_data.tags.append("Simplified-Chinese")
345 if l_data.word != "": 345 ↛ 326line 345 didn't jump to line 326 because the condition on line 345 was always true
346 current_data.append(l_data)
348 for data in current_data:
349 data.raw_tags.extend(raw_tags)
350 data.roman = roman
351 translate_raw_tags(data)
352 linkage_list.extend(current_data)
354 return linkage_list
357def process_linkage_templates_in_gloss(
358 wxr: WiktextractContext,
359 word_entry: WordEntry,
360 t_node: TemplateNode,
361 linkage_type: str,
362 sense: str,
363) -> None:
364 # https://en.wiktionary.org/wiki/Template:synonyms
365 expanded_node = wxr.wtp.parse(
366 wxr.wtp.node_to_wikitext(t_node), expand_all=True
367 )
368 lang_code = clean_node(wxr, None, t_node.template_parameters.get(1, ""))
369 l_list = []
370 raw_tags = []
371 for top_span_tag in expanded_node.find_html("span"):
372 for node in top_span_tag.children:
373 if isinstance(node, HTMLNode) and node.tag == "span":
374 span_lang = node.attrs.get("lang", "")
375 span_class = node.attrs.get("class", "")
376 if span_lang == lang_code:
377 l_data = Linkage(
378 word=clean_node(wxr, None, node),
379 sense=sense,
380 raw_tags=raw_tags,
381 )
382 if span_class == "Hant":
383 l_data.tags.append("Traditional-Chinese")
384 elif span_class == "Hans":
385 l_data.tags.append("Simplified-Chinese")
386 if l_data.word != "": 386 ↛ 372line 386 didn't jump to line 372 because the condition on line 386 was always true
387 l_list.append(l_data)
388 elif span_lang == f"{lang_code}-Latn" or "tr" in span_class:
389 roman = clean_node(wxr, None, node)
390 for d in l_list:
391 d.roman = roman
392 elif span_class == "mention-gloss": 392 ↛ 393line 392 didn't jump to line 393 because the condition on line 392 was never true
393 sense = clean_node(wxr, None, node)
394 for d in l_list:
395 d.sense = sense
396 elif "qualifier-content" in span_class:
397 raw_tag_str = clean_node(wxr, None, node)
398 for raw_tag in raw_tag_str.split("，"):
399 raw_tag = raw_tag.strip()
400 if raw_tag != "": 400 ↛ 398line 400 didn't jump to line 398 because the condition on line 400 was always true
401 raw_tags.append(raw_tag)
402 elif isinstance(node, str) and node.strip() == "、":
403 getattr(word_entry, linkage_type).extend(l_list)
404 l_list.clear()
406 getattr(word_entry, linkage_type).extend(l_list)
407 for data in getattr(word_entry, linkage_type):
408 translate_raw_tags(data)
411def extract_ja_r_multi_template(
412 wxr: WiktextractContext, template_node: TemplateNode, sense: str
413) -> Linkage:
414 expanded_node = wxr.wtp.parse(
415 wxr.wtp.node_to_wikitext(template_node), expand_all=True
416 )
417 linkage_list = []
418 for list_node in expanded_node.find_child(NodeKind.LIST):
419 for list_item in list_node.find_child(NodeKind.LIST_ITEM):
420 linkage_list.append(
421 process_expanded_ja_r_node(wxr, list_item, sense, [])
422 )
424 return linkage_list