Coverage for src/wiktextract/extractor/zh/linkage.py: 91%
248 statements
« prev ^ index » next coverage.py v7.10.7, created at 2025-10-13 10:14 +0000
1import re
3from wikitextprocessor import (
4 HTMLNode,
5 LevelNode,
6 NodeKind,
7 TemplateNode,
8 WikiNode,
9)
11from ...page import clean_node
12from ...wxr_context import WiktextractContext
13from ..ruby import extract_ruby
14from .models import Form, Linkage, WordEntry
15from .tags import translate_raw_tags
18def extract_linkage_section(
19 wxr: WiktextractContext,
20 page_data: list[WordEntry],
21 level_node: LevelNode,
22 linkage_type: str,
23) -> None:
24 sense = ""
25 linkage_list = []
26 for node in level_node.find_child(NodeKind.TEMPLATE | NodeKind.LIST):
27 if node.kind == NodeKind.LIST:
28 for item_node in node.find_child(NodeKind.LIST_ITEM):
29 sense, new_linkage_list = process_linkage_list_item(
30 wxr, item_node, sense
31 )
32 linkage_list.extend(new_linkage_list)
33 elif isinstance(node, TemplateNode): 33 ↛ 26line 33 didn't jump to line 26 because the condition on line 33 was always true
34 if node.template_name in ["s", "sense"]:
35 sense = clean_node(wxr, None, node).strip("()： ")
36 elif node.template_name == "zh-dial":
37 linkage_list.extend(extract_zh_dial_template(wxr, node, sense))
38 elif re.fullmatch(
39 r"(?:col|der|rel)\d", node.template_name, re.I
40 ) or node.template_name.endswith("-saurus"):
41 linkage_list.extend(
42 process_linkage_col_template(wxr, node, sense)
43 )
44 elif node.template_name == "ja-r/multi":
45 linkage_list.extend(
46 extract_ja_r_multi_template(wxr, node, sense)
47 )
49 if linkage_type == "alt_forms":
50 forms = [
51 Form(
52 form=l_data.word,
53 sense=l_data.sense,
54 tags=l_data.tags + ["alternative"],
55 raw_tags=l_data.raw_tags,
56 roman=l_data.roman,
57 ruby=l_data.ruby,
58 attestations=l_data.attestations,
59 )
60 for l_data in linkage_list
61 ]
62 page_data[-1].forms.extend(forms)
63 else:
64 getattr(page_data[-1], linkage_type).extend(linkage_list)
65 for data in page_data[:-1]:
66 if ( 66 ↛ 72line 66 didn't jump to line 72 because the condition on line 66 was never true
67 data.lang_code == page_data[-1].lang_code
68 and data.sounds == page_data[-1].sounds
69 and data.etymology_text == page_data[-1].etymology_text
70 and data.pos_level == page_data[-1].pos_level == level_node.kind
71 ):
72 getattr(data, linkage_type).extend(linkage_list)
75def process_linkage_list_item(
76 wxr: WiktextractContext, list_item: WikiNode, sense: str
77) -> tuple[str, list[Linkage]]:
78 raw_tags = []
79 linkage_list = []
80 for item_child in list_item.children:
81 if isinstance(item_child, TemplateNode):
82 if item_child.template_name in ["s", "sense"]:
83 sense = clean_node(wxr, None, item_child).strip("()： ")
84 elif item_child.template_name in ["qualifier", "qual"]:
85 raw_tags.append(clean_node(wxr, None, item_child).strip("()"))
86 elif item_child.template_name == "zh-l":
87 linkage_list.extend(
88 process_zh_l_template(wxr, item_child, sense, raw_tags)
89 )
90 raw_tags.clear()
91 elif item_child.template_name == "ja-r":
92 linkage_list.append(
93 process_ja_r_template(wxr, item_child, sense, raw_tags)
94 )
95 raw_tags.clear()
96 elif item_child.template_name.lower() in [
97 "l",
98 "link",
99 "alter",
100 "alt",
101 ]:
102 linkage_list.extend(
103 process_l_template(wxr, item_child, sense, raw_tags)
104 )
105 raw_tags.clear()
106 elif ( 106 ↛ 80line 106 didn't jump to line 80 because the condition on line 106 was always true
107 item_child.template_name.lower() in ["defdate", "datedef"]
108 and len(linkage_list) > 0
109 ):
110 from .gloss import extract_defdate_template
112 extract_defdate_template(wxr, linkage_list[-1], item_child)
113 elif (
114 isinstance(item_child, WikiNode)
115 and item_child.kind == NodeKind.LINK
116 ):
117 word = clean_node(wxr, None, item_child)
118 if len(word) > 0: 118 ↛ 80line 118 didn't jump to line 80 because the condition on line 118 was always true
119 linkage_data = Linkage(
120 word=word, sense=sense, raw_tags=raw_tags
121 )
122 translate_raw_tags(linkage_data)
123 linkage_list.append(linkage_data)
124 raw_tags.clear()
125 elif ( 125 ↛ 129line 125 didn't jump to line 129 because the condition on line 125 was never true
126 isinstance(item_child, WikiNode)
127 and item_child.kind == NodeKind.LIST
128 ):
129 for child_list_item in item_child.find_child(NodeKind.LIST_ITEM):
130 _, new_list = process_linkage_list_item(
131 wxr, child_list_item, sense
132 )
133 linkage_list.extend(new_list)
135 return sense, linkage_list
138def extract_zh_dial_template(
139 wxr: WiktextractContext, template_node: TemplateNode, sense: str
140) -> list[Linkage]:
141 from .pronunciation import split_zh_pron_raw_tag
143 linkage_list = []
144 expanded_node = wxr.wtp.parse(
145 wxr.wtp.node_to_wikitext(template_node), expand_all=True
146 )
147 for table_node in expanded_node.find_child_recursively(NodeKind.TABLE):
148 is_note_row = False
149 note_tags = {}
150 for row_node in table_node.find_child(NodeKind.TABLE_ROW):
151 for cell_node in row_node.find_child(
152 NodeKind.TABLE_HEADER_CELL | NodeKind.TABLE_CELL
153 ):
154 if cell_node.kind == NodeKind.TABLE_HEADER_CELL:
155 is_note_row = clean_node(wxr, None, cell_node) == "註解"
156 elif is_note_row:
157 for note_str in clean_node(wxr, None, cell_node).split(";"):
158 if "-" in note_str: 158 ↛ 157line 158 didn't jump to line 157 because the condition on line 158 was always true
159 note_symbol, note = note_str.split("-", maxsplit=1)
160 note_symbol = note_symbol.strip()
161 note = note.strip()
162 if note_symbol != "" and note != "": 162 ↛ 157line 162 didn't jump to line 157 because the condition on line 162 was always true
163 note_tags[note_symbol] = note
164 lang_tags = []
165 region_tags = []
166 for row_node in table_node.find_child(NodeKind.TABLE_ROW):
167 if not row_node.contain_node(NodeKind.TABLE_CELL):
168 continue # skip header row
169 for header_node in row_node.find_child(NodeKind.TABLE_HEADER_CELL):
170 lang_tags = split_zh_pron_raw_tag(
171 clean_node(wxr, None, header_node)
172 )
173 if lang_tags == ["註解"]: # skip last note row
174 continue
175 for cell_node in row_node.find_child(NodeKind.TABLE_CELL):
176 for link_node in cell_node.find_child(NodeKind.LINK):
177 region_tags = split_zh_pron_raw_tag(
178 clean_node(wxr, None, link_node)
179 )
180 for span_tag in cell_node.find_html("span"):
181 span_text = clean_node(wxr, None, span_tag)
182 if span_text == "": 182 ↛ 183line 182 didn't jump to line 183 because the condition on line 182 was never true
183 continue
184 if (
185 span_tag.attrs.get("lang", "") == "zh"
186 and span_text != wxr.wtp.title
187 ):
188 l_data = Linkage(word=span_text, sense=sense)
189 if len(lang_tags) > 0: 189 ↛ 191line 189 didn't jump to line 191 because the condition on line 189 was always true
190 l_data.raw_tags.extend(lang_tags)
191 if len(region_tags) > 0:
192 l_data.raw_tags.extend(region_tags)
193 translate_raw_tags(l_data)
194 linkage_list.append(l_data)
195 elif (
196 span_tag.attrs.get("style", "") == "font-size:60%"
197 and len(linkage_list) > 0
198 ):
199 for note_symbol in span_text.split(","):
200 note_symbol = note_symbol.strip()
201 raw_tag = note_symbol
202 if note_symbol in note_tags:
203 raw_tag = note_tags[note_symbol]
204 if raw_tag != "": 204 ↛ 199line 204 didn't jump to line 199 because the condition on line 204 was always true
205 linkage_list[-1].raw_tags.append(raw_tag)
206 translate_raw_tags(linkage_list[-1])
208 return linkage_list
211def process_zh_l_template(
212 wxr: WiktextractContext,
213 t_node: TemplateNode,
214 sense: str,
215 raw_tags: list[str] = [],
216) -> list[Linkage]:
217 # https://zh.wiktionary.org/wiki/Template:Zh-l
218 expanded_node = wxr.wtp.parse(
219 wxr.wtp.node_to_wikitext(t_node), expand_all=True
220 )
221 roman = ""
222 linkage_list = []
223 new_sense = clean_node(wxr, None, t_node.template_parameters.get(2, ""))
224 if new_sense != "": 224 ↛ 225line 224 didn't jump to line 225 because the condition on line 224 was never true
225 sense = new_sense
226 for i_tag in expanded_node.find_html_recursively(
227 "span", attr_name="class", attr_value="Latn"
228 ):
229 roman = clean_node(wxr, None, i_tag)
230 for span_tag in expanded_node.find_html(
231 "span", attr_name="lang", attr_value="zh"
232 ):
233 linkage_data = Linkage(
234 sense=sense,
235 raw_tags=raw_tags,
236 roman=roman,
237 word=clean_node(wxr, None, span_tag),
238 )
239 lang_attr = span_tag.attrs.get("lang", "")
240 if lang_attr == "zh-Hant":
241 linkage_data.tags.append("Traditional-Chinese")
242 elif lang_attr == "zh-Hans":
243 linkage_data.tags.append("Simplified-Chinese")
244 if linkage_data.word not in ["／", ""]:
245 translate_raw_tags(linkage_data)
246 linkage_list.append(linkage_data)
247 return linkage_list
250def process_ja_r_template(
251 wxr: WiktextractContext,
252 template_node: TemplateNode,
253 sense: str,
254 raw_tags: list[str] = [],
255) -> Linkage:
256 # https://zh.wiktionary.org/wiki/Template:Ja-r
257 expanded_node = wxr.wtp.parse(
258 wxr.wtp.node_to_wikitext(template_node), expand_all=True
259 )
260 return process_expanded_ja_r_node(wxr, expanded_node, sense, raw_tags)
263def process_expanded_ja_r_node(
264 wxr: WiktextractContext,
265 expanded_node: WikiNode,
266 sense: str,
267 raw_tags: list[str] = [],
268) -> Linkage:
269 linkage_data = Linkage(sense=sense, raw_tags=raw_tags)
270 for span_node in expanded_node.find_html("span"):
271 span_class = span_node.attrs.get("class", "")
272 if "lang" in span_node.attrs:
273 ruby_data, no_ruby_nodes = extract_ruby(wxr, span_node)
274 linkage_data.word = clean_node(wxr, None, no_ruby_nodes)
275 linkage_data.ruby = ruby_data
276 elif "tr" in span_class:
277 linkage_data.roman = clean_node(wxr, None, span_node)
278 elif "mention-gloss" == span_class: 278 ↛ 279line 278 didn't jump to line 279 because the condition on line 278 was never true
279 linkage_data.sense = clean_node(wxr, None, span_node)
281 translate_raw_tags(linkage_data)
282 return linkage_data
285def process_l_template(
286 wxr: WiktextractContext,
287 t_node: TemplateNode,
288 sense: str,
289 raw_tags: list[str] = [],
290) -> None:
291 # https://zh.wiktionary.org/wiki/Template:l
292 expanded_node = wxr.wtp.parse(
293 wxr.wtp.node_to_wikitext(t_node), expand_all=True
294 )
295 linkage_list = []
296 lang_code = clean_node(wxr, None, t_node.template_parameters.get(1, ""))
297 for span_tag in expanded_node.find_html("span"):
298 span_lang = span_tag.attrs.get("lang", "")
299 span_class = span_tag.attrs.get("class", "")
300 if span_lang == lang_code:
301 linkage_data = Linkage(
302 sense=sense,
303 raw_tags=raw_tags,
304 word=clean_node(wxr, None, span_tag),
305 )
306 if len(linkage_data.word) > 0: 306 ↛ 297line 306 didn't jump to line 297 because the condition on line 306 was always true
307 translate_raw_tags(linkage_data)
308 linkage_list.append(linkage_data)
309 elif span_lang.endswith("-Latn") and len(linkage_list) > 0:
310 linkage_list[-1].roman = clean_node(wxr, None, span_tag)
311 elif "mention-gloss" == span_class and len(linkage_list) > 0:
312 linkage_list[-1].sense = clean_node(wxr, None, span_tag)
314 return linkage_list
317def process_linkage_col_template(
318 wxr: WiktextractContext, template_node: TemplateNode, sense: str
319) -> list[Linkage]:
320 # https://zh.wiktionary.org/wiki/Template:Col3
321 linkage_list = []
322 expanded_template = wxr.wtp.parse(
323 wxr.wtp.node_to_wikitext(template_node), expand_all=True
324 )
325 for ui_tag in expanded_template.find_html_recursively("li"):
326 current_data = []
327 roman = ""
328 raw_tags = []
329 for span_tag in ui_tag.find_html("span"):
330 span_lang = span_tag.attrs.get("lang", "")
331 if span_lang.endswith("-Latn"): 331 ↛ 332line 331 didn't jump to line 332 because the condition on line 331 was never true
332 roman = clean_node(wxr, None, span_tag)
333 elif "qualifier-content" in span_tag.attrs.get("class", ""):
334 span_text = clean_node(wxr, None, span_tag)
335 for raw_tag in re.split(r"或|、", span_text):
336 raw_tag = raw_tag.strip()
337 if raw_tag != "": 337 ↛ 335line 337 didn't jump to line 335 because the condition on line 337 was always true
338 raw_tags.append(raw_tag)
339 elif span_lang != "":
340 l_data = Linkage(
341 word=clean_node(wxr, None, span_tag), sense=sense
342 )
343 class_names = span_tag.attrs.get("class", "")
344 if class_names == "Hant": 344 ↛ 345line 344 didn't jump to line 345 because the condition on line 344 was never true
345 l_data.tags.append("Traditional-Chinese")
346 elif class_names == "Hans": 346 ↛ 347line 346 didn't jump to line 347 because the condition on line 346 was never true
347 l_data.tags.append("Simplified-Chinese")
348 if l_data.word != "": 348 ↛ 329line 348 didn't jump to line 329 because the condition on line 348 was always true
349 current_data.append(l_data)
351 for data in current_data:
352 data.raw_tags.extend(raw_tags)
353 data.roman = roman
354 translate_raw_tags(data)
355 linkage_list.extend(current_data)
357 return linkage_list
360def process_linkage_templates_in_gloss(
361 wxr: WiktextractContext,
362 word_entry: WordEntry,
363 t_node: TemplateNode,
364 linkage_type: str,
365 sense: str,
366) -> None:
367 # https://en.wiktionary.org/wiki/Template:synonyms
368 expanded_node = wxr.wtp.parse(
369 wxr.wtp.node_to_wikitext(t_node), expand_all=True
370 )
371 lang_code = clean_node(wxr, None, t_node.template_parameters.get(1, ""))
372 l_list = []
373 raw_tags = []
374 for top_span_tag in expanded_node.find_html("span"):
375 for node in top_span_tag.children:
376 if isinstance(node, HTMLNode) and node.tag == "span":
377 span_lang = node.attrs.get("lang", "")
378 span_class = node.attrs.get("class", "")
379 if span_lang == lang_code:
380 l_data = Linkage(
381 word=clean_node(wxr, None, node),
382 sense=sense,
383 raw_tags=raw_tags,
384 )
385 if span_class == "Hant":
386 l_data.tags.append("Traditional-Chinese")
387 elif span_class == "Hans":
388 l_data.tags.append("Simplified-Chinese")
389 if l_data.word != "": 389 ↛ 375line 389 didn't jump to line 375 because the condition on line 389 was always true
390 l_list.append(l_data)
391 elif span_lang == f"{lang_code}-Latn" or "tr" in span_class:
392 roman = clean_node(wxr, None, node)
393 for d in l_list:
394 d.roman = roman
395 elif span_class == "mention-gloss": 395 ↛ 396line 395 didn't jump to line 396 because the condition on line 395 was never true
396 sense = clean_node(wxr, None, node)
397 for d in l_list:
398 d.sense = sense
399 elif "qualifier-content" in span_class:
400 raw_tag_str = clean_node(wxr, None, node)
401 for raw_tag in raw_tag_str.split("，"):
402 raw_tag = raw_tag.strip()
403 if raw_tag != "": 403 ↛ 401line 403 didn't jump to line 401 because the condition on line 403 was always true
404 raw_tags.append(raw_tag)
405 elif isinstance(node, str) and node.strip() == "、":
406 getattr(word_entry, linkage_type).extend(l_list)
407 l_list.clear()
409 getattr(word_entry, linkage_type).extend(l_list)
410 for data in getattr(word_entry, linkage_type):
411 translate_raw_tags(data)
414def extract_ja_r_multi_template(
415 wxr: WiktextractContext, template_node: TemplateNode, sense: str
416) -> Linkage:
417 expanded_node = wxr.wtp.parse(
418 wxr.wtp.node_to_wikitext(template_node), expand_all=True
419 )
420 linkage_list = []
421 for list_node in expanded_node.find_child(NodeKind.LIST):
422 for list_item in list_node.find_child(NodeKind.LIST_ITEM):
423 linkage_list.append(
424 process_expanded_ja_r_node(wxr, list_item, sense, [])
425 )
427 return linkage_list