Coverage for src/wiktextract/extractor/zh/page.py: 87%
234 statements
« prev ^ index » next coverage.py v7.10.6, created at 2025-09-18 10:14 +0000
1import re
2from typing import Any
4from mediawiki_langcodes import name_to_code
5from wikitextprocessor.parser import (
6 LEVEL_KIND_FLAGS,
7 HTMLNode,
8 LevelNode,
9 NodeKind,
10 TemplateNode,
11 WikiNode,
12)
14from ...page import clean_node
15from ...wxr_context import WiktextractContext
16from ...wxr_logging import logger
17from .descendant import extract_descendant_section
18from .etymology import extract_etymology_section
19from .gloss import extract_gloss
20from .headword_line import extract_pos_head_line_nodes
21from .inflection import extract_inflections
22from .linkage import extract_linkage_section
23from .models import Form, Linkage, Sense, WordEntry
24from .note import extract_note_section
25from .pronunciation import extract_pronunciation_section
26from .section_titles import (
27 DESCENDANTS_TITLES,
28 ETYMOLOGY_TITLES,
29 IGNORED_TITLES,
30 INFLECTION_TITLES,
31 LINKAGE_TITLES,
32 POS_TITLES,
33 PRONUNCIATION_TITLES,
34 TRANSLATIONS_TITLES,
35 USAGE_NOTE_TITLES,
36)
37from .tags import translate_raw_tags
38from .translation import extract_translation_section
41def parse_section(
42 wxr: WiktextractContext,
43 page_data: list[WordEntry],
44 base_data: WordEntry,
45 level_node: LevelNode,
46) -> None:
47 subtitle = clean_node(wxr, None, level_node.largs)
48 # remove number suffix from subtitle
49 subtitle = re.sub(r"\s*(?:（.+）|\d+)$", "", subtitle)
50 wxr.wtp.start_subsection(subtitle)
51 if subtitle in IGNORED_TITLES: 51 ↛ 52line 51 didn't jump to line 52 because the condition on line 51 was never true
52 pass
53 elif subtitle in POS_TITLES:
54 process_pos_block(wxr, page_data, base_data, level_node, subtitle)
55 if len(page_data[-1].senses) == 0 and subtitle in LINKAGE_TITLES:
56 page_data.pop()
57 extract_linkage_section(
58 wxr,
59 page_data if len(page_data) > 0 else [base_data],
60 level_node,
61 LINKAGE_TITLES[subtitle],
62 )
63 elif wxr.config.capture_etymologies and subtitle.startswith(
64 tuple(ETYMOLOGY_TITLES)
65 ):
66 if level_node.contain_node(LEVEL_KIND_FLAGS): 66 ↛ 68line 66 didn't jump to line 68 because the condition on line 66 was always true
67 base_data = base_data.model_copy(deep=True)
68 extract_etymology_section(wxr, page_data, base_data, level_node)
69 elif wxr.config.capture_pronunciation and subtitle in PRONUNCIATION_TITLES:
70 if level_node.contain_node(LEVEL_KIND_FLAGS):
71 base_data = base_data.model_copy(deep=True)
72 extract_pronunciation_section(wxr, base_data, level_node)
73 elif wxr.config.capture_linkages and subtitle in LINKAGE_TITLES:
74 is_descendant_section = False
75 if subtitle in DESCENDANTS_TITLES:
76 for t_node in level_node.find_child_recursively(NodeKind.TEMPLATE): 76 ↛ 86line 76 didn't jump to line 86 because the loop on line 76 didn't complete
77 if t_node.template_name.lower() in [ 77 ↛ 76line 77 didn't jump to line 76 because the condition on line 77 was always true
78 "desc",
79 "descendant",
80 "desctree",
81 "descendants tree",
82 "cjkv",
83 ]:
84 is_descendant_section = True
85 break
86 if is_descendant_section and wxr.config.capture_descendants:
87 extract_descendant_section(
88 wxr,
89 level_node,
90 page_data if len(page_data) > 0 else [base_data],
91 )
92 elif not is_descendant_section: 92 ↛ 121line 92 didn't jump to line 121 because the condition on line 92 was always true
93 extract_linkage_section(
94 wxr,
95 page_data if len(page_data) > 0 else [base_data],
96 level_node,
97 LINKAGE_TITLES[subtitle],
98 )
99 elif wxr.config.capture_translations and subtitle in TRANSLATIONS_TITLES:
100 if len(page_data) == 0: 100 ↛ 101line 100 didn't jump to line 101 because the condition on line 100 was never true
101 page_data.append(base_data.model_copy(deep=True))
102 extract_translation_section(wxr, page_data[-1], level_node)
103 elif wxr.config.capture_inflections and subtitle in INFLECTION_TITLES: 103 ↛ 104line 103 didn't jump to line 104 because the condition on line 103 was never true
104 extract_inflections(
105 wxr, page_data if len(page_data) > 0 else [base_data], level_node
106 )
107 elif wxr.config.capture_descendants and subtitle in DESCENDANTS_TITLES:
108 extract_descendant_section(
109 wxr, level_node, page_data if len(page_data) > 0 else [base_data]
110 )
111 elif subtitle in USAGE_NOTE_TITLES: 111 ↛ 116line 111 didn't jump to line 116 because the condition on line 111 was always true
112 extract_note_section(
113 wxr, page_data[-1] if len(page_data) > 0 else base_data, level_node
114 )
115 else:
116 wxr.wtp.debug(
117 f"Unhandled subtitle: {subtitle}",
118 sortid="extractor/zh/page/parse_section/192",
119 )
121 for next_level_node in level_node.find_child(LEVEL_KIND_FLAGS):
122 parse_section(wxr, page_data, base_data, next_level_node)
124 for template in level_node.find_child(NodeKind.TEMPLATE):
125 add_page_end_categories(
126 wxr, page_data if len(page_data) else [base_data], template
127 )
130def process_pos_block(
131 wxr: WiktextractContext,
132 page_data: list[WordEntry],
133 base_data: WordEntry,
134 level_node: LevelNode,
135 pos_title: str,
136):
137 pos_data = POS_TITLES[pos_title]
138 pos_type = pos_data["pos"]
139 base_data.pos = pos_type
140 page_data.append(base_data.model_copy(deep=True))
141 page_data[-1].pos_title = pos_title
142 page_data[-1].pos_level = level_node.kind
143 page_data[-1].tags.extend(pos_data.get("tags", []))
144 first_gloss_list_index = len(level_node.children)
145 for index, child in enumerate(level_node.children):
146 if (
147 isinstance(child, WikiNode)
148 and child.kind == NodeKind.LIST
149 and child.sarg.startswith("#")
150 ):
151 if index < first_gloss_list_index: 151 ↛ 153line 151 didn't jump to line 153 because the condition on line 151 was always true
152 first_gloss_list_index = index
153 extract_gloss(wxr, page_data, child, Sense())
155 extract_pos_head_line_nodes(
156 wxr, page_data[-1], level_node.children[:first_gloss_list_index]
157 )
159 if len(page_data[-1].senses) == 0 and not level_node.contain_node(
160 NodeKind.LIST
161 ):
162 # low quality pages don't put gloss in list
163 expanded_node = wxr.wtp.parse(
164 wxr.wtp.node_to_wikitext(
165 list(
166 level_node.invert_find_child(
167 LEVEL_KIND_FLAGS, include_empty_str=True
168 )
169 )
170 ),
171 expand_all=True,
172 )
173 if not expanded_node.contain_node(NodeKind.LIST):
174 gloss_text = clean_node(
175 wxr,
176 page_data[-1],
177 expanded_node,
178 )
179 if len(gloss_text) > 0: 179 ↛ 182line 179 didn't jump to line 182 because the condition on line 179 was always true
180 page_data[-1].senses.append(Sense(glosses=[gloss_text]))
181 else:
182 page_data[-1].senses.append(Sense(tags=["no-gloss"]))
185def parse_page(
186 wxr: WiktextractContext, page_title: str, page_text: str
187) -> list[dict[str, Any]]:
188 # page layout documents
189 # https://zh.wiktionary.org/wiki/Wiktionary:佈局解釋
190 # https://zh.wiktionary.org/wiki/Wiktionary:体例说明
191 # https://zh.wiktionary.org/wiki/Wiktionary:格式手冊
193 # skip translation pages
194 if page_title.endswith( 194 ↛ 197line 194 didn't jump to line 197 because the condition on line 194 was never true
195 tuple("/" + tr_title for tr_title in TRANSLATIONS_TITLES) + ("/衍生詞",)
196 ):
197 return []
199 if wxr.config.verbose: 199 ↛ 200line 199 didn't jump to line 200 because the condition on line 199 was never true
200 logger.info(f"Parsing page: {page_title}")
201 wxr.config.word = page_title
202 wxr.wtp.start_page(page_title)
204 # Parse the page, pre-expanding those templates that are likely to
205 # influence parsing
206 tree = wxr.wtp.parse(page_text, pre_expand=True)
208 page_data = []
209 for level2_node in tree.find_child(NodeKind.LEVEL2):
210 categories = {}
211 lang_name = clean_node(wxr, categories, level2_node.largs)
212 lang_code = name_to_code(lang_name, "zh")
213 if lang_code == "": 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true
214 wxr.wtp.warning(
215 f"Unrecognized language name: {lang_name}",
216 sortid="extractor/zh/page/parse_page/509",
217 )
218 lang_code = "unknown"
219 if ( 219 ↛ 223line 219 didn't jump to line 223 because the condition on line 219 was never true
220 wxr.config.capture_language_codes is not None
221 and lang_code not in wxr.config.capture_language_codes
222 ):
223 continue
224 wxr.wtp.start_section(lang_name)
225 base_data = WordEntry(
226 word=wxr.wtp.title,
227 lang_code=lang_code,
228 lang=lang_name,
229 pos="unknown",
230 )
231 base_data.categories = categories.get("categories", [])
232 for template_node in level2_node.find_child(NodeKind.TEMPLATE):
233 if template_node.template_name == "zh-forms":
234 process_zh_forms(wxr, base_data, template_node)
236 for level3_node in level2_node.find_child(NodeKind.LEVEL3):
237 parse_section(wxr, page_data, base_data, level3_node)
238 if not level2_node.contain_node(NodeKind.LEVEL3):
239 page_data.append(base_data.model_copy(deep=True))
240 process_low_quality_page(wxr, level2_node, page_data[-1])
241 if page_data[-1] == base_data: 241 ↛ 242line 241 didn't jump to line 242 because the condition on line 241 was never true
242 page_data.pop()
244 for data in page_data:
245 if len(data.senses) == 0:
246 data.senses.append(Sense(tags=["no-gloss"]))
248 return [d.model_dump(exclude_defaults=True) for d in page_data]
251def process_low_quality_page(
252 wxr: WiktextractContext, level_node: WikiNode, word_entry: WordEntry
253) -> None:
254 is_soft_redirect = False
255 for template_node in level_node.find_child(NodeKind.TEMPLATE):
256 if template_node.template_name in ("ja-see", "ja-see-kango", "zh-see"):
257 process_soft_redirect_template(wxr, template_node, word_entry)
258 is_soft_redirect = True
260 if not is_soft_redirect: # only have a gloss text
261 has_gloss_list = False
262 for list_node in level_node.find_child(NodeKind.LIST): 262 ↛ 263line 262 didn't jump to line 263 because the loop on line 262 never started
263 if list_node.sarg == "#":
264 extract_gloss(wxr, [word_entry], list_node, Sense())
265 has_gloss_list = True
266 if not has_gloss_list: 266 ↛ exitline 266 didn't return from function 'process_low_quality_page' because the condition on line 266 was always true
267 gloss_text = clean_node(wxr, word_entry, level_node.children)
268 if len(gloss_text) > 0: 268 ↛ exitline 268 didn't return from function 'process_low_quality_page' because the condition on line 268 was always true
269 for cat in word_entry.categories:
270 cat = cat.removeprefix(word_entry.lang).strip()
271 if cat in POS_TITLES: 271 ↛ 269line 271 didn't jump to line 269 because the condition on line 271 was always true
272 pos_data = POS_TITLES[cat]
273 word_entry.pos = pos_data["pos"]
274 word_entry.tags.extend(pos_data.get("tags", []))
275 break
276 word_entry.senses.append(Sense(glosses=[gloss_text]))
279def process_soft_redirect_template(
280 wxr: WiktextractContext, t_node: TemplateNode, word_entry: WordEntry
281) -> None:
282 # https://zh.wiktionary.org/wiki/Template:Ja-see
283 # https://zh.wiktionary.org/wiki/Template:Ja-see-kango
284 # https://zh.wiktionary.org/wiki/Template:Zh-see
285 template_name = t_node.template_name.lower()
286 if template_name == "zh-see":
287 word_entry.redirects.append(
288 clean_node(wxr, None, t_node.template_parameters.get(1, ""))
289 )
290 elif template_name in ("ja-see", "ja-see-kango"): 290 ↛ 295line 290 didn't jump to line 295 because the condition on line 290 was always true
291 for key, value in t_node.template_parameters.items():
292 if isinstance(key, int): 292 ↛ 291line 292 didn't jump to line 291 because the condition on line 292 was always true
293 word_entry.redirects.append(clean_node(wxr, None, value))
295 if word_entry.pos == "unknown": 295 ↛ exitline 295 didn't return from function 'process_soft_redirect_template' because the condition on line 295 was always true
296 word_entry.pos = "soft-redirect"
299def process_zh_forms(
300 wxr: WiktextractContext, base_data: WordEntry, t_node: TemplateNode
301):
302 # https://zh.wiktionary.org/wiki/Template:zh-forms
303 base_data.literal_meaning = clean_node(
304 wxr, None, t_node.template_parameters.get("lit", "")
305 )
306 expanded_node = wxr.wtp.parse(
307 wxr.wtp.node_to_wikitext(t_node), expand_all=True
308 )
309 for table in expanded_node.find_child(NodeKind.TABLE):
310 for row in table.find_child(NodeKind.TABLE_ROW):
311 row_header = ""
312 row_header_tags = []
313 header_has_span = False
314 for cell in row.find_child(
315 NodeKind.TABLE_HEADER_CELL | NodeKind.TABLE_CELL
316 ):
317 if cell.kind == NodeKind.TABLE_HEADER_CELL:
318 row_header, row_header_tags, header_has_span = (
319 extract_zh_forms_header_cell(wxr, base_data, cell)
320 )
321 elif not header_has_span:
322 extract_zh_forms_data_cell(
323 wxr, base_data, cell, row_header, row_header_tags
324 )
327def extract_zh_forms_header_cell(
328 wxr: WiktextractContext, base_data: WordEntry, header_cell: WikiNode
329) -> tuple[str, list[str], bool]:
330 row_header = ""
331 row_header_tags = []
332 header_has_span = False
333 first_span_index = len(header_cell.children)
334 for index, span_tag in header_cell.find_html("span", with_index=True):
335 if index < first_span_index: 335 ↛ 337line 335 didn't jump to line 337 because the condition on line 335 was always true
336 first_span_index = index
337 header_has_span = True
338 row_header = clean_node(wxr, None, header_cell.children[:first_span_index])
339 for raw_tag in re.split(r"/|與", row_header):
340 raw_tag = raw_tag.strip()
341 if raw_tag != "":
342 row_header_tags.append(raw_tag)
343 for span_tag in header_cell.find_html_recursively("span"):
344 span_lang = span_tag.attrs.get("lang", "")
345 form_nodes = []
346 sup_title = ""
347 for node in span_tag.children:
348 if isinstance(node, HTMLNode) and node.tag == "sup":
349 for sup_span in node.find_html("span"):
350 sup_title = sup_span.attrs.get("title", "")
351 else:
352 form_nodes.append(node)
353 if span_lang in ["zh-Hant", "zh-Hans"]:
354 for word in clean_node(wxr, None, form_nodes).split("/"):
355 if word not in [base_data.word, ""]:
356 form = Form(form=word, raw_tags=row_header_tags)
357 if sup_title != "":
358 form.raw_tags.append(sup_title)
359 translate_raw_tags(form)
360 base_data.forms.append(form)
361 return row_header, row_header_tags, header_has_span
364def extract_zh_forms_data_cell(
365 wxr: WiktextractContext,
366 base_data: WordEntry,
367 cell: WikiNode,
368 row_header: str,
369 row_header_tags: list[str],
370):
371 for top_span_tag in cell.find_html("span"):
372 forms = []
373 for span_tag in top_span_tag.find_html("span"):
374 span_lang = span_tag.attrs.get("lang", "")
375 if span_lang in ["zh-Hant", "zh-Hans", "zh"]:
376 word = clean_node(wxr, None, span_tag)
377 if word not in ["", "／", base_data.word]:
378 form = Form(form=word)
379 if row_header != "異序詞":
380 form.raw_tags = row_header_tags
381 if span_lang == "zh-Hant":
382 form.tags.append("Traditional-Chinese")
383 elif span_lang == "zh-Hans":
384 form.tags.append("Simplified-Chinese")
385 translate_raw_tags(form)
386 forms.append(form)
387 elif "font-size:80%" in span_tag.attrs.get("style", ""): 387 ↛ 373line 387 didn't jump to line 373 because the condition on line 387 was always true
388 raw_tag = clean_node(wxr, None, span_tag)
389 if raw_tag != "": 389 ↛ 373line 389 didn't jump to line 373 because the condition on line 389 was always true
390 for form in forms:
391 form.raw_tags.append(raw_tag)
392 translate_raw_tags(form)
393 if row_header == "異序詞":
394 for form in forms:
395 base_data.anagrams.append(
396 Linkage(
397 word=form.form,
398 raw_tags=form.raw_tags,
399 tags=form.tags,
400 )
401 )
402 else:
403 base_data.forms.extend(forms)
406# https://zh.wiktionary.org/wiki/Template:Zh-cat
407# https://zh.wiktionary.org/wiki/Template:Catlangname
408CATEGORY_TEMPLATES = frozenset(
409 [
410 "zh-cat",
411 "cln",
412 "catlangname",
413 "c",
414 "topics",
415 "top",
416 "catlangcode",
417 "topic",
418 ]
419)
422def add_page_end_categories(
423 wxr: WiktextractContext, page_data: list[WordEntry], template: TemplateNode
424) -> None:
425 if template.template_name.lower() in CATEGORY_TEMPLATES: 425 ↛ 426line 425 didn't jump to line 426 because the condition on line 425 was never true
426 categories = {}
427 clean_node(wxr, categories, template)
428 for data in page_data:
429 if data.lang_code == page_data[-1].lang_code:
430 data.categories.extend(categories.get("categories", []))