Coverage for src/wiktextract/extractor/zh/page.py: 86%
234 statements
« prev ^ index » next coverage.py v7.10.3, created at 2025-08-15 05:18 +0000
1import re
2from typing import Any
4from mediawiki_langcodes import name_to_code
5from wikitextprocessor.parser import (
6 LEVEL_KIND_FLAGS,
7 HTMLNode,
8 LevelNode,
9 NodeKind,
10 TemplateNode,
11 WikiNode,
12)
14from ...page import clean_node
15from ...wxr_context import WiktextractContext
16from ...wxr_logging import logger
17from .descendant import extract_descendant_section
18from .etymology import extract_etymology_section
19from .gloss import extract_gloss
20from .headword_line import extract_pos_head_line_nodes
21from .inflection import extract_inflections
22from .linkage import extract_linkage_section
23from .models import Form, Linkage, Sense, WordEntry
24from .note import extract_note_section
25from .pronunciation import extract_pronunciation_section
26from .section_titles import (
27 DESCENDANTS_TITLES,
28 ETYMOLOGY_TITLES,
29 IGNORED_TITLES,
30 INFLECTION_TITLES,
31 LINKAGE_TITLES,
32 POS_TITLES,
33 PRONUNCIATION_TITLES,
34 TRANSLATIONS_TITLES,
35 USAGE_NOTE_TITLES,
36)
37from .tags import translate_raw_tags
38from .translation import extract_translation
41def parse_section(
42 wxr: WiktextractContext,
43 page_data: list[WordEntry],
44 base_data: WordEntry,
45 level_node: LevelNode,
46) -> None:
47 subtitle = clean_node(wxr, None, level_node.largs)
48 # remove number suffix from subtitle
49 subtitle = re.sub(r"\s*(?:（.+）|\d+)$", "", subtitle)
50 wxr.wtp.start_subsection(subtitle)
51 if subtitle in IGNORED_TITLES: 51 ↛ 52line 51 didn't jump to line 52 because the condition on line 51 was never true
52 pass
53 elif subtitle in POS_TITLES:
54 process_pos_block(wxr, page_data, base_data, level_node, subtitle)
55 if len(page_data[-1].senses) == 0 and subtitle in LINKAGE_TITLES:
56 page_data.pop()
57 extract_linkage_section(
58 wxr,
59 page_data if len(page_data) > 0 else [base_data],
60 level_node,
61 LINKAGE_TITLES[subtitle],
62 )
63 elif wxr.config.capture_etymologies and subtitle.startswith(
64 tuple(ETYMOLOGY_TITLES)
65 ):
66 if level_node.contain_node(LEVEL_KIND_FLAGS): 66 ↛ 68line 66 didn't jump to line 68 because the condition on line 66 was always true
67 base_data = base_data.model_copy(deep=True)
68 extract_etymology_section(wxr, page_data, base_data, level_node)
69 elif wxr.config.capture_pronunciation and subtitle in PRONUNCIATION_TITLES:
70 if level_node.contain_node(LEVEL_KIND_FLAGS):
71 base_data = base_data.model_copy(deep=True)
72 extract_pronunciation_section(wxr, base_data, level_node)
73 elif wxr.config.capture_linkages and subtitle in LINKAGE_TITLES:
74 is_descendant_section = False
75 if subtitle in DESCENDANTS_TITLES:
76 for t_node in level_node.find_child_recursively(NodeKind.TEMPLATE): 76 ↛ 86line 76 didn't jump to line 86 because the loop on line 76 didn't complete
77 if t_node.template_name.lower() in [ 77 ↛ 76line 77 didn't jump to line 76 because the condition on line 77 was always true
78 "desc",
79 "descendant",
80 "desctree",
81 "descendants tree",
82 "cjkv",
83 ]:
84 is_descendant_section = True
85 break
86 if is_descendant_section and wxr.config.capture_descendants:
87 extract_descendant_section(
88 wxr,
89 level_node,
90 page_data if len(page_data) > 0 else [base_data],
91 )
92 elif not is_descendant_section: 92 ↛ 121line 92 didn't jump to line 121 because the condition on line 92 was always true
93 extract_linkage_section(
94 wxr,
95 page_data if len(page_data) > 0 else [base_data],
96 level_node,
97 LINKAGE_TITLES[subtitle],
98 )
99 elif wxr.config.capture_translations and subtitle in TRANSLATIONS_TITLES: 99 ↛ 100line 99 didn't jump to line 100 because the condition on line 99 was never true
100 if len(page_data) == 0:
101 page_data.append(base_data.model_copy(deep=True))
102 extract_translation(wxr, page_data, level_node)
103 elif wxr.config.capture_inflections and subtitle in INFLECTION_TITLES: 103 ↛ 104line 103 didn't jump to line 104 because the condition on line 103 was never true
104 extract_inflections(
105 wxr, page_data if len(page_data) > 0 else [base_data], level_node
106 )
107 elif wxr.config.capture_descendants and subtitle in DESCENDANTS_TITLES: 107 ↛ 108line 107 didn't jump to line 108 because the condition on line 107 was never true
108 extract_descendant_section(
109 wxr, level_node, page_data if len(page_data) > 0 else [base_data]
110 )
111 elif subtitle in USAGE_NOTE_TITLES: 111 ↛ 116line 111 didn't jump to line 116 because the condition on line 111 was always true
112 extract_note_section(
113 wxr, page_data[-1] if len(page_data) > 0 else base_data, level_node
114 )
115 else:
116 wxr.wtp.debug(
117 f"Unhandled subtitle: {subtitle}",
118 sortid="extractor/zh/page/parse_section/192",
119 )
121 for next_level_node in level_node.find_child(LEVEL_KIND_FLAGS):
122 parse_section(wxr, page_data, base_data, next_level_node)
124 for template in level_node.find_child(NodeKind.TEMPLATE):
125 add_page_end_categories(wxr, page_data, template)
128def process_pos_block(
129 wxr: WiktextractContext,
130 page_data: list[WordEntry],
131 base_data: WordEntry,
132 level_node: LevelNode,
133 pos_title: str,
134):
135 pos_data = POS_TITLES[pos_title]
136 pos_type = pos_data["pos"]
137 base_data.pos = pos_type
138 page_data.append(base_data.model_copy(deep=True))
139 page_data[-1].pos_title = pos_title
140 page_data[-1].pos_level = level_node.kind
141 page_data[-1].tags.extend(pos_data.get("tags", []))
142 first_gloss_list_index = len(level_node.children)
143 for index, child in enumerate(level_node.children):
144 if (
145 isinstance(child, WikiNode)
146 and child.kind == NodeKind.LIST
147 and child.sarg.startswith("#")
148 ):
149 if index < first_gloss_list_index: 149 ↛ 151line 149 didn't jump to line 151 because the condition on line 149 was always true
150 first_gloss_list_index = index
151 extract_gloss(wxr, page_data, child, Sense())
153 extract_pos_head_line_nodes(
154 wxr, page_data[-1], level_node.children[:first_gloss_list_index]
155 )
157 if len(page_data[-1].senses) == 0 and not level_node.contain_node(
158 NodeKind.LIST
159 ):
160 # low quality pages don't put gloss in list
161 expanded_node = wxr.wtp.parse(
162 wxr.wtp.node_to_wikitext(
163 list(level_node.invert_find_child(LEVEL_KIND_FLAGS))
164 ),
165 expand_all=True,
166 )
167 if not expanded_node.contain_node(NodeKind.LIST):
168 gloss_text = clean_node(
169 wxr,
170 page_data[-1],
171 expanded_node,
172 )
173 if len(gloss_text) > 0: 173 ↛ 176line 173 didn't jump to line 176 because the condition on line 173 was always true
174 page_data[-1].senses.append(Sense(glosses=[gloss_text]))
175 else:
176 page_data[-1].senses.append(Sense(tags=["no-gloss"]))
179def parse_page(
180 wxr: WiktextractContext, page_title: str, page_text: str
181) -> list[dict[str, Any]]:
182 # page layout documents
183 # https://zh.wiktionary.org/wiki/Wiktionary:佈局解釋
184 # https://zh.wiktionary.org/wiki/Wiktionary:体例说明
185 # https://zh.wiktionary.org/wiki/Wiktionary:格式手冊
187 # skip translation pages
188 if page_title.endswith( 188 ↛ 191line 188 didn't jump to line 191 because the condition on line 188 was never true
189 tuple("/" + tr_title for tr_title in TRANSLATIONS_TITLES) + ("/衍生詞",)
190 ):
191 return []
193 if wxr.config.verbose: 193 ↛ 194line 193 didn't jump to line 194 because the condition on line 193 was never true
194 logger.info(f"Parsing page: {page_title}")
195 wxr.config.word = page_title
196 wxr.wtp.start_page(page_title)
198 # Parse the page, pre-expanding those templates that are likely to
199 # influence parsing
200 tree = wxr.wtp.parse(page_text, pre_expand=True)
202 page_data = []
203 for level2_node in tree.find_child(NodeKind.LEVEL2):
204 categories = {}
205 lang_name = clean_node(wxr, categories, level2_node.largs)
206 lang_code = name_to_code(lang_name, "zh")
207 if lang_code == "": 207 ↛ 208line 207 didn't jump to line 208 because the condition on line 207 was never true
208 wxr.wtp.warning(
209 f"Unrecognized language name: {lang_name}",
210 sortid="extractor/zh/page/parse_page/509",
211 )
212 lang_code = "unknown"
213 if ( 213 ↛ 217line 213 didn't jump to line 217 because the condition on line 213 was never true
214 wxr.config.capture_language_codes is not None
215 and lang_code not in wxr.config.capture_language_codes
216 ):
217 continue
218 wxr.wtp.start_section(lang_name)
219 base_data = WordEntry(
220 word=wxr.wtp.title,
221 lang_code=lang_code,
222 lang=lang_name,
223 pos="unknown",
224 )
225 base_data.categories = categories.get("categories", [])
226 for template_node in level2_node.find_child(NodeKind.TEMPLATE):
227 if template_node.template_name == "zh-forms":
228 process_zh_forms(wxr, base_data, template_node)
230 for level3_node in level2_node.find_child(NodeKind.LEVEL3):
231 parse_section(wxr, page_data, base_data, level3_node)
232 if not level2_node.contain_node(NodeKind.LEVEL3):
233 page_data.append(base_data.model_copy(deep=True))
234 process_low_quality_page(wxr, level2_node, page_data[-1])
235 if page_data[-1] == base_data: 235 ↛ 236line 235 didn't jump to line 236 because the condition on line 235 was never true
236 page_data.pop()
238 for data in page_data:
239 if len(data.senses) == 0:
240 data.senses.append(Sense(tags=["no-gloss"]))
242 return [d.model_dump(exclude_defaults=True) for d in page_data]
245def process_low_quality_page(
246 wxr: WiktextractContext, level_node: WikiNode, word_entry: WordEntry
247) -> None:
248 is_soft_redirect = False
249 for template_node in level_node.find_child(NodeKind.TEMPLATE):
250 if template_node.template_name in ("ja-see", "ja-see-kango", "zh-see"):
251 process_soft_redirect_template(wxr, template_node, word_entry)
252 is_soft_redirect = True
254 if not is_soft_redirect: # only have a gloss text
255 has_gloss_list = False
256 for list_node in level_node.find_child(NodeKind.LIST): 256 ↛ 257line 256 didn't jump to line 257 because the loop on line 256 never started
257 if list_node.sarg == "#":
258 extract_gloss(wxr, [word_entry], list_node, Sense())
259 has_gloss_list = True
260 if not has_gloss_list: 260 ↛ exitline 260 didn't return from function 'process_low_quality_page' because the condition on line 260 was always true
261 gloss_text = clean_node(wxr, word_entry, level_node.children)
262 if len(gloss_text) > 0: 262 ↛ exitline 262 didn't return from function 'process_low_quality_page' because the condition on line 262 was always true
263 for cat in word_entry.categories:
264 cat = cat.removeprefix(word_entry.lang).strip()
265 if cat in POS_TITLES: 265 ↛ 263line 265 didn't jump to line 263 because the condition on line 265 was always true
266 pos_data = POS_TITLES[cat]
267 word_entry.pos = pos_data["pos"]
268 word_entry.tags.extend(pos_data.get("tags", []))
269 break
270 word_entry.senses.append(Sense(glosses=[gloss_text]))
273def process_soft_redirect_template(
274 wxr: WiktextractContext, t_node: TemplateNode, word_entry: WordEntry
275) -> None:
276 # https://zh.wiktionary.org/wiki/Template:Ja-see
277 # https://zh.wiktionary.org/wiki/Template:Ja-see-kango
278 # https://zh.wiktionary.org/wiki/Template:Zh-see
279 template_name = t_node.template_name.lower()
280 if template_name == "zh-see":
281 word_entry.redirects.append(
282 clean_node(wxr, None, t_node.template_parameters.get(1, ""))
283 )
284 elif template_name in ("ja-see", "ja-see-kango"): 284 ↛ 289line 284 didn't jump to line 289 because the condition on line 284 was always true
285 for key, value in t_node.template_parameters.items():
286 if isinstance(key, int): 286 ↛ 285line 286 didn't jump to line 285 because the condition on line 286 was always true
287 word_entry.redirects.append(clean_node(wxr, None, value))
289 if word_entry.pos == "unknown": 289 ↛ exitline 289 didn't return from function 'process_soft_redirect_template' because the condition on line 289 was always true
290 word_entry.pos = "soft-redirect"
293def process_zh_forms(
294 wxr: WiktextractContext, base_data: WordEntry, t_node: TemplateNode
295):
296 # https://zh.wiktionary.org/wiki/Template:zh-forms
297 base_data.literal_meaning = clean_node(
298 wxr, None, t_node.template_parameters.get("lit", "")
299 )
300 expanded_node = wxr.wtp.parse(
301 wxr.wtp.node_to_wikitext(t_node), expand_all=True
302 )
303 for table in expanded_node.find_child(NodeKind.TABLE):
304 for row in table.find_child(NodeKind.TABLE_ROW):
305 row_header = ""
306 row_header_tags = []
307 header_has_span = False
308 for cell in row.find_child(
309 NodeKind.TABLE_HEADER_CELL | NodeKind.TABLE_CELL
310 ):
311 if cell.kind == NodeKind.TABLE_HEADER_CELL:
312 row_header, row_header_tags, header_has_span = (
313 extract_zh_forms_header_cell(wxr, base_data, cell)
314 )
315 elif not header_has_span:
316 extract_zh_forms_data_cell(
317 wxr, base_data, cell, row_header, row_header_tags
318 )
321def extract_zh_forms_header_cell(
322 wxr: WiktextractContext, base_data: WordEntry, header_cell: WikiNode
323) -> tuple[str, list[str], bool]:
324 row_header = ""
325 row_header_tags = []
326 header_has_span = False
327 first_span_index = len(header_cell.children)
328 for index, span_tag in header_cell.find_html("span", with_index=True):
329 if index < first_span_index: 329 ↛ 331line 329 didn't jump to line 331 because the condition on line 329 was always true
330 first_span_index = index
331 header_has_span = True
332 row_header = clean_node(wxr, None, header_cell.children[:first_span_index])
333 for raw_tag in re.split(r"/|與", row_header):
334 raw_tag = raw_tag.strip()
335 if raw_tag != "":
336 row_header_tags.append(raw_tag)
337 for span_tag in header_cell.find_html_recursively("span"):
338 span_lang = span_tag.attrs.get("lang", "")
339 form_nodes = []
340 sup_title = ""
341 for node in span_tag.children:
342 if isinstance(node, HTMLNode) and node.tag == "sup":
343 for sup_span in node.find_html("span"):
344 sup_title = sup_span.attrs.get("title", "")
345 else:
346 form_nodes.append(node)
347 if span_lang in ["zh-Hant", "zh-Hans"]:
348 for word in clean_node(wxr, None, form_nodes).split("/"):
349 if word not in [base_data.word, ""]:
350 form = Form(form=word, raw_tags=row_header_tags)
351 if sup_title != "":
352 form.raw_tags.append(sup_title)
353 translate_raw_tags(form)
354 base_data.forms.append(form)
355 return row_header, row_header_tags, header_has_span
358def extract_zh_forms_data_cell(
359 wxr: WiktextractContext,
360 base_data: WordEntry,
361 cell: WikiNode,
362 row_header: str,
363 row_header_tags: list[str],
364):
365 for top_span_tag in cell.find_html("span"):
366 forms = []
367 for span_tag in top_span_tag.find_html("span"):
368 span_lang = span_tag.attrs.get("lang", "")
369 if span_lang in ["zh-Hant", "zh-Hans", "zh"]:
370 word = clean_node(wxr, None, span_tag)
371 if word not in ["", "／", base_data.word]:
372 form = Form(form=word)
373 if row_header != "異序詞":
374 form.raw_tags = row_header_tags
375 if span_lang == "zh-Hant":
376 form.tags.append("Traditional-Chinese")
377 elif span_lang == "zh-Hans":
378 form.tags.append("Simplified-Chinese")
379 translate_raw_tags(form)
380 forms.append(form)
381 elif "font-size:80%" in span_tag.attrs.get("style", ""): 381 ↛ 367line 381 didn't jump to line 367 because the condition on line 381 was always true
382 raw_tag = clean_node(wxr, None, span_tag)
383 if raw_tag != "": 383 ↛ 367line 383 didn't jump to line 367 because the condition on line 383 was always true
384 for form in forms:
385 form.raw_tags.append(raw_tag)
386 translate_raw_tags(form)
387 if row_header == "異序詞":
388 for form in forms:
389 base_data.anagrams.append(
390 Linkage(
391 word=form.form,
392 raw_tags=form.raw_tags,
393 tags=form.tags,
394 )
395 )
396 else:
397 base_data.forms.extend(forms)
400# https://zh.wiktionary.org/wiki/Template:Zh-cat
401# https://zh.wiktionary.org/wiki/Template:Catlangname
402CATEGORY_TEMPLATES = frozenset(["zh-cat", "cln", "catlangname", "c", "topics"])
405def add_page_end_categories(
406 wxr: WiktextractContext, page_data: list[WordEntry], template: TemplateNode
407) -> None:
408 if template.template_name.lower() in CATEGORY_TEMPLATES: 408 ↛ 409line 408 didn't jump to line 409 because the condition on line 408 was never true
409 categories = {}
410 clean_node(wxr, categories, template)
411 for data in page_data:
412 if data.lang_code == page_data[-1].lang_code:
413 data.categories.extend(categories.get("categories", []))