Coverage for src/wiktextract/extractor/ru/page.py: 78%
246 statements
« prev ^ index » next coverage.py v7.17.2, created at 2026-11-12 07:03 +0000
1import re
2from typing import Any
4from wikitextprocessor.parser import (
5 LEVEL_KIND_FLAGS,
6 LevelNode,
7 NodeKind,
8 TemplateNode,
9 WikiNode,
10)
12from ...config import POSSubtitleData
13from ...page import clean_node
14from ...wxr_context import WiktextractContext
15from ...wxr_logging import logger
16from .etymology import extract_etymology
17from .gloss import extract_gloss, process_meaning_template
18from .inflection import (
19 extract_прил_ru_comparative_forms,
20 parse_html_forms_table,
21 parse_wikitext_forms_table,
22)
23from .linkage import (
24 extract_alt_form_section,
25 extract_linkage_section,
26 extract_phrase_section,
27)
28from .models import AltForm, Form, Hyphenation, Sense, Sound, WordEntry
29from .pronunciation import (
30 extract_homophone_section,
31 extract_pronunciation_section,
32 extract_rhyme_section,
33)
34from .section_titles import (
35 ALT_FORM_SECTIONS,
36 LINKAGE_TITLES,
37 POS_TEMPLATE_NAMES,
38 POS_DATA,
39)
40from .tags import MORPHOLOGICAL_TEMPLATE_TAGS
41from .translation import extract_translations
44def process_semantic_section(
45 wxr: WiktextractContext, page_data: list[WordEntry], level_node: LevelNode
46):
47 for list_node in level_node.find_child(NodeKind.LIST):
48 for template_node in list_node.find_child_recursively(
49 NodeKind.TEMPLATE
50 ):
51 if template_node.template_name == "значение": 51 ↛ 48line 51 didn't jump to line 48 because the condition on line 51 was always true
52 sense = process_meaning_template(
53 wxr, None, page_data[-1], template_node
54 )
55 if len(sense.glosses) > 0: 55 ↛ 48line 55 didn't jump to line 48 because the condition on line 55 was always true
56 page_data[-1].senses.append(sense)
59MORPH_TEMPLATE_ARGS = {
60 "p": "prefix",
61 "prefix": "prefix",
62 "i": "interfix",
63 "interfix": "interfix",
64 "in": "infix",
65 "infix": "infix",
66 "s": "suffix",
67 "suffix": "suffix",
68 "t": "transfix",
69 "transfix": "transfix",
70 "po": "suffix",
71 "postfix": "suffix",
72 "c": "circumfix",
73 "confix": "circumfix",
74 "circumfix": "circumfix",
75 "r": "root",
76 "e": "suffix",
77 "ending": "suffix",
78}
81def get_pos_from_template(
82 wxr: WiktextractContext, template_node: TemplateNode
83) -> POSSubtitleData | None:
84 # Search for POS in template names
85 template_name = template_node.template_name.lower()
86 if template_name == "morph":
87 # https://ru.wiktionary.org/wiki/Шаблон:morph
88 pos_type = template_node.template_parameters.get("тип", "")
89 if pos_type in MORPH_TEMPLATE_ARGS: 89 ↛ 107line 89 didn't jump to line 107 because the condition on line 89 was always true
90 return {
91 "pos": MORPH_TEMPLATE_ARGS[pos_type],
92 "tags": ["morpheme"],
93 }
94 elif (
95 template_name in {"заголовок", "з"}
96 and 1 in template_node.template_parameters
97 ):
98 pos_text = clean_node(
99 wxr, None, template_node.template_parameters[1]
100 ).strip("()")
101 if len(pos_text) == 0: 101 ↛ 102line 101 didn't jump to line 102 because the condition on line 101 was never true
102 return
103 pos_text = pos_text.split()[0]
104 if pos_text in POS_DATA:
105 return POS_DATA[pos_text]
107 if template_name.startswith("прил ru"):
108 pos_arg = clean_node(
109 wxr, None, template_node.template_parameters.get("часть речи", "")
110 ).lower()
111 if pos_arg != "": 111 ↛ 112line 111 didn't jump to line 112 because the condition on line 111 was never true
112 for pos_string in POS_DATA.keys():
113 if pos_string in pos_arg:
114 return POS_DATA[pos_string]
115 else:
116 return {"pos": "adj"}
118 for part in template_name.split(maxsplit=2):
119 for subpart in part.split("-", maxsplit=2):
120 if subpart in POS_TEMPLATE_NAMES:
121 return POS_TEMPLATE_NAMES[subpart]
124def get_pos(
125 wxr: WiktextractContext, level_node: WikiNode
126) -> POSSubtitleData | None:
127 for template_node in level_node.find_child(NodeKind.TEMPLATE):
128 pos_data = get_pos_from_template(wxr, template_node)
129 if pos_data is not None: 129 ↛ 127line 129 didn't jump to line 127 because the condition on line 129 was always true
130 return pos_data
131 # POS text could also in level node content
132 for template_node in level_node.find_content(NodeKind.TEMPLATE):
133 pos_data = get_pos_from_template(wxr, template_node)
134 if pos_data is not None:
135 return pos_data
137 # Search for POS in section text
138 text = clean_node(
139 wxr,
140 None,
141 list(
142 level_node.invert_find_child(
143 LEVEL_KIND_FLAGS, include_empty_str=True
144 )
145 ),
146 )
147 for pos_string in POS_DATA.keys():
148 if pos_string in text.lower():
149 return POS_DATA[pos_string]
152def extract_morphological_section(
153 wxr: WiktextractContext, page_data: list[WordEntry], level_node: WikiNode
154) -> None:
155 pos_data = get_pos(wxr, level_node)
156 if pos_data is not None:
157 page_data[-1].pos = pos_data["pos"]
158 page_data[-1].tags.extend(pos_data.get("tags", []))
159 for child_node in level_node.find_child(NodeKind.TEMPLATE):
160 expanded_template = wxr.wtp.parse(
161 wxr.wtp.node_to_wikitext(child_node), expand_all=True
162 )
163 clean_node(wxr, page_data[-1], expanded_template) # add category links
164 if child_node.template_name.startswith(
165 (
166 "прил ru",
167 "прил-ru",
168 "сущ ",
169 "сущ-ru",
170 "гл ",
171 "гл-ru",
172 "мест ru ",
173 "числ ru ",
174 "числ-",
175 "прич ru ",
176 "Фам ru ",
177 "падежи ",
178 )
179 ):
180 for table_node in expanded_template.find_child_recursively(
181 NodeKind.TABLE
182 ):
183 parse_wikitext_forms_table(wxr, page_data[-1], table_node)
184 for table_tag in expanded_template.find_html("table"):
185 parse_html_forms_table(wxr, page_data[-1], table_tag)
186 h_str = clean_node(
187 wxr, None, child_node.template_parameters.get("слоги", "")
188 )
189 if h_str != "":
190 page_data[-1].hyphenations.append(
191 Hyphenation(parts=h_str.split("-"))
192 )
194 if child_node.template_name.startswith(("прил ru", "прил-ru")):
195 extract_прил_ru_comparative_forms(
196 wxr, page_data[-1], expanded_template
197 )
199 for node in expanded_template.children:
200 node_text = clean_node(wxr, page_data[-1], node)
201 for text in node_text.split(","):
202 text = text.strip()
203 if text in MORPHOLOGICAL_TEMPLATE_TAGS:
204 tr_tag = MORPHOLOGICAL_TEMPLATE_TAGS[text]
205 if isinstance(tr_tag, str): 205 ↛ 207line 205 didn't jump to line 207 because the condition on line 205 was always true
206 page_data[-1].tags.append(tr_tag)
207 elif isinstance(tr_tag, list):
208 page_data[-1].tags.extend(tr_tag)
211def parse_section(
212 wxr: WiktextractContext, page_data: list[WordEntry], level_node: WikiNode
213) -> None:
214 section_title = clean_node(wxr, None, level_node.largs).lower()
215 wxr.wtp.start_subsection(section_title)
216 if section_title in [
217 # Morphological and syntactic properties
218 "морфологические и синтаксические свойства",
219 # Type and syntactic properties of the word combination
220 "тип и синтаксические свойства сочетания",
221 "тип и свойства сочетания",
222 ]:
223 extract_morphological_section(wxr, page_data, level_node)
224 elif section_title in POS_DATA:
225 pos_data = POS_DATA[section_title]
226 page_data[-1].pos = pos_data["pos"]
227 page_data[-1].tags.extend(pos_data.get("tags", []))
228 extract_gloss(wxr, page_data[-1], level_node)
229 elif section_title == "произношение" and wxr.config.capture_pronunciation:
230 extract_pronunciation_section(wxr, page_data[-1], level_node)
231 elif section_title == "семантические свойства": # Semantic properties
232 process_semantic_section(wxr, page_data, level_node)
233 elif section_title in [
234 "значение",
235 "значения",
236 "как самостоятельный глагол",
237 "в значении вспомогательного глагола или связки",
238 ]:
239 extract_gloss(wxr, page_data[-1], level_node)
240 elif section_title == "этимология" and wxr.config.capture_etymologies:
241 extract_etymology(wxr, page_data[-1], level_node)
242 elif (
243 section_title
244 in [
245 "фразеологизмы и устойчивые сочетания",
246 "типичные сочетания",
247 "фразеологизмы",
248 "пословицы и поговорки",
249 ]
250 and wxr.config.capture_linkages
251 ):
252 extract_phrase_section(wxr, page_data[-1], level_node, section_title)
253 elif (
254 section_title in ["перевод", "иноязычные аналоги"]
255 and wxr.config.capture_translations
256 ):
257 extract_translations(wxr, page_data[-1], level_node)
258 elif section_title in LINKAGE_TITLES and wxr.config.capture_linkages: 258 ↛ 259line 258 didn't jump to line 259 because the condition on line 258 was never true
259 extract_linkage_section(
260 wxr, page_data[-1], LINKAGE_TITLES[section_title], level_node
261 )
262 elif section_title == "библиография": 262 ↛ 263line 262 didn't jump to line 263 because the condition on line 262 was never true
263 pass
264 elif section_title in ["латиница (latinça)", "латиница (latinca)"]: 264 ↛ 265line 264 didn't jump to line 265 because the condition on line 264 was never true
265 parse_roman_section(wxr, page_data[-1], level_node)
266 elif section_title == "прочее": 266 ↛ 267line 266 didn't jump to line 267 because the condition on line 266 was never true
267 pass
268 elif section_title == "омофоны" and wxr.config.capture_pronunciation: 268 ↛ 269line 268 didn't jump to line 269 because the condition on line 268 was never true
269 extract_homophone_section(wxr, page_data[-1], level_node)
270 elif section_title in ALT_FORM_SECTIONS: 270 ↛ 271line 270 didn't jump to line 271 because the condition on line 270 was never true
271 extract_alt_form_section(
272 wxr, page_data[-1], level_node, ALT_FORM_SECTIONS[section_title]
273 )
274 elif section_title == "рифмы": 274 ↛ 276line 274 didn't jump to line 276 because the condition on line 274 was always true
275 extract_rhyme_section(wxr, page_data[-1], level_node)
276 elif section_title not in ["см. также", "смотреть также", "смотрите также"]:
277 wxr.wtp.debug(
278 f"Unprocessed section {section_title}",
279 sortid="wixtextract/extractor/ru/page/parse_section/66",
280 )
282 for next_level_node in level_node.find_child(LEVEL_KIND_FLAGS):
283 parse_section(wxr, page_data, next_level_node)
285 extract_section_end_templates(wxr, page_data[-1], level_node)
288def parse_page(
289 wxr: WiktextractContext, page_title: str, page_text: str
290) -> list[dict[str, Any]]:
291 # Help site describing page structure:
292 # https://ru.wiktionary.org/wiki/Викисловарь:Правила_оформления_статей
294 if wxr.config.verbose: 294 ↛ 295line 294 didn't jump to line 295 because the condition on line 294 was never true
295 logger.info(f"Parsing page: {page_title}")
296 wxr.config.word = page_title
297 wxr.wtp.start_page(page_title)
298 tree = wxr.wtp.parse(page_text)
299 page_data: list[WordEntry] = []
301 for level1_node in tree.find_child(NodeKind.LEVEL1):
302 lang_code = ""
303 for subtitle_template in level1_node.find_content(NodeKind.TEMPLATE): 303 ↛ 306line 303 didn't jump to line 306 because the loop on line 303 didn't complete
304 lang_code = subtitle_template.template_name.strip(" -")
305 break
306 if lang_code == "": 306 ↛ 307line 306 didn't jump to line 307 because the condition on line 306 was never true
307 lang_code = "unknown"
308 if ( 308 ↛ 312line 308 didn't jump to line 312 because the condition on line 308 was never true
309 wxr.config.capture_language_codes is not None
310 and lang_code not in wxr.config.capture_language_codes
311 ):
312 continue
313 categories = {"categories": []}
314 lang_name = clean_node(wxr, categories, level1_node.largs)
315 wxr.wtp.start_section(lang_name)
316 base_data = WordEntry(
317 lang=lang_name,
318 lang_code=lang_code,
319 word=page_title,
320 pos="unknown",
321 )
322 base_data.categories.extend(categories["categories"])
323 extract_section_end_templates(wxr, base_data, level1_node)
324 pos_data = get_pos(wxr, level1_node)
325 if pos_data is not None:
326 base_data.pos = pos_data["pos"]
327 base_data.tags.extend(pos_data.get("tags", []))
329 for level2_node in level1_node.find_child(NodeKind.LEVEL2):
330 if base_data.pos == "unknown":
331 pos_data = get_pos(wxr, level2_node)
332 if pos_data is not None:
333 base_data.pos = pos_data["pos"]
334 base_data.tags.extend(pos_data.get("tags", []))
335 page_data.append(base_data.model_copy(deep=True))
336 extract_level2_node_contents(wxr, page_data[-1], level2_node)
337 has_level3 = False
338 for level3_node in level2_node.find_child(NodeKind.LEVEL3):
339 parse_section(wxr, page_data, level3_node)
340 has_level3 = True
341 if page_data[-1] == base_data or not has_level3: 341 ↛ 342line 341 didn't jump to line 342 because the condition on line 341 was never true
342 page_data.pop()
343 extract_low_quality_page(wxr, page_data, base_data, level2_node)
345 for any_level_index, any_level_node in enumerate(
346 level1_node.find_child(LEVEL_KIND_FLAGS & ~NodeKind.LEVEL2)
347 ):
348 if any_level_index == 0 and (
349 len(page_data) == 0
350 or page_data[-1].lang_code != base_data.lang_code
351 ):
352 page_data.append(base_data.model_copy(deep=True))
353 parse_section(wxr, page_data, any_level_node)
355 if len(page_data) > 0 and page_data[-1] == base_data: 355 ↛ 356line 355 didn't jump to line 356 because the condition on line 355 was never true
356 page_data.pop()
357 extract_low_quality_page(wxr, page_data, base_data, level1_node)
359 for d in page_data:
360 if len(d.senses) == 0: 360 ↛ 361line 360 didn't jump to line 361 because the condition on line 360 was never true
361 d.senses.append(Sense(tags=["no-gloss"]))
362 return [d.model_dump(exclude_defaults=True) for d in page_data]
365def extract_low_quality_page(
366 wxr: WiktextractContext,
367 page_data: list[WordEntry],
368 base_data: WordEntry,
369 level_node: WikiNode,
370) -> None:
371 for node in level_node.invert_find_child(LEVEL_KIND_FLAGS):
372 if isinstance(node, TemplateNode) and node.template_name.startswith(
373 "Форма-"
374 ):
375 process_form_template(wxr, page_data, base_data, node)
376 elif isinstance(node, WikiNode):
377 for template_node in node.find_child_recursively(NodeKind.TEMPLATE):
378 if template_node.template_name.startswith("Форма-"): 378 ↛ 379line 378 didn't jump to line 379 because the condition on line 378 was never true
379 process_form_template(
380 wxr, page_data, base_data, template_node
381 )
384def process_form_template(
385 wxr: WiktextractContext,
386 page_data: list[WordEntry],
387 base_data: WordEntry,
388 template_node: TemplateNode,
389) -> None:
390 # https://ru.wiktionary.org/wiki/Шаблон:Форма-сущ
391 # Шаблон:Форма-гл, "Шаблон:форма-гл en"
392 pos_data = get_pos_from_template(wxr, template_node)
393 if pos_data is not None: 393 ↛ 397line 393 didn't jump to line 397 because the condition on line 393 was always true
394 base_data.pos = pos_data["pos"]
395 base_data.tags.extend(pos_data.get("tags", []))
397 form_of = clean_node(
398 wxr,
399 None,
400 template_node.template_parameters.get(
401 "база", template_node.template_parameters.get(1, "")
402 ),
403 )
404 ipa = clean_node(
405 wxr, None, template_node.template_parameters.get("МФА", "")
406 )
407 expanded_node = wxr.wtp.parse(
408 wxr.wtp.node_to_wikitext(template_node), expand_all=True
409 )
410 current_data = base_data.model_copy(deep=True)
411 for list_item in expanded_node.find_child_recursively(NodeKind.LIST_ITEM):
412 gloss_text = clean_node(wxr, None, list_item.children)
413 if len(gloss_text) > 0: 413 ↛ 411line 413 didn't jump to line 411 because the condition on line 413 was always true
414 sense = Sense(glosses=[gloss_text])
415 if len(form_of) > 0: 415 ↛ 418line 415 didn't jump to line 418 because the condition on line 415 was always true
416 sense.form_of.append(AltForm(word=form_of))
417 sense.tags.append("form-of")
418 current_data.senses.append(sense)
420 if len(ipa) > 0: 420 ↛ 422line 420 didn't jump to line 422 because the condition on line 420 was always true
421 current_data.sounds.append(Sound(ipa=ipa))
422 if len(current_data.senses) > 0 or len(current_data.sounds) > 0: 422 ↛ exitline 422 didn't return from function 'process_form_template' because the condition on line 422 was always true
423 clean_node(wxr, current_data, template_node)
424 page_data.append(current_data)
427def parse_roman_section(
428 wxr: WiktextractContext, word_entry: WordEntry, level_node: WikiNode
429) -> None:
430 for link_node in level_node.find_child(NodeKind.LINK):
431 form_text = clean_node(wxr, None, link_node)
432 if form_text != "":
433 form = Form(form=form_text, tags=["romanization"])
434 word_entry.forms.append(form)
437def extract_section_end_templates(
438 wxr: WiktextractContext, word_entry: WordEntry, level_node: WikiNode
439) -> None:
440 # category link templates
441 # https://ru.wiktionary.org/wiki/Категория:Викисловарь:Шаблоны_категоризации
442 for template_node in level_node.find_child(NodeKind.TEMPLATE):
443 if template_node.template_name in { 443 ↛ 455line 443 didn't jump to line 455 because the condition on line 443 was never true
444 "-ание",
445 "-атель",
446 "-ация",
447 "-ение",
448 "-ка",
449 "длина слова",
450 "Категория",
451 "Омонимы",
452 "forms",
453 "multilang",
454 }:
455 clean_node(wxr, word_entry, template_node)
456 elif template_node.template_name == "zh-forms": 456 ↛ 457line 456 didn't jump to line 457 because the condition on line 456 was never true
457 extract_zh_forms_template(wxr, word_entry, template_node)
460def extract_zh_forms_template(
461 wxr: WiktextractContext,
462 base_data: WordEntry,
463 template_node: TemplateNode,
464) -> None:
465 # https://ru.wiktionary.org/wiki/Шаблон:zh-forms
466 # https://ru.wiktionary.org/wiki/Модуль:zh-forms
467 # similar to en and zh edition template
468 for p_name, p_value in template_node.template_parameters.items():
469 if not isinstance(p_name, str):
470 continue
471 if re.fullmatch(r"s\d*", p_name):
472 form_data = Form(
473 form=clean_node(wxr, None, p_value), tags=["Simplified-Chinese"]
474 )
475 if form_data.form not in ["", wxr.wtp.title]:
476 base_data.forms.append(form_data)
477 elif re.fullmatch(r"t\d*", p_name):
478 form_data = Form(
479 form=clean_node(wxr, None, p_value),
480 tags=["Traditional-Chinese"],
481 )
482 if form_data.form not in ["", wxr.wtp.title]:
483 base_data.forms.append(form_data)
484 elif p_name == "lit":
485 base_data.literal_meaning = clean_node(wxr, None, p_value)
488def extract_level2_node_contents(
489 wxr: WiktextractContext, word_entry: WordEntry, level_node: LevelNode
490) -> None:
491 for t_node in level_node.find_content(NodeKind.TEMPLATE):
492 if t_node.template_name in ["заголовок", "з"]: 492 ↛ 491line 492 didn't jump to line 491 because the condition on line 492 was always true
493 # https://ru.wiktionary.org/wiki/Шаблон:з
494 stressed_form = clean_node(
495 wxr, None, t_node.template_parameters.get("ударение", "")
496 )
497 if "(" in stressed_form:
498 stressed_form = stressed_form[
499 : stressed_form.index("(")
500 ].strip()
501 if stressed_form not in ["", wxr.wtp.title]:
502 word_entry.forms.append(
503 Form(form=stressed_form, tags=["stressed"])
504 )