Coverage for src/wiktextract/extractor/fr/etymology.py: 96%
173 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-07 00:55 +0000
1from collections import defaultdict
2from dataclasses import dataclass, field
4from wikitextprocessor.parser import (
5 LEVEL_KIND_FLAGS,
6 LevelNode,
7 NodeKind,
8 TemplateNode,
9 WikiNode,
10)
12from ...page import clean_node
13from ...wxr_context import WiktextractContext
14from .models import AttestationData, Example, WordEntry
16ATTESTATION_TEMPLATES = {"siècle", "circa", "date"}
19@dataclass
20class EtymologyData:
21 texts: list[str] = field(default_factory=list)
22 categories: list[str] = field(default_factory=list)
23 attestations: list[AttestationData] = field(default_factory=list)
24 links: list[tuple[str, str]] = field(default_factory=list)
27EtymologyDict = dict[tuple[str, str], EtymologyData]
30def extract_etymology(
31 wxr: WiktextractContext, level_node: LevelNode, base_data: WordEntry
32) -> EtymologyDict:
33 etymology_dict: EtymologyDict = defaultdict(EtymologyData)
34 level_node_index = len(level_node.children)
35 pos_id = ""
36 pos_title = ""
37 for node_index, node in level_node.find_child(
38 NodeKind.LIST | LEVEL_KIND_FLAGS, True
39 ):
40 if node.kind in LEVEL_KIND_FLAGS and node_index < level_node_index:
41 level_node_index = node_index
42 elif node.kind == NodeKind.LIST: 42 ↛ 37line 42 didn't jump to line 37 because the condition on line 42 was always true
43 for etymology_item in node.find_child(NodeKind.LIST_ITEM):
44 pos_id, pos_title = extract_etymology_list_item(
45 wxr, etymology_item, etymology_dict, pos_id, pos_title
46 )
48 if len(etymology_dict) == 0:
49 categories = {}
50 e_nodes = []
51 attestations = []
52 for node in level_node.children[:level_node_index]:
53 if (
54 isinstance(node, TemplateNode)
55 and node.template_name in ATTESTATION_TEMPLATES
56 ):
57 attestations.extend(extract_date_template(wxr, base_data, node))
58 else:
59 e_nodes.append(node)
60 links = []
61 etymology_text = clean_node(
62 wxr, categories, e_nodes, link_collector=links
63 )
64 if len(etymology_text) > 0: 64 ↛ 74line 64 didn't jump to line 74 because the condition on line 64 was always true
65 etymology_dict[("", "")].texts.extend(
66 list(filter(None, map(str.strip, etymology_text.splitlines())))
67 )
68 etymology_dict[("", "")].attestations = attestations
69 etymology_dict[("", "")].links.extend(links)
70 etymology_dict[(pos_id, pos_title)].categories.extend(
71 categories.get("categories", [])
72 )
74 if ("", "") in etymology_dict and etymology_dict.get(("", "")).texts == [
75 " "
76 ]:
77 # remove "ébauche-étym" template placeholder
78 del etymology_dict[("", "")]
80 return etymology_dict
83def extract_etymology_list_item(
84 wxr: WiktextractContext,
85 list_item: WikiNode,
86 etymology_dict: EtymologyDict,
87 pos_id: str,
88 pos_title: str,
89) -> tuple[str, str]:
90 etymology_data = find_pos_in_etymology_list(wxr, list_item)
91 if etymology_data is not None:
92 pos_id, pos_title, etymology_data = etymology_data
93 if len(etymology_data.texts) > 0:
94 etymology_dict[(pos_id, pos_title)].texts.extend(
95 etymology_data.texts
96 )
97 etymology_dict[(pos_id, pos_title)].links.extend(
98 etymology_data.links
99 )
100 etymology_dict[(pos_id, pos_title)].categories.extend(
101 etymology_data.categories
102 )
103 etymology_dict[(pos_id, pos_title)].attestations.extend(
104 etymology_data.attestations
105 )
106 else:
107 etymology_data = extract_etymology_list_item_nodes(
108 wxr, list_item.children
109 )
110 if len(etymology_data.texts) > 0: 110 ↛ 124line 110 didn't jump to line 124 because the condition on line 110 was always true
111 etymology_dict[(pos_id, pos_title)].texts.extend(
112 etymology_data.texts
113 )
114 etymology_dict[(pos_id, pos_title)].links.extend(
115 etymology_data.links
116 )
117 etymology_dict[(pos_id, pos_title)].categories.extend(
118 etymology_data.categories
119 )
120 etymology_dict[(pos_id, pos_title)].attestations.extend(
121 etymology_data.attestations
122 )
124 for child_list in list_item.find_child(NodeKind.LIST):
125 for child_list_item in child_list.find_child(NodeKind.LIST_ITEM):
126 extract_etymology_list_item(
127 wxr, child_list_item, etymology_dict, pos_id, pos_title
128 )
130 return pos_id, pos_title
133def find_pos_in_etymology_list(
134 wxr: WiktextractContext, list_item_node: WikiNode
135) -> tuple[str, str, EtymologyData] | None:
136 """
137 Return tuple of POS id, title, etymology text, categories if the passed
138 list item node starts with italic POS node or POS template, otherwise
139 return `None`.
140 """
141 for template_node in list_item_node.find_child(NodeKind.TEMPLATE):
142 if template_node.template_name == "ébauche-étym":
143 return "", "", EtymologyData(" ", [], []) # missing etymology
145 for index, node in list_item_node.find_child(
146 NodeKind.TEMPLATE | NodeKind.LINK | NodeKind.ITALIC, True
147 ):
148 if isinstance(node, TemplateNode) and node.template_name in (
149 "lien-ancre-étym",
150 "laé",
151 ):
152 expanded_template = wxr.wtp.parse(
153 wxr.wtp.node_to_wikitext(node), expand_all=True
154 )
155 for italic_node in expanded_template.find_child(NodeKind.ITALIC): 155 ↛ 145line 155 didn't jump to line 145 because the loop on line 155 didn't complete
156 for link_node in italic_node.find_child(NodeKind.LINK): 156 ↛ 155line 156 didn't jump to line 155 because the loop on line 156 didn't complete
157 if isinstance( 157 ↛ 156line 157 didn't jump to line 156 because the condition on line 157 was always true
158 link_node.largs[0][0], str
159 ) and link_node.largs[0][0].startswith("#"):
160 pos_id = link_node.largs[0][0].removeprefix("#")
161 return (
162 pos_id,
163 clean_node(wxr, None, link_node).strip(": "),
164 extract_etymology_list_item_nodes(
165 wxr, list_item_node.children[index + 1 :]
166 ),
167 )
168 elif (
169 node.kind == NodeKind.LINK
170 and isinstance(node.largs[0][0], str)
171 and node.largs[0][0].startswith("#")
172 ):
173 pos_id = node.largs[0][0].removeprefix("#")
174 return (
175 pos_id,
176 clean_node(wxr, None, node).strip(": "),
177 extract_etymology_list_item_nodes(
178 wxr, list_item_node.children[index + 1 :]
179 ),
180 )
181 elif node.kind == NodeKind.ITALIC:
182 for link_node in node.find_child(NodeKind.LINK):
183 if isinstance(link_node.largs[0][0], str) and link_node.largs[
184 0
185 ][0].startswith("#"):
186 pos_id = link_node.largs[0][0].removeprefix("#")
187 e_data = extract_etymology_list_item_nodes(
188 wxr, list_item_node.children[index + 1 :]
189 )
190 e_data.texts = [t.lstrip(") ") for t in e_data.texts]
191 return (
192 pos_id,
193 clean_node(wxr, None, link_node).strip(": "),
194 e_data,
195 )
196 italic_text = clean_node(wxr, None, node)
197 if (
198 index <= 1 # first node is empty string
199 and italic_text.startswith("(")
200 and italic_text.endswith(")")
201 ):
202 return (
203 "",
204 italic_text.strip("() "),
205 extract_etymology_list_item_nodes(
206 wxr, list_item_node.children[index + 1 :]
207 ),
208 )
211def extract_etymology_list_item_nodes(
212 wxr: WiktextractContext, nodes: list[WikiNode]
213) -> EtymologyData:
214 used_nodes = []
215 cats = {}
216 e_data = EtymologyData()
217 is_first_attest_template = True
218 for node in nodes:
219 if (
220 is_first_attest_template
221 and isinstance(node, TemplateNode)
222 and node.template_name in ATTESTATION_TEMPLATES
223 ):
224 e_data.attestations = extract_date_template(wxr, cats, node)
225 is_first_attest_template = False
226 elif not (isinstance(node, WikiNode) and node.kind == NodeKind.LIST):
227 used_nodes.append(node)
228 e_text = clean_node(wxr, cats, used_nodes, link_collector=e_data.links)
229 if e_text != "":
230 e_data.texts.append(e_text)
231 e_data.categories = cats.get("categories", [])
232 return e_data
235def insert_etymology_data(
236 lang_code: str, page_data: list[WordEntry], etymology_dict: EtymologyDict
237) -> None:
238 """
239 Insert list of etymology data extracted from the level 3 node to each sense
240 dictionary matches the language and POS.
241 """
242 sense_dict = defaultdict(list) # group by pos title and id
243 for sense_data in page_data:
244 if sense_data.lang_code == lang_code:
245 sense_dict[sense_data.pos_title].append(sense_data)
246 sense_dict[sense_data.pos_id].append(sense_data)
247 if sense_data.pos_id.endswith("-1"):
248 # extra ids for the first title
249 sense_dict[sense_data.pos_title.replace(" ", "_")].append(
250 sense_data
251 )
252 sense_dict[sense_data.pos_id.removesuffix("-1")].append(
253 sense_data
254 )
256 added_sense = []
257 for pos_id_title, etymology_data in etymology_dict.items():
258 if pos_id_title == ("", ""): # add to all sense dictionaries
259 for sense_data_list in sense_dict.values():
260 for sense_data in sense_data_list:
261 if sense_data not in added_sense:
262 sense_data.etymology_texts = etymology_data.texts
263 sense_data.etymology_links = etymology_data.links.copy()
264 sense_data.categories.extend(etymology_data.categories)
265 sense_data.attestations.extend(
266 etymology_data.attestations
267 )
268 added_sense.append(sense_data)
269 else:
270 for pos_key in pos_id_title:
271 if pos_key in sense_dict:
272 for sense_data in sense_dict[pos_key]:
273 if sense_data not in added_sense:
274 sense_data.etymology_texts = etymology_data.texts
275 sense_data.etymology_links = (
276 etymology_data.links.copy()
277 )
278 sense_data.categories.extend(
279 etymology_data.categories
280 )
281 sense_data.attestations.extend(
282 etymology_data.attestations
283 )
284 added_sense.append(sense_data)
287def extract_etymology_examples(
288 wxr: WiktextractContext,
289 level_node: LevelNode,
290 base_data: WordEntry,
291) -> None:
292 for list_node in level_node.find_child(NodeKind.LIST):
293 for list_item in list_node.find_child(NodeKind.LIST_ITEM):
294 extract_etymology_example_list_item(wxr, list_item, base_data, "")
297def extract_etymology_example_list_item(
298 wxr: WiktextractContext,
299 list_item: WikiNode,
300 base_data: WordEntry,
301 note: str,
302) -> None:
303 from .gloss import process_exemple_template
305 attestations = []
306 source = ""
307 example_nodes = []
308 has_exemple_template = False
309 for node in list_item.children:
310 if isinstance(node, TemplateNode):
311 if node.template_name in ATTESTATION_TEMPLATES:
312 attestations = extract_date_template(wxr, base_data, node)
313 elif node.template_name == "exemple":
314 has_exemple_template = True
315 example_data = process_exemple_template(
316 wxr, node, base_data, attestations
317 )
318 if example_data.text != "": 318 ↛ 309line 318 didn't jump to line 309 because the condition on line 318 was always true
319 example_data.note = note
320 base_data.etymology_examples.append(example_data)
321 elif node.template_name == "source": 321 ↛ 324line 321 didn't jump to line 324 because the condition on line 321 was always true
322 source = clean_node(wxr, base_data, node).strip("— ()")
323 else:
324 example_nodes.append(node)
325 else:
326 example_nodes.append(node)
328 if not has_exemple_template:
329 if len(attestations) == 0 and list_item.contain_node(NodeKind.LIST):
330 note = clean_node(
331 wxr,
332 base_data,
333 list(
334 list_item.invert_find_child(
335 NodeKind.LIST, include_empty_str=True
336 )
337 ),
338 )
339 for next_list in list_item.find_child(NodeKind.LIST):
340 for next_list_item in next_list.find_child(NodeKind.LIST_ITEM):
341 extract_etymology_example_list_item(
342 wxr, next_list_item, base_data, note
343 )
344 elif len(example_nodes) > 0: 344 ↛ exitline 344 didn't return from function 'extract_etymology_example_list_item' because the condition on line 344 was always true
345 example_str = clean_node(wxr, base_data, example_nodes)
346 if example_str != "": 346 ↛ exitline 346 didn't return from function 'extract_etymology_example_list_item' because the condition on line 346 was always true
347 example_data = Example(
348 text=example_str,
349 ref=source,
350 note=note,
351 attestations=attestations,
352 )
353 base_data.etymology_examples.append(example_data)
356def extract_date_template(
357 wxr: WiktextractContext, word_entry: WordEntry, t_node: TemplateNode
358) -> list[AttestationData]:
359 date_list = []
360 date = clean_node(wxr, word_entry, t_node).strip("()")
361 if date not in ["", "Date à préciser"]: 361 ↛ 363line 361 didn't jump to line 363 because the condition on line 361 was always true
362 date_list.append(AttestationData(date=date))
363 return date_list