Coverage for src/wiktextract/clean.py: 92%
328 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-23 06:29 +0000
« prev ^ index » next coverage.py v7.17.0, created at 2026-10-23 06:29 +0000
1# This file contains code to clean Wiktionary annotations from a string and to
2# produce plain text from it, typically for glossary entries but this is also
3# called for various other data to produce clean strings.
4#
5# This file also contains code for cleaning qualifiers for the "tags" field.
6#
7# Copyright (c) 2018-2022 Tatu Ylonen. See file LICENSE and https://ylonen.org
9import html
10import re
11import unicodedata
12from typing import Callable, Optional, Union
14from wikitextprocessor.common import MAGIC_FIRST, MAGIC_LAST, URL_STARTS
15from wikitextprocessor.core import NamespaceDataEntry, TemplateArgs
16from wikitextprocessor.parser import TemplateParameters, balance_apostrophes
18from .wxr_context import WiktextractContext
20######################################################################
21# Cleaning values into plain text.
22######################################################################
24superscript_ht: dict[str, str] = {
25 "0": "⁰",
26 "1": "¹",
27 "2": "²",
28 "3": "³",
29 "4": "⁴",
30 "5": "⁵",
31 "6": "⁶",
32 "7": "⁷",
33 "8": "⁸",
34 "9": "⁹",
35 "+": "⁺",
36 "-": "⁻",
37 "−": "⁻",
38 "‐": "⁻",
39 "–": "⁻",
40 "—": "⁻",
41 "一": "⁻",
42 "=": "⁼",
43 "(": "⁽",
44 ")": "⁾",
45 "A": "ᴬ",
46 "B": "ᴮ",
47 "D": "ᴰ",
48 "E": "ᴱ",
49 "G": "ᴳ",
50 "H": "ᴴ",
51 "I": "ᴵ",
52 "J": "ᴶ",
53 "K": "ᴷ",
54 "L": "ᴸ",
55 "M": "ᴹ",
56 "N": "ᴺ",
57 "O": "ᴼ",
58 "P": "ᴾ",
59 "R": "ᴿ",
60 "T": "ᵀ",
61 "U": "ᵁ",
62 "V": "ⱽ",
63 "W": "ᵂ",
64 "a": "ᵃ",
65 "b": "ᵇ",
66 "c": "ᶜ",
67 "d": "ᵈ",
68 "e": "ᵉ",
69 "f": "ᶠ",
70 "g": "ᵍ",
71 "h": "ʰ",
72 "i": "ⁱ",
73 "j": "ʲ",
74 "k": "ᵏ",
75 "l": "ˡ",
76 "m": "ᵐ",
77 "n": "ⁿ",
78 "o": "ᵒ",
79 "p": "ᵖ",
80 "r": "ʳ",
81 "s": "ˢ",
82 "t": "ᵗ",
83 "u": "ᵘ",
84 "v": "ᵛ",
85 "w": "ʷ",
86 "x": "ˣ",
87 "y": "ʸ",
88 "z": "ᶻ",
89 "β": "ᵝ",
90 "γ": "ᵞ",
91 "δ": "ᵟ",
92 "θ": "ᶿ",
93 "ι": "ᶥ",
94 "φ": "ᵠ",
95 "χ": "ᵡ",
96 "∞": "\u2002᪲", # This is a KLUDGE
97}
99subscript_ht: dict[str, str] = {
100 "0": "₀",
101 "1": "₁",
102 "2": "₂",
103 "3": "₃",
104 "4": "₄",
105 "5": "₅",
106 "6": "₆",
107 "7": "₇",
108 "8": "₈",
109 "9": "₉",
110 "+": "₊",
111 "-": "₋",
112 "−": "₋",
113 "=": "₌",
114 "(": "₍",
115 ")": "₎",
116 "a": "ₐ",
117 "e": "ₑ",
118 "h": "ₕ",
119 "i": "ᵢ",
120 "j": "ⱼ",
121 "k": "ₖ",
122 "l": "ₗ",
123 "m": "ₘ",
124 "n": "ₙ",
125 "o": "ₒ",
126 "p": "ₚ",
127 "r": "ᵣ",
128 "s": "ₛ",
129 "t": "ₜ",
130 "u": "ᵤ",
131 "v": "ᵥ",
132 "x": "ₓ",
133 "ə": "ₔ",
134 "ρ": "ᵨ",
135 "φ": "ᵩ",
136 "χ": "ᵪ",
137}
140def to_superscript(text: str) -> str:
141 "Converts text to superscript."
142 if not text:
143 return ""
144 if all(x in superscript_ht for x in text):
145 return "".join(superscript_ht[x] for x in text)
146 if len(text) == 1:
147 return "^" + text
148 return "^({})".format(text)
151def to_subscript(text: str) -> str:
152 """Converts text to subscript."""
153 if not text: 153 ↛ 154line 153 didn't jump to line 154 because the condition on line 153 was never true
154 return ""
155 if all(x in subscript_ht for x in text):
156 return "".join(subscript_ht[x] for x in text)
157 if len(text) == 1: 157 ↛ 159line 157 didn't jump to line 159 because the condition on line 157 was always true
158 return "_" + text
159 return "_({})".format(text)
162def to_chem(text: str) -> str:
163 """Converts text to chemical formula, making digits subscript."""
164 return "".join(to_subscript(x) if x.isdigit() else x for x in text)
167# Mapping from Latex names to Unicode characters/strings. This is the
168# default mapping (some cases are handled specially in the code).
169math_map: dict[str, str] = {
170 # XXX should probably change greek characters to non-slanted ones?
171 "AC": "∿",
172 "APLcomment": "⍝",
173 "APLdownarrowbox": "⍗",
174 "APLinput": "⍞",
175 "APLinv": "⌹",
176 "APLleftarrowbox": "⍇",
177 "APLlog": "⍟",
178 "APLrightarrowbox": "⍈",
179 "APLuparrowbox": "⍐",
180 "Angstroem": "Å",
181 "Bot": "⫫",
182 "Box": "□",
183 "Bumpeq": "≎",
184 "CIRCLE": "●",
185 "Cap": "⋒",
186 "CapitalDifferentialD": "ⅅ",
187 "CheckedBox": "☑",
188 "Circle": "○",
189 "Coloneqq": "⩴",
190 "ComplexI": "ⅈ",
191 "ComplexJ": "ⅉ",
192 "Cup": "⋓",
193 "Delta": "Δ",
194 "Diamond": "◇",
195 "Diamondblack": "◆",
196 "Diamonddot": "⟐",
197 "DifferentialD": "ⅆ",
198 "Digamma": "Ϝ",
199 "Doteq": "≑",
200 "DownArrowBar": "⤓",
201 "DownLeftTeeVector": "⥞",
202 "DownLeftVectorBar": "⥖",
203 "DownRightTeeVector": "⥟",
204 "DownRightVectorBar": "⥗",
205 "Downarrow": "⇓",
206 "Equal": "⩵",
207 "Euler": "Ɛ",
208 "ExponentialE": "ⅇ",
209 "ExponetialE": "ⅇ",
210 "Finv": "Ⅎ",
211 "Gamma": "Γ",
212 "Im": "ℑ",
213 "Join": "⨝",
214 "Koppa": "Ϟ",
215 "LEFTCIRCLE": "◖",
216 "LEFTcircle": "◐",
217 "LHD": "◀",
218 "LVec": "x⃖",
219 "Lambda": "Λ",
220 "Lbag": "⟅",
221 "LeftArrowBar": "⇤",
222 "LeftDownTeeVector": "⥡",
223 "LeftDownVectorBar": "⥙",
224 "LeftTeeVector": "⥚",
225 "LeftTriangleBar": "⧏",
226 "LeftUpTeeVector": "⥠",
227 "LeftUpVectorBar": "⥘",
228 "LeftVectorBar": "⥒",
229 "Leftarrow": "⇐",
230 "Leftrightarrow": "⇔",
231 "Lleftarrow": "⇚",
232 "Longleftarrow": "⟸",
233 "Longleftrightarrow": "⟺",
234 "Longmapsfrom": "⟽",
235 "Longmapsto": "⟾",
236 "Longrightarrow": "⟹",
237 "Lparen": "⦅",
238 "Lsh": "↰",
239 "MapsDown": "↧",
240 "MapsUp": "↥",
241 "Mapsfrom": "⤆",
242 "Mapsto": "⤇",
243 "Micro": "µ",
244 "Nearrow": "⇗",
245 "NestedGreaterGreater": "⪢",
246 "NestedLessLess": "⪡",
247 "NotGreaterLess": "≹",
248 "NotGreaterTilde": "≵",
249 "NotLessTilde": "≴",
250 "Nwarrow": "⇖",
251 "Omega": "Ω",
252 "Phi": "Φ",
253 "Pi": "Π",
254 "Proportion": "∷",
255 "Psi": "Ψ",
256 "Qoppa": "Ϙ",
257 "RHD": "▶",
258 "RIGHTCIRCLE": "◗",
259 "RIGHTcircle": "◑",
260 "Rbag": "⟆",
261 "Re": "ℜ",
262 "RightArrowBar": "⇥",
263 "RightDownTeeVector": "⥝",
264 "RightDownVectorBar": "⥕",
265 "RightTeeVector": "⥛",
266 "RightTriangleBar": "⧐",
267 "RightUpTeeVector": "⥜",
268 "RightUpVectorBar": "⥔",
269 "RightVectorBar": "⥓",
270 "Rightarrow": "⇒",
271 "Rparen": "⦆",
272 "Rrightarrow": "⇛",
273 "Rsh": "↱",
274 "S": "§",
275 "Same": "⩶",
276 "Sampi": "Ϡ",
277 "Searrow": "⇘",
278 "Sigma": "Σ",
279 "Square": "☐",
280 "Stigma": "Ϛ",
281 "Subset": "⋐",
282 "Sun": "☉",
283 "Supset": "⋑",
284 "Swarrow": "⇙",
285 "Theta": "Θ",
286 "Top": "⫪",
287 "UpArrowBar": "⤒",
288 "Uparrow": "⇑",
289 "Updownarrow": "⇕",
290 "Upsilon": "Υ",
291 "VDash": "⊫",
292 "VERT": "⦀",
293 "Vdash": "⊩",
294 "Vert": "‖",
295 "Vvdash": "⊪",
296 "XBox": "☒",
297 "Xi": "Ξ",
298 "Yup": "⅄",
299 "_": "_",
300 "aleph": "א",
301 "alpha": "α",
302 "amalg": "⨿",
303 "anchor": "⚓",
304 "angle": "∠",
305 "approx": "≈",
306 "approxeq": "≊",
307 "aquarius": "♒",
308 "arg": "arg",
309 "aries": "♈",
310 "arrowbullet": "➢",
311 "ast": "∗",
312 "asymp": "≍",
313 "backepsilon": "϶",
314 "backprime": "‵",
315 "backsim": "∽",
316 "backsimeq": "⋍",
317 "backslash": "",
318 "ballotx": "✗",
319 "barin": "⋶",
320 "barleftharpoon": "⥫",
321 "barrightharpoon": "⥭",
322 "barwedge": "⊼",
323 "because": "∵",
324 "beta": "β",
325 "beth": "ב",
326 "between": "≬",
327 "bigcap": "∩",
328 "bigcup": "∪",
329 "biginterleave": "⫼",
330 "bigodot": "⨀",
331 "bigoplus": "⨁",
332 "bigotimes": "⨂",
333 "bigsqcap": "⨅",
334 "bigsqcup": "⨆",
335 "bigstar": "★",
336 "bigtriangledown": "▽",
337 "bigtriangleup": "△",
338 "biguplus": "⨄",
339 "bigvee": "∨",
340 "bigwedge": "∧",
341 "bij": "⤖",
342 "biohazard": "☣",
343 "blacklozenge": "⧫",
344 "blacksmiley": "☻",
345 "blacksquare": "■",
346 "blacktriangledown": "▾",
347 "blacktriangleleft": "◂",
348 "blacktriangleright": "▸",
349 "blacktriangleup": "▴",
350 "bot": "⊥",
351 "bowtie": "⋈",
352 "boxast": "⧆",
353 "boxbar": "◫",
354 "boxbox": "⧈",
355 "boxbslash": "⧅",
356 "boxcircle": "⧇",
357 "boxdot": "⊡",
358 "boxminus": "⊟",
359 "boxplus": "⊞",
360 "boxslash": "⧄",
361 "boxtimes": "⊠",
362 "bullet": "•",
363 "bumpeq": "≏",
364 "cancer": "♋",
365 "cap": "∩",
366 "capricornus": "♑",
367 "capwedge": "⩄",
368 "cat": "⁀",
369 "cdot": "·",
370 "cdots": "⋯",
371 "cent": "¢",
372 "checkmark": "✓",
373 "chi": "χ",
374 "circ": "∘",
375 "circeq": "≗",
376 "circlearrowleft": "↺",
377 "circlearrowright": "↻",
378 "circledR": "®",
379 "circledast": "⊛",
380 "circledbslash": "⦸",
381 "circledcirc": "⊚",
382 "circleddash": "⊝",
383 "circledgtr": "⧁",
384 "circledless": "⧀",
385 "clubsuit": "♣",
386 "colon": ":",
387 "coloneq": "≔",
388 "complement": "∁",
389 "cong": "≅",
390 "coprod": "∐",
391 "corresponds": "≙",
392 "cup": "∪",
393 "curlyeqprec": "⋞",
394 "curlyeqsucc": "⋟",
395 "curlyvee": "⋎",
396 "curlywedge": "⋏",
397 "curvearrowleft": "↶",
398 "curvearrowright": "↷",
399 "dagger": "†",
400 "daleth": "ד",
401 "dashleftarrow": "⇠",
402 "dashrightarrow": "⇢",
403 "dashv": "⊣",
404 "ddagger": "‡",
405 "delta": "δ",
406 "diameter": "∅",
407 "diamond": "⋄",
408 "diamondsuit": "♢",
409 "digamma": "ϝ",
410 "div": "÷",
411 "divideontimes": "⋇",
412 "dlsh": "↲",
413 "dot\\bigvee": "⩒",
414 "dot\\cap": "⩀",
415 "dot\\cup": "⊍",
416 "dot\\lor": "⩒",
417 "dot\\vee": "⩒",
418 "doteq": "≐",
419 "dotplus": "∔",
420 "dots": "…",
421 "doublebarwedge": "⩞",
422 "downarrow": "↓",
423 "downdownarrows": "⇊",
424 "downdownharpoons": "⥥",
425 "downharpoonleft": "⇃",
426 "downharpoonright": "⇂",
427 "downuparrows": "⇵",
428 "downupharpoons": "⥯",
429 "drsh": "↳",
430 "dsub": "⩤",
431 "earth": "♁",
432 "eighthnote": "♪",
433 "ell": "ℓ",
434 "emptyset": "∅",
435 "epsilon": "ϵ",
436 "eqcirc": "≖",
437 "eqcolon": "∹",
438 "eqsim": "≂",
439 "eqslantgtr": "⪖",
440 "eqslantless": "⪕",
441 "equiv": "≡",
442 "eta": "η",
443 "eth": "ð",
444 "exists": "∃",
445 "fallingdotseq": "≒",
446 "fcmp": "⨾",
447 "female": "♀",
448 "ffun": "⇻",
449 "finj": "⤕",
450 "fint": "⨏",
451 "flat": "♭",
452 "footnotesize": "",
453 "forall": "∀",
454 "fourth": "⁗",
455 "frown": "⌢",
456 "frownie": "☹",
457 "gamma": "γ",
458 "ge": ">",
459 "gemini": "♊",
460 "geq": "≥",
461 "geqq": "≧",
462 "geqslant": "⩾",
463 "gg": "≫",
464 "ggcurly": "⪼",
465 "ggg": "⋙",
466 "gimel": "ג",
467 "gnapprox": "⪊",
468 "gneq": "⪈",
469 "gneqq": "≩",
470 "gnsim": "⋧",
471 "gtrapprox": "⪆",
472 "gtrdot": "⋗",
473 "gtreqless": "⋛",
474 "gtreqqless": "⪌",
475 "gtrless": "≷",
476 "gtrsim": "≳",
477 "hash": "⋕",
478 "heartsuit": "♡",
479 "hookleftarrow": "↩",
480 "hookrightarrow": "↪",
481 "hslash": "ℏ",
482 "iddots": "⋰",
483 "iff": "⟺",
484 "iiiint": "⨌",
485 "iiint": "∭",
486 "iint": "∬",
487 "imath": "ı",
488 "implies": "⟹",
489 "in": "∈",
490 "infty": "∞",
491 "int": "∫",
492 "intercal": "⊺",
493 "interleave": "⫴",
494 "invamp": "⅋",
495 "invdiameter": "⍉",
496 "invneg": "⌐",
497 "iota": "ι",
498 "jmath": "ȷ",
499 "jupiter": "♃",
500 "kappa": "κ",
501 "koppa": "ϟ",
502 "lambda": "λ",
503 "land": "∧",
504 "lang": "⟪",
505 "langle": "⟨",
506 "large": "",
507 "lblot": "⦉",
508 "lbrace": "{",
509 "lbrack": "[",
510 "lceil": "⌈",
511 "ldots": "…",
512 "le": "<",
513 "leadsto": "⤳",
514 "leftarrow": "←",
515 "leftarrowtail": "↢",
516 "leftarrowtriangle": "⇽",
517 "leftbarharpoon": "⥪",
518 "leftharpoondown": "↽",
519 "leftharpoonup": "↼",
520 "leftleftarrows": "⇇",
521 "leftleftharpoons": "⥢",
522 "leftmoon": "☾",
523 "leftrightarrow": "↔",
524 "leftrightarrows": "⇆",
525 "leftrightarrowtriangle": "⇿",
526 "leftrightharpoon": "⥊",
527 "leftrightharpoondown": "⥐",
528 "leftrightharpoons": "⇋",
529 "leftrightharpoonup": "⥎",
530 "leftrightsquigarrow": "↭",
531 "leftslice": "⪦",
532 "leftsquigarrow": "⇜",
533 "leftthreetimes": "⋋",
534 "leftupdownharpoon": "⥑",
535 "leo": "♌",
536 "leq": "≤",
537 "leqq": "≦",
538 "leqslant": "⩽",
539 "lessapprox": "⪅",
540 "lessdot": "⋖",
541 "lesseqgtr": "⋚",
542 "lesseqqgtr": "⪋",
543 "lessgtr": "≶",
544 "lessim": "≲",
545 "lesssim": "≲",
546 "lfloor": "⌊",
547 "lgroup": "⟮",
548 "lhd": "◁",
549 "libra": "♎",
550 "lightning": "↯",
551 "limg": "⦇",
552 "ll": "≪",
553 "llbracket": "⟦",
554 "llcorner": "⌞",
555 "llcurly": "⪻",
556 "lll": "⋘",
557 "lnapprox": "⪉",
558 "lneq": "⪇",
559 "lneqq": "≨",
560 "lnot": "¬",
561 "lnsim": "⋦",
562 "longleftarrow": "⟵",
563 "longleftrightarrow": "⟷",
564 "longmapsfrom": "⟻",
565 "longmapsto": "⟼",
566 "longrightarrow": "⟶",
567 "looparrowleft": "↫",
568 "looparrowright": "↬",
569 "lor": "∨",
570 "lozenge": "◊",
571 "lrcorner": "⌟",
572 "ltimes": "⋉",
573 "male": "♂",
574 "maltese": "✠",
575 "mapsfrom": "↤",
576 "mapsto": "↦",
577 "measuredangle": "∡",
578 "medbullet": "⚫",
579 "medcirc": "⚪",
580 "mercury": "☿",
581 "mho": "℧",
582 "mid": "∣",
583 "mlcp": "⫛",
584 "mod": " mod ",
585 "models": "⊧",
586 "mp": "∓",
587 "mu": "μ",
588 "multimap": "⊸",
589 "multimapboth": "⧟",
590 "multimapdotbothA": "⊶",
591 "multimapdotbothB": "⊷",
592 "multimapinv": "⟜",
593 "nLeftarrow": "⇍",
594 "nLeftrightarrow": "⇎",
595 "nRightarrow": "⇏",
596 "nVDash": "⊯",
597 "nVdash": "⊮",
598 "nabla": "∇",
599 "napprox": "≉",
600 "natural": "♮",
601 "ncong": "≇",
602 "nearrow": "↗",
603 "neg": "¬",
604 "neptune": "♆",
605 "neq": "≠",
606 "nequiv": "≢",
607 "nexists": "∄",
608 "ngeq": "≱",
609 "ngtr": "≯",
610 "ni": "∋",
611 "nleftarrow": "↚",
612 "nleftrightarrow": "↮",
613 "nleq": "≰",
614 "nless": "≮",
615 "nmid": "∤",
616 "nni": "∌",
617 "normalsize": "",
618 "not\\in": "∉",
619 "not\\ni": "∌",
620 "not\\preceq": "⋠",
621 "not\\subset": "⊄",
622 "not\\subseteq": "⊈",
623 "not\\succeq": "⋡",
624 "not\\supset": "⊅",
625 "not\\supseteq": "⊉",
626 "not\\trianglelefteq": "⋬",
627 "not\\trianglerighteq": "⋭",
628 "not\\vartriangleleft": "⋪",
629 "not\\vartriangleright": "⋫",
630 "notasymp": "≭",
631 "notbackslash": "⍀",
632 "notin": "∉",
633 "notslash": "⌿",
634 "nparallel": "∦",
635 "nprec": "⊀",
636 "npreceq": "⋠",
637 "nrightarrow": "↛",
638 "nsim": "≁",
639 "nsimeq": "≄",
640 "nsqsubseteq": "⋢",
641 "nsqsupseteq": "⋣",
642 "nsubset": "⊄",
643 "nsubseteq": "⊈",
644 "nsucc": "⊁",
645 "nsucceq": "⋡",
646 "nsupset": "⊅",
647 "nsupseteq": "⊉",
648 "ntriangleleft": "⋪",
649 "ntrianglelefteq": "⋬",
650 "ntriangleright": "⋫",
651 "ntrianglerighteq": "⋭",
652 "nu": "ν",
653 "nvDash": "⊭",
654 "nvdash": "⊬",
655 "nwarrow": "↖",
656 "odot": "⊙",
657 "oiiint": "∰",
658 "oiint": "∯",
659 "oint": "∮",
660 "ointctrclockwise": "∳",
661 "omega": "ω",
662 "ominus": "⊖",
663 "oplus": "⊕",
664 "oslash": "⊘",
665 "otimes": "⊗",
666 "over": "/",
667 "overbrace": "⏞",
668 "overleftrightarrow": "x⃡",
669 "overparen": "⏜",
670 "overset?=": "≟",
671 "overset{?}{=}": "≟",
672 "overset{\\operatorname{def}}{=}": "≝",
673 "parallel": "∥",
674 "partial": "∂",
675 "pencil": "✎",
676 "perp": "⊥",
677 "pfun": "⇸",
678 "phi": "ϕ",
679 "pi": "π",
680 "pinj": "⤔",
681 "pisces": "♓",
682 "pitchfork": "⋔",
683 "pluto": "♇",
684 "pm": "±",
685 "pointright": "☞",
686 "pounds": "£",
687 "prec": "≺",
688 "precapprox": "⪷",
689 "preccurlyeq": "≼",
690 "preceq": "⪯",
691 "preceqq": "⪳",
692 "precnapprox": "⪹",
693 "precnsim": "⋨",
694 "precsim": "≾",
695 "prime": "′",
696 "prod": "∏",
697 "propto": "∝",
698 "psi": "ψ",
699 "psur": "⤀",
700 "qoppa": "ϙ",
701 "quad": " ",
702 "quarternote": "♩",
703 "radiation": "☢",
704 "rang": "⟫",
705 "rangle": "⟩",
706 "rarr": "→",
707 "rblot": "⦊",
708 "rbrace": "}",
709 "rbrack": "]",
710 "rceil": "⌉",
711 "recycle": "♻",
712 "rfloor": "⌋",
713 "rgroup": "⟯",
714 "rhd": "▷",
715 "rho": "ρ",
716 "rightangle": "∟",
717 "rightarrow": "→",
718 "rightarrowtail": "↣",
719 "rightarrowtriangle": "⇾",
720 "rightbarharpoon": "⥬",
721 "rightharpoondown": "⇁",
722 "rightharpoonup": "⇀",
723 "rightleftarrows": "⇄",
724 "rightleftharpoon": "⥋",
725 "rightleftharpoons": "⇌",
726 "rightmoon": "☽",
727 "rightrightarrows": "⇉",
728 "rightrightharpoons": "⥤",
729 "rightslice": "⪧",
730 "rightsquigarrow": "⇝",
731 "rightthreetimes": "⋌",
732 "rightupdownharpoon": "⥏",
733 "rimg": "⦈",
734 "risingdotseq": "≓",
735 "rrbracket": "⟧",
736 "rsub": "⩥",
737 "rtimes": "⋊",
738 "sagittarius": "♐",
739 "sampi": "ϡ",
740 "saturn": "♄",
741 "scorpio": "♏",
742 "scriptsize": "",
743 "searrow": "↘",
744 "second": "″",
745 "setminus": "⧵",
746 "sharp": "♯",
747 "sigma": "σ",
748 "sim": "∼",
749 "simeq": "≃",
750 "sixteenthnote": "♬",
751 "skull": "☠",
752 "slash": "∕",
753 "small": "",
754 "smallsetminus": "∖",
755 "smalltriangledown": "▿",
756 "smalltriangleleft": "◃",
757 "smalltriangleright": "▹",
758 "smalltriangleup": "▵",
759 "smile": "⌣",
760 "smiley": "☺",
761 "spadesuit": "♠",
762 "spddot": "¨",
763 "sphat": "^",
764 "sphericalangle": "∢",
765 "spot": "⦁",
766 "sptilde": "~",
767 "sqcap": "⊓",
768 "sqcup": "⊔",
769 "sqint": "⨖",
770 "sqrt": "√", # ∛ ∜ - partly special handling below
771 "sqrt[3]": "∛",
772 "sqrt[4]": "∜",
773 "sqsubset": "⊏",
774 "sqsubseteq": "⊑",
775 "sqsupset": "⊐",
776 "sqsupseteq": "⊒",
777 "square": "□",
778 "sslash": "⫽",
779 "star": "⋆",
780 "steaming": "☕",
781 "stigma": "ϛ",
782 "strictfi": "⥼",
783 "strictif": "⥽",
784 "subset": "⊂",
785 "subseteq": "⊆",
786 "subseteqq": "⫅",
787 "subsetneq": "⊊",
788 "subsetneqq": "⫋",
789 "succ": "≻",
790 "succapprox": "⪸",
791 "succcurlyeq": "≽",
792 "succeq": "⪰",
793 "succeqq": "⪴",
794 "succnapprox": "⪺",
795 "succnsim": "⋩",
796 "succsim": "≿",
797 "sum": "∑",
798 "sun": "☼",
799 "supset": "⊃",
800 "supseteq": "⊇",
801 "supseteqq": "⫆",
802 "supsetneq": "⊋",
803 "supsetneqq": "⫌",
804 "swarrow": "↙",
805 "swords": "⚔",
806 "talloblong": "⫾",
807 "tau": "τ",
808 "taurus": "♉",
809 "tcohm": "Ω",
810 "textbackslash": "\\",
811 "textbar": "|",
812 "textbullet": "•",
813 "textgreater": ">",
814 "textless": "<",
815 "textprime": "′",
816 "therefore": "∴",
817 "theta": "θ",
818 "third": "‴",
819 "times": "×",
820 "tiny": "",
821 "to": "→",
822 "top": "⊤",
823 "triangle": "∆",
824 "trianglelefteq": "⊴",
825 "triangleq": "≜",
826 "trianglerighteq": "⊵",
827 "twoheadleftarrow": "↞",
828 "twoheadrightarrow": "↠",
829 "twonotes": "♫",
830 "ulcorner": "⌜",
831 "underbar": " ̱",
832 "underbrace": "⏟",
833 "underleftarrow": "x⃮",
834 "underline": " ̲",
835 "underparen": "⏝",
836 "underrightarrow": "x⃯",
837 "uparrow": "↑",
838 "updownarrow": "↕",
839 "updownarrows": "⇅",
840 "updownharpoons": "⥮",
841 "upharpoonleft": "↿",
842 "upharpoonright": "↾",
843 "uplus": "⊎",
844 "upsilon": "υ",
845 "upuparrows": "⇈",
846 "upupharpoons": "⥣",
847 "uranus": "♅",
848 "urcorner": "⌝",
849 "utilde": " ̰",
850 "vDash": "⊨",
851 "varbeta": "β",
852 "varclubsuit": "♧",
853 "vardiamondsuit": "♦",
854 "varepsilon": "ε",
855 "varheartsuit": "♥",
856 "varkappa": "ϰ",
857 "varnothing": "∅",
858 "varointclockwise": "∲",
859 "varphi": "φ",
860 "varpi": "ϖ",
861 "varprod": "⨉",
862 "varrho": "ϱ",
863 "varsigma": "ς",
864 "varspadesuit": "♤",
865 "vartheta": "θ",
866 "vartriangleleft": "⊲",
867 "vartriangleright": "⊳",
868 "vdash": "⊢",
869 "vdots": "⋮",
870 "vee": "∨",
871 "veebar": "⊻",
872 "vert": "|",
873 "virgo": "♍",
874 "warning": "⚠",
875 "wasylozenge": "⌑",
876 "wedge": "∧",
877 "widehat=": "≙",
878 "widehat{=}": "≙",
879 "wp": "℘",
880 "wr": "≀",
881 "xi": "ξ",
882 "yen": "¥",
883 "yinyang": "☯",
884 "zcmp": "⨟",
885 "zeta": "ζ",
886 "zhide": "⧹",
887 "zpipe": "⨠",
888 "zproject": "⨡",
889 "|": "‖",
890 # Accents XXX these really should be handled specially with diacritics
891 # after argument
892 "acute": "́",
893 "bar": "̄",
894 "breve": "̆",
895 "check": "̌",
896 "ddddot": "⃜",
897 "dddot": "⃛",
898 "ddot": "̈",
899 "ddots": "⋱",
900 "dot": "̇",
901 "grave": "̀",
902 "hat": "̂",
903 "lvec": "⃐",
904 "mathring": "̊",
905 "not": "̸",
906 "overline": "◌̅",
907 "tilde": "̃",
908 "vec": "⃑",
909 # Some ignored operators
910 "bigl": "",
911 "bigr": "",
912 "left": "",
913 "right": "",
914 "style": "",
915 "textstyle": "",
916 "mathrm": "",
917}
919mathcal_map: dict[str, str] = {
920 "A": "𝒜",
921 "B": "ℬ",
922 "C": "𝒞",
923 "D": "𝒟",
924 "E": "ℰ",
925 "F": "ℱ",
926 "G": "𝒢",
927 "H": "ℋ",
928 "I": "ℐ",
929 "J": "𝒥",
930 "K": "𝒦",
931 "L": "ℒ",
932 "M": "ℳ",
933 "N": "𝒩",
934 "O": "𝒪",
935 "P": "𝒫",
936 "Q": "𝒬",
937 "R": "ℛ",
938 "S": "𝒮",
939 "T": "𝒯",
940 "U": "𝒰",
941 "V": "𝒱",
942 "W": "𝒲",
943 "X": "𝒳",
944 "Y": "𝒴",
945 "Z": "𝒵",
946 "a": "𝒶",
947 "b": "𝒷",
948 "c": "𝒸",
949 "d": "𝒹",
950 "e": "ℯ",
951 "f": "𝒻",
952 "g": "ℊ",
953 "h": "𝒽",
954 "i": "𝒾",
955 "j": "𝒿",
956 "k": "𝓀",
957 "l": "𝓁",
958 "m": "𝓂",
959 "n": "𝓃",
960 "o": "ℴ",
961 "p": "𝓅",
962 "q": "𝓆",
963 "r": "𝓇",
964 "s": "𝓈",
965 "t": "𝓉",
966 "u": "𝓊",
967 "v": "𝓋",
968 "w": "𝓌",
969 "x": "𝓍",
970 "y": "𝓎",
971 "z": "𝓏",
972}
974mathfrak_map: dict[str, str] = {
975 "A": "𝔄",
976 "B": "𝔅",
977 "C": "ℭ",
978 "D": "𝔇",
979 "E": "𝔈",
980 "F": "𝔉",
981 "G": "𝔊",
982 "H": "ℌ",
983 "J": "𝔍",
984 "K": "𝔎",
985 "L": "𝔏",
986 "M": "𝔐",
987 "N": "𝔑",
988 "O": "𝔒",
989 "P": "𝔓",
990 "Q": "𝔔",
991 "S": "𝔖",
992 "T": "𝔗",
993 "U": "𝔘",
994 "V": "𝔙",
995 "W": "𝔚",
996 "X": "𝔛",
997 "Y": "𝔜",
998 "Z": "ℨ",
999}
1001mathbb_map: dict[str, str] = {
1002 "A": "𝔸",
1003 "B": "𝔹",
1004 "C": "ℂ",
1005 "D": "𝔻",
1006 "E": "𝔼",
1007 "F": "𝔽",
1008 "G": "𝔾",
1009 "H": "ℍ",
1010 "I": "𝕀",
1011 "J": "𝕁",
1012 "K": "𝕂",
1013 "L": "𝕃",
1014 "M": "𝕄",
1015 "N": "ℕ",
1016 "O": "𝕆",
1017 "P": "ℙ",
1018 "Q": "ℚ",
1019 "R": "ℝ",
1020 "S": "𝕊",
1021 "T": "𝕋",
1022 "U": "𝕌",
1023 "V": "𝕍",
1024 "W": "𝕎",
1025 "X": "𝕏",
1026 "Y": "𝕐",
1027 "Z": "ℤ",
1028 "a": "𝕒",
1029 "b": "𝕓",
1030 "c": "𝕔",
1031 "d": "𝕕",
1032 "e": "𝕖",
1033 "f": "𝕗",
1034 "g": "𝕘",
1035 "h": "𝕙",
1036 "i": "𝕚",
1037 "j": "𝕛",
1038 "k": "𝕜",
1039 "l": "𝕝",
1040 "m": "𝕞",
1041 "n": "𝕟",
1042 "o": "𝕠",
1043 "p": "𝕡",
1044 "q": "𝕢",
1045 "r": "𝕣",
1046 "s": "𝕤",
1047 "t": "𝕥",
1048 "u": "𝕦",
1049 "v": "𝕧",
1050 "w": "𝕨",
1051 "x": "𝕩",
1052 "y": "𝕪",
1053 "z": "𝕫",
1054 "pi": "ℼ",
1055 "gamma": "ℽ",
1056 "Gamma": "ℾ",
1057 "Pi": "ℿ",
1058 "Sigma": "⅀",
1059 "0": "𝟘",
1060 "1": "𝟙",
1061 "2": "𝟚",
1062 "3": "𝟛",
1063 "4": "𝟜",
1064 "5": "𝟝",
1065 "6": "𝟞",
1066 "7": "𝟟",
1067 "8": "𝟠",
1068 "9": "𝟡",
1069}
1072def mathcal_fn(text: str) -> str:
1073 return "".join(mathcal_map.get(x, x) for x in text)
1076def mathfrak_fn(text: str) -> str:
1077 return "".join(mathfrak_map.get(x, x) for x in text)
1080def mathbb_fn(text: str) -> str:
1081 return "".join(mathbb_map.get(x, x) for x in text)
1084def to_math(text: str) -> str:
1085 """Converts a mathematical formula to ASCII."""
1086 # print("to_math: {!r}".format(text))
1087 magic_vec: list[str] = []
1089 def expand(text: str) -> str:
1090 while True:
1091 orig = text
1092 # formatting with {:c} converts input into character
1093 text = re.sub(
1094 r"[{:c}-{:c}]".format(MAGIC_FIRST, MAGIC_LAST),
1095 lambda m: magic_vec[ord(m.group(0)) - MAGIC_FIRST],
1096 text,
1097 )
1098 if text == orig:
1099 break
1100 return text
1102 def recurse(text: str) -> str:
1103 def math_magic(
1104 text: str, left: str, right: str, fn: Callable[[str], str]
1105 ) -> str:
1106 regexp_str = r"{}([^{}{}]+){}".format(
1107 re.escape(left),
1108 re.escape(left),
1109 re.escape(right),
1110 re.escape(right),
1111 )
1112 regexp = re.compile(regexp_str)
1114 def repl(m: re.Match) -> str:
1115 magic = chr(MAGIC_FIRST + len(magic_vec))
1116 t = fn(m.group(1)).strip()
1117 magic_vec.append(t)
1118 return magic
1120 while True:
1121 orig = text
1122 text = re.sub(regexp, repl, text)
1123 if text == orig:
1124 break
1125 return text
1127 def expand_group(v: str) -> str:
1128 fn: Optional[Callable[[str], str]] = None
1129 if re.match(r"\\mathcal\b", v):
1130 fn = mathcal_fn
1131 v = v[8:].strip()
1132 elif re.match(r"\\mathfrak\b", v):
1133 fn = mathfrak_fn
1134 v = v[9:].strip()
1135 elif re.match(r"\\mathbb\b", v):
1136 fn = mathbb_fn
1137 v = v[7:]
1138 elif re.match(r"\\(begin|end)\b", v): 1138 ↛ 1139line 1138 didn't jump to line 1139 because the condition on line 1138 was never true
1139 v = "" # Skip
1140 elif re.match(r"\\text\b", v): 1140 ↛ 1141line 1140 didn't jump to line 1141 because the condition on line 1140 was never true
1141 v = v[5:]
1142 elif re.match(r"\\pmod\b", v): 1142 ↛ 1143line 1142 didn't jump to line 1143 because the condition on line 1142 was never true
1143 v = v[5:].strip()
1144 v = "(mod " + expand_group(v) + ")"
1145 elif re.match(r"\\sqrt($|[0-9\[]|\b)", v):
1146 m = re.match(r"\\sqrt\s*(?:\[([^\]]*)\])?(.*)$", v, re.DOTALL)
1147 assert m is not None
1148 index, radicand = m.groups()
1149 index = (index or "").strip()
1150 if index in ("", "2"):
1151 root = "√"
1152 elif index == "3":
1153 root = "∛"
1154 elif index == "4":
1155 root = "∜"
1156 else:
1157 root = to_superscript(index) + "√"
1158 # Keep the radicand after the root sign, in parentheses unless
1159 # it is a single symbol or number.
1160 radicand = expand_group(radicand.strip()).strip()
1161 if len(radicand) > 1 and not radicand.isalnum():
1162 radicand = "(" + radicand + ")"
1163 v = root + radicand
1164 elif re.match(r"\\(frac|binom)($|[0-9]|\b)", v):
1165 m = re.match(
1166 r"\\(frac|binom)\s*(\\[a-zA-Z]+|\\.|.)\s*"
1167 r"(\\[a-zA-Z]+|\\.|.)$",
1168 v,
1169 )
1170 if not m: 1170 ↛ 1171line 1170 didn't jump to line 1171 because the condition on line 1170 was never true
1171 print("MATH FRAC/BINOM ERROR: {!r}".format(v))
1172 return v
1173 op, a, b = m.groups()
1174 a = expand_group(a).strip()
1175 b = expand_group(b).strip()
1176 if len(a) > 1:
1177 a = "(" + a + ")"
1178 if len(b) > 1:
1179 b = "(" + b + ")"
1180 if op == "frac": 1180 ↛ 1182line 1180 didn't jump to line 1182 because the condition on line 1180 was always true
1181 v = a + "/" + b
1182 elif op == "binom":
1183 v = "binom({}, {})".format(a, b)
1184 else:
1185 # Should never get here
1186 v = "{}({})".format(op, v)
1187 elif v.startswith("_"):
1188 fn = to_subscript
1189 v = v[1:]
1190 elif v.startswith("^"):
1191 fn = to_superscript
1192 v = v[1:]
1193 if v.startswith("\\"):
1194 mapped = math_map.get(v[1:].strip())
1195 if mapped is None:
1196 if v[1:].strip().isalnum(): 1196 ↛ 1199line 1196 didn't jump to line 1199 because the condition on line 1196 was always true
1197 v = " " + v[1:].strip() + " "
1198 else:
1199 v = v[1:].strip()
1200 else:
1201 v = mapped
1202 elif v.isspace() or v in ("&",): # Ignore certain special chars 1202 ↛ 1203line 1202 didn't jump to line 1203 because the condition on line 1202 was never true
1203 v = ""
1204 if fn is not None:
1205 v = expand(v)
1206 v = fn(v)
1207 v = expand(v)
1208 return v
1210 parts: list[str] = []
1211 while True:
1212 orig = text
1213 text = math_magic(text, "{", "}", recurse)
1214 if text == orig:
1215 break
1216 for m in re.finditer(
1217 r"\s+|"
1218 r"\\frac\s*(\\[a-zA-Z]+|\\.|.)\s*"
1219 r"(\\dot\\(bigvee|cup|cap|lor|vee)|"
1220 r"\\not\\(subset|supset|subseteq|supseteq|in|ni|"
1221 r"preceq|succeq|vartrianglelefteq|"
1222 r"vartrianglerighteq|trianglelefteq|"
1223 r"trianglerighteq)|"
1224 r"\\widehat\{=\}|\\widehat=|"
1225 r"\\overset\{?\}\{=\}|"
1226 r"\\overset\?=|"
1227 r"\\overset\{\\operatorname\{def\}\}\{=\}|"
1228 r"\\[a-zA-Z]+|\\.|.)|"
1229 r"(\\(mathcal|mathfrak|mathbb|text|begin|end|pmod)"
1230 r"\b\s*|"
1231 r"\\sqrt\b(\[\d+\])?)?"
1232 r"[_^]?(\\[a-zA-Z]+\s*|\\.|\w+|.)",
1233 text,
1234 ):
1235 v = m.group(0).strip()
1236 if not v:
1237 continue
1238 v = expand_group(v)
1239 if v: 1239 ↛ 1216line 1239 didn't jump to line 1216 because the condition on line 1239 was always true
1240 if (
1241 parts and parts[-1][-1].isalpha() and v[0] in "0123456789"
1242 ) or (
1243 parts
1244 and parts[-1][-1] in "0123456789"
1245 and v[0] in "0123456789"
1246 ):
1247 v = " " + v
1248 parts.append(v)
1250 text = "".join(parts)
1251 return text
1253 text = recurse(text)
1254 # print("math text final: {!r}".format(text))
1255 return text
1258def bold_follows(parts: list[str], i: int) -> bool:
1259 """Checks if there is a bold (''') in parts after parts[i]. We allow
1260 intervening italics ('')."""
1261 parts = parts[i + 1 :]
1262 for p in parts: 1262 ↛ 1267line 1262 didn't jump to line 1267 because the loop on line 1262 didn't complete
1263 if not p.startswith("''"):
1264 continue
1265 if p.startswith("'''"): 1265 ↛ 1262line 1265 didn't jump to line 1262 because the condition on line 1265 was always true
1266 return True
1267 return False
1270def remove_italic_and_bold(text: str) -> str:
1271 """Based on token_iter in wikitextprocessor"""
1272 assert isinstance(text, str)
1273 lines = re.split(r"(\n+)", text) # Lines and separators
1274 parts_re = re.compile(r"(''+)")
1275 new_text_parts = []
1276 for line in lines:
1277 parts = balance_apostrophes(re.split(parts_re, line))
1278 state = 0 # 1=in italic 2=in bold 3=in both
1279 for i, part in enumerate(parts):
1280 if part.startswith("''"):
1281 # This is a bold/italic part. Scan the rest of the line
1282 # to determine how it should be interpreted if there are
1283 # more than two apostrophes.
1284 if part.startswith("'''''"):
1285 if state == 1: # in italic
1286 part = part[5:]
1287 state = 2
1288 elif state == 2: # in bold
1289 part = part[5:]
1290 state = 1
1291 elif state == 3: # in both
1292 state = 0
1293 part = part[5:]
1294 else: # in nothing
1295 part = part[5:]
1296 state = 3
1297 elif part.startswith("'''"):
1298 if state == 1: # in italic
1299 if bold_follows(parts, i): 1299 ↛ 1303line 1299 didn't jump to line 1303 because the condition on line 1299 was always true
1300 part = part[3:]
1301 state = 3
1302 else:
1303 part = part[2:]
1304 state = 0
1305 elif state == 2: # in bold
1306 part = part[3:]
1307 state = 0
1308 elif state == 3: # in both
1309 part = part[3:]
1310 state = 1
1311 else: # in nothing
1312 part = part[3:]
1313 state = 2
1314 elif part.startswith("''"): 1314 ↛ 1327line 1314 didn't jump to line 1327 because the condition on line 1314 was always true
1315 if state == 1: # in italic
1316 part = part[2:]
1317 state = 0
1318 elif state == 2: # in bold
1319 part = part[2:]
1320 state = 3
1321 elif state == 3: # in both
1322 part = part[2:]
1323 state = 2
1324 else: # in nothing
1325 part = part[2:]
1326 state = 1
1327 if part: 1327 ↛ 1328line 1327 didn't jump to line 1328 because the condition on line 1327 was never true
1328 new_text_parts.append(part)
1329 continue
1330 new_text_parts.append(part)
1331 new_text_parts.append("\n")
1332 new_text_parts = new_text_parts[:-1] # remove last \n
1333 return "".join(new_text_parts)
1336# regex to find File/Image link attributes that would mean an image
1337# is *not* inline
1338NOT_INLINE_IMG_RE = re.compile(r"\|\s*(right|left|center|thumb|frame)\s*\|")
1341URL_STARTS_RE = re.compile(
1342 r"({})".format(r"|".join(URL_STARTS)), flags=re.IGNORECASE
1343)
1345IMAGE_LINK_RE: Optional[re.Pattern] = None
1348def remove_invisible_markup(title: str) -> str:
1349 """Remove omitted content before text and link extraction."""
1350 # remove nowiki tag returned from `Wtp.node_to_html()`
1351 title = re.sub(r"<nowiki\s*/>", "", title)
1353 # Remove any remaining templates
1354 # title = re.sub(r"\{\{[^}]+\}\}", "", title)
1356 # Remove tables, which can contain other tables
1357 prev = ""
1358 while title != prev:
1359 prev = title
1360 title = re.sub(
1361 r"\{\|((?!\{\|)(?!\|\}).)*\|\}",
1362 "\n",
1363 title,
1364 flags=re.DOTALL,
1365 )
1366 # title = re.sub(r"(?s)\{\|.*?\|\}", "\n", title)
1367 # Remove second reference tags (<ref name="ref_name"/>)
1368 title = re.sub(r"<ref\s+name=\"[^\"]+\"\s*/>", "", title)
1369 # Remove references (<ref>...</ref>).
1370 title = re.sub(r"(?is)<ref\b\s*[^>/]*?>\s*.*?</ref\s*>", "", title)
1371 # Replace <span>...</span> by stripped content without newlines
1372 title = re.sub(
1373 r"(?is)<span\b\s*[^>]*?>(.*?)\s*</span\s*>",
1374 lambda m: re.sub(r"\s+", " ", m.group(1)),
1375 title,
1376 )
1377 # Replace <br/> by comma space (it is used to express alternatives in some
1378 # declensions)
1379 title = re.sub(r"(?si)\s*<br\s*/?>\n*", "\n", title)
1380 # Remove divs with floatright class (generated e.g. by {{ja-kanji|...}})
1381 title = re.sub(
1382 r'(?si)<div\b[^>]*?\bclass="[^"]*?\bfloatright\b[^>]*?>'
1383 r"((<div\b(<div\b.*?</div\s*>|.)*?</div>)|.)*?"
1384 r"</div\s*>",
1385 "",
1386 title,
1387 )
1388 # Remove divs with float: attribute
1389 title = re.sub(
1390 r'(?si)<div\b[^>]*?\bstyle="[^"]*?\bfloat:[^>]*?>'
1391 r"((<div\b(<div\b.*?</div\s*>|.)*?</div>)|.)*?"
1392 r"</div\s*>",
1393 "",
1394 title,
1395 )
1396 # Remove <sup> with previewonly class (generated e.g. by {{taxlink|...}})
1397 title = re.sub(
1398 r'(?si)<sup\b[^>]*?\bclass="[^"<>]*?'
1399 r"\bpreviewonly\b[^>]*?>"
1400 r".+?</sup\s*>",
1401 "",
1402 title,
1403 )
1404 # Remove <strong class="error">...</strong>
1405 title = re.sub(
1406 r'(?si)<strong\b[^>]*?\bclass="[^"]*?\berror\b[^>]*?>'
1407 r".+?</strong\s*>",
1408 "",
1409 title,
1410 )
1411 return title
1414def clean_value(
1415 wxr: WiktextractContext, title: str, no_strip=False, no_html_strip=False
1416) -> str:
1417 """Cleans a title or value into a normal string. This should basically
1418 remove any Wikimedia formatting from it: HTML tags, templates, links,
1419 emphasis, etc. This will also merge multiple whitespaces into one
1420 normal space and will remove any surrounding whitespace."""
1421 assert isinstance(wxr, WiktextractContext)
1422 assert isinstance(title, str)
1424 global IMAGE_LINK_RE
1425 if IMAGE_LINK_RE is None:
1426 image_link_prefixes = wxr.wtp.namespace_prefixes(
1427 wxr.wtp.NAMESPACE_DATA["File"]["id"], suffix=""
1428 )
1429 IMAGE_LINK_RE = re.compile(
1430 rf"(?:\s*{'|'.join(image_link_prefixes)})\s*:", re.IGNORECASE
1431 )
1433 def repl_1(m: re.Match) -> str:
1434 return clean_value(wxr, m.group(1), no_strip=True)
1436 def repl_exturl(m: re.Match) -> str:
1437 args = re.split(r"\s+", m.group(1))
1438 i = 0
1439 while i < len(args) - 1:
1440 if not URL_STARTS_RE.match(args[i]):
1441 break
1442 i += 1
1443 return " ".join(args[i:])
1445 def repl_link(m: re.Match) -> str:
1446 before_colon = m.group(1)
1447 after_colon = m.group(3)
1448 if (
1449 before_colon is not None
1450 and IMAGE_LINK_RE.match(before_colon) is not None
1451 ):
1452 return ""
1453 if before_colon is not None and before_colon.strip(": ") in ("w", "s"):
1454 # Wikipedia or Wikisource link
1455 v = after_colon.split("|")[0]
1456 else:
1457 v = m.group(0).strip("[]").split("|")[0]
1458 return clean_value(wxr, v, no_strip=True)
1460 def repl_link_bars(m: re.Match) -> str:
1461 link = m.group(1)
1462 if IMAGE_LINK_RE.match(link) is not None:
1463 # Handle File / Image / Fichier 'links' here.
1464 if NOT_INLINE_IMG_RE.match(m.group(0)) is None and "alt" in m.group(
1465 0
1466 ):
1467 # This image should be inline, so let's print its alt text
1468 alt_m = re.search(r"\|\s*alt\s*=([^]|]+)(\||\]\])", m.group(0))
1469 if alt_m is not None: 1469 ↛ 1471line 1469 didn't jump to line 1471 because the condition on line 1469 was always true
1470 return "[Alt: " + alt_m.group(1) + "]"
1471 return ""
1472 # m.group(5) is always the last matching group because you can
1473 # only access the last matched group; the indexes don't 'grow'
1474 return clean_value(wxr, m.group(3) or "", no_strip=True)
1476 def repl_1_sup(m: re.Match) -> str:
1477 return to_superscript(clean_value(wxr, m.group(1)))
1479 def repl_1_sub(m: re.Match) -> str:
1480 return to_subscript(clean_value(wxr, m.group(1)))
1482 def repl_1_chem(m: re.Match) -> str:
1483 return to_chem(clean_value(wxr, m.group(1)))
1485 def repl_1_math(m: re.Match) -> str:
1486 v = to_math(m.group(1))
1487 # print("to_math:", ascii(v))
1488 return v
1490 def repl_1_syntaxhighlight(m: re.Match) -> str:
1491 # Content is preformatted
1492 return "`" + m.group(1).strip() + "`"
1494 title = remove_invisible_markup(title)
1495 # Change <div> and </div> to newlines. Ditto for tr, li, table, dl, ul, ol
1496 title = re.sub(r"(?si)</?(div|tr|li|table|dl|ul|ol)\b[^>]*>", "\n", title)
1497 # Change <dt>, <dd>, </dt> and </dd> into newlines;
1498 # these generate new rows/lines.
1499 title = re.sub(r"(?i)</?d[dt]\s*>", "\n", title)
1500 # Change <td> </td> to spaces. Ditto for th.
1501 title = re.sub(r"(?si)</?(td|th)\b[^>]*>", " ", title)
1502 # Change <sup> ... </sup> to ^
1503 title = re.sub(r"(?si)<sup\b[^>]*>\s*</sup\s*>", "", title)
1504 title = re.sub(r"(?si)<sup\b[^>]*>(.*?)</sup\s*>", repl_1_sup, title)
1505 # Change <sub> ... </sub> to _
1506 title = re.sub(r"(?si)<sub\b[^>]*>\s*</sub\s*>", "", title)
1507 title = re.sub(r"(?si)<sub\b[^>]*>(.*?)</sub\s*>", repl_1_sub, title)
1508 # Change <chem> ... </chem> using subscripts for digits
1509 title = re.sub(r"(?si)<chem\b[^>]*>(.*?)</chem\s*>", repl_1_chem, title)
1510 # Change <math> ... </math> using special formatting.
1511 title = re.sub(r"(?si)<math\b[^>]*>(.*?)</math\s*>", repl_1_math, title)
1512 # Change <syntaxhighlight> ... </syntaxhighlight> using special formatting.
1513 title = re.sub(
1514 r"(?si)<syntaxhighlight\b[^>]*>(.*?)" r"</syntaxhighlight\s*>",
1515 repl_1_syntaxhighlight,
1516 title,
1517 )
1518 # Remove any remaining HTML tags.
1519 if not no_html_strip:
1520 title = re.sub(r"(?s)<[/!a-zA-Z][^>]*>", "", title)
1521 title = re.sub(r"(?s)</[^>]+>", "", title)
1522 else:
1523 # Strip <noinclude/> anyway
1524 title = re.sub(r"(?si)<noinclude\s*/\s*>", "", title)
1525 # Replace [...]
1526 title = re.sub(r"(?s)\[\s*\.\.\.\s*\]", "…", title)
1527 # Remove http links in superscript
1528 title = re.sub(r"\^\(\[?(https?:)?//[^]()]+\]?\)", "", title)
1529 # Remove any edit links to local pages
1530 title = re.sub(r"\[//[^]\s]+\s+edit\s*\]", "", title)
1531 # Replace links by their text
1533 category_ns_data: NamespaceDataEntry
1534 # XXX "Category" -> config variable for portability
1535 category_ns_data = wxr.wtp.NAMESPACE_DATA.get("Category", {}) # type: ignore[typeddict-item]
1536 # Fail if we received empty dict from .get()
1537 category_ns_names = {"Category", category_ns_data["name"]} | set(
1538 category_ns_data["aliases"]
1539 )
1540 category_names_pattern = rf"(?:{'|'.join(category_ns_names)})"
1541 while True:
1542 # Links may be nested, so keep replacing until there is no more change.
1543 orig = title
1544 title = re.sub(
1545 rf"(?si)\s*\[\[\s*{category_names_pattern}\s*:\s*([^]]+?)\s*\]\]",
1546 "",
1547 title,
1548 )
1549 title = re.sub(
1550 r"(?s)\[\[\s*:?([^]|#<>:&]+?)(#[^][|<>]*?)?\]\]", repl_1, title
1551 )
1552 title = re.sub(
1553 r"(?s)\[\[\s*(([\w\d]+)\s*:)?(\s*[^][#|<>]+?)"
1554 r"(#[^][|]*?)?\|?\]\]",
1555 repl_link,
1556 title,
1557 )
1558 title = re.sub(
1559 # [[ (...) |
1560 r"(?s)\[\[\s*([^][|<>]+?)\s*"
1561 # (|((...OR[...])+))*|]]
1562 r"(\|(([^][|]|\[[^]]*\])+?))*\|*\]\]",
1563 repl_link_bars,
1564 title,
1565 )
1566 if title == orig:
1567 break
1568 # Replace remaining HTML links by the URL.
1569 while True:
1570 orig = title
1571 title = re.sub(
1572 r"\[\s*((https?:|mailto:)?//([^][]+?))\s*\]", repl_exturl, title
1573 )
1574 if title == orig:
1575 break
1577 # Remove italic and bold
1578 title = remove_italic_and_bold(title)
1580 # Replace HTML entities
1581 title = html.unescape(title)
1582 title = title.replace("\xa0", " ") # nbsp
1583 # Remove left-to-right and right-to-left, zero-with characters
1584 title = re.sub(r"[\u200e\u200f\u200b\u200d\u200c\ufeff]", "", title)
1585 # Replace whitespace sequences by a single space.
1586 # https://en.wikipedia.org/wiki/En_(typography)
1587 title = re.sub(r"[ \t\r\u2002]+", " ", title)
1588 title = re.sub(r" *\n+", "\n", title)
1589 # Eliminate spaces around ellipsis in brackets
1590 title = re.sub(r"\[\s*…\s*\]", "[…]", title)
1592 # This unicode quote seems to be used instead of apostrophe quite randomly
1593 # (about 4% of apostrophes in English entries, some in Finnish entries).
1594 # title = re.sub("\u2019", "'", title) # Note: no r"..." here!
1595 # Replace strange unicode quotes with normal quotes
1596 # title = re.sub(r"”", '"', title)
1597 # Replace unicode long dash by normal dash
1598 # title = re.sub(r"–", "-", title)
1600 # Remove whitespace before periods and commas etc
1601 # XXX we might re-enable this, now trying without as it is removing some
1602 # instances where we would want to leave the space
1603 # title = re.sub(r" ([.,;:!?)])", repl_1, title)
1604 # Strip surrounding whitespace.
1605 if not no_strip:
1606 title = title.strip()
1607 # Normalize different ways of writing accents into the NFC canonical form
1608 title = unicodedata.normalize("NFC", title)
1609 return title
1612def clean_template_args(
1613 wxr: WiktextractContext,
1614 ht: Union[TemplateArgs, TemplateParameters],
1615 no_strip=False,
1616) -> dict[Union[str, int], str]:
1617 """Cleans all values in a template argument dictionary and returns the
1618 cleaned dictionary."""
1619 assert isinstance(wxr, WiktextractContext)
1620 assert isinstance(ht, dict)
1621 return {
1622 clean_value(wxr, str(k), no_html_strip=True): clean_value(
1623 wxr, str(v), no_strip=no_strip, no_html_strip=True
1624 )
1625 for k, v in ht.items()
1626 }