Coverage for src/wiktextract/clean.py: 89%
323 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-07 00:55 +0000
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-07 00:55 +0000
1# This file contains code to clean Wiktionary annotations from a string and to
2# produce plain text from it, typically for glossary entries but this is also
3# called for various other data to produce clean strings.
4#
5# This file also contains code for cleaning qualifiers for the "tags" field.
6#
7# Copyright (c) 2018-2022 Tatu Ylonen. See file LICENSE and https://ylonen.org
9import html
10import re
11import unicodedata
12from typing import Callable, Optional, Union
14from wikitextprocessor.common import MAGIC_FIRST, MAGIC_LAST, URL_STARTS
15from wikitextprocessor.core import NamespaceDataEntry, TemplateArgs
16from wikitextprocessor.parser import TemplateParameters, balance_apostrophes
18from .wxr_context import WiktextractContext
20######################################################################
21# Cleaning values into plain text.
22######################################################################
24superscript_ht: dict[str, str] = {
25 "0": "⁰",
26 "1": "¹",
27 "2": "²",
28 "3": "³",
29 "4": "⁴",
30 "5": "⁵",
31 "6": "⁶",
32 "7": "⁷",
33 "8": "⁸",
34 "9": "⁹",
35 "+": "⁺",
36 "-": "⁻",
37 "−": "⁻",
38 "‐": "⁻",
39 "–": "⁻",
40 "—": "⁻",
41 "一": "⁻",
42 "=": "⁼",
43 "(": "⁽",
44 ")": "⁾",
45 "A": "ᴬ",
46 "B": "ᴮ",
47 "D": "ᴰ",
48 "E": "ᴱ",
49 "G": "ᴳ",
50 "H": "ᴴ",
51 "I": "ᴵ",
52 "J": "ᴶ",
53 "K": "ᴷ",
54 "L": "ᴸ",
55 "M": "ᴹ",
56 "N": "ᴺ",
57 "O": "ᴼ",
58 "P": "ᴾ",
59 "R": "ᴿ",
60 "T": "ᵀ",
61 "U": "ᵁ",
62 "V": "ⱽ",
63 "W": "ᵂ",
64 "a": "ᵃ",
65 "b": "ᵇ",
66 "c": "ᶜ",
67 "d": "ᵈ",
68 "e": "ᵉ",
69 "f": "ᶠ",
70 "g": "ᵍ",
71 "h": "ʰ",
72 "i": "ⁱ",
73 "j": "ʲ",
74 "k": "ᵏ",
75 "l": "ˡ",
76 "m": "ᵐ",
77 "n": "ⁿ",
78 "o": "ᵒ",
79 "p": "ᵖ",
80 "r": "ʳ",
81 "s": "ˢ",
82 "t": "ᵗ",
83 "u": "ᵘ",
84 "v": "ᵛ",
85 "w": "ʷ",
86 "x": "ˣ",
87 "y": "ʸ",
88 "z": "ᶻ",
89 "β": "ᵝ",
90 "γ": "ᵞ",
91 "δ": "ᵟ",
92 "θ": "ᶿ",
93 "ι": "ᶥ",
94 "φ": "ᵠ",
95 "χ": "ᵡ",
96 "∞": "\u2002᪲", # This is a KLUDGE
97}
99subscript_ht: dict[str, str] = {
100 "0": "₀",
101 "1": "₁",
102 "2": "₂",
103 "3": "₃",
104 "4": "₄",
105 "5": "₅",
106 "6": "₆",
107 "7": "₇",
108 "8": "₈",
109 "9": "₉",
110 "+": "₊",
111 "-": "₋",
112 "−": "₋",
113 "=": "₌",
114 "(": "₍",
115 ")": "₎",
116 "a": "ₐ",
117 "e": "ₑ",
118 "h": "ₕ",
119 "i": "ᵢ",
120 "j": "ⱼ",
121 "k": "ₖ",
122 "l": "ₗ",
123 "m": "ₘ",
124 "n": "ₙ",
125 "o": "ₒ",
126 "p": "ₚ",
127 "r": "ᵣ",
128 "s": "ₛ",
129 "t": "ₜ",
130 "u": "ᵤ",
131 "v": "ᵥ",
132 "x": "ₓ",
133 "ə": "ₔ",
134 "ρ": "ᵨ",
135 "φ": "ᵩ",
136 "χ": "ᵪ",
137}
140def to_superscript(text: str) -> str:
141 "Converts text to superscript."
142 if not text:
143 return ""
144 if all(x in superscript_ht for x in text):
145 return "".join(superscript_ht[x] for x in text)
146 if len(text) == 1:
147 return "^" + text
148 return "^({})".format(text)
151def to_subscript(text: str) -> str:
152 """Converts text to subscript."""
153 if not text: 153 ↛ 154line 153 didn't jump to line 154 because the condition on line 153 was never true
154 return ""
155 if all(x in subscript_ht for x in text):
156 return "".join(subscript_ht[x] for x in text)
157 if len(text) == 1: 157 ↛ 159line 157 didn't jump to line 159 because the condition on line 157 was always true
158 return "_" + text
159 return "_({})".format(text)
162def to_chem(text: str) -> str:
163 """Converts text to chemical formula, making digits subscript."""
164 return "".join(to_subscript(x) if x.isdigit() else x for x in text)
167# Mapping from Latex names to Unicode characters/strings. This is the
168# default mapping (some cases are handled specially in the code).
169math_map: dict[str, str] = {
170 # XXX should probably change greek characters to non-slanted ones?
171 "AC": "∿",
172 "APLcomment": "⍝",
173 "APLdownarrowbox": "⍗",
174 "APLinput": "⍞",
175 "APLinv": "⌹",
176 "APLleftarrowbox": "⍇",
177 "APLlog": "⍟",
178 "APLrightarrowbox": "⍈",
179 "APLuparrowbox": "⍐",
180 "Angstroem": "Å",
181 "Bot": "⫫",
182 "Box": "□",
183 "Bumpeq": "≎",
184 "CIRCLE": "●",
185 "Cap": "⋒",
186 "CapitalDifferentialD": "ⅅ",
187 "CheckedBox": "☑",
188 "Circle": "○",
189 "Coloneqq": "⩴",
190 "ComplexI": "ⅈ",
191 "ComplexJ": "ⅉ",
192 "Cup": "⋓",
193 "Delta": "Δ",
194 "Diamond": "◇",
195 "Diamondblack": "◆",
196 "Diamonddot": "⟐",
197 "DifferentialD": "ⅆ",
198 "Digamma": "Ϝ",
199 "Doteq": "≑",
200 "DownArrowBar": "⤓",
201 "DownLeftTeeVector": "⥞",
202 "DownLeftVectorBar": "⥖",
203 "DownRightTeeVector": "⥟",
204 "DownRightVectorBar": "⥗",
205 "Downarrow": "⇓",
206 "Equal": "⩵",
207 "Euler": "Ɛ",
208 "ExponentialE": "ⅇ",
209 "ExponetialE": "ⅇ",
210 "Finv": "Ⅎ",
211 "Gamma": "Γ",
212 "Im": "ℑ",
213 "Join": "⨝",
214 "Koppa": "Ϟ",
215 "LEFTCIRCLE": "◖",
216 "LEFTcircle": "◐",
217 "LHD": "◀",
218 "LVec": "x⃖",
219 "Lambda": "Λ",
220 "Lbag": "⟅",
221 "LeftArrowBar": "⇤",
222 "LeftDownTeeVector": "⥡",
223 "LeftDownVectorBar": "⥙",
224 "LeftTeeVector": "⥚",
225 "LeftTriangleBar": "⧏",
226 "LeftUpTeeVector": "⥠",
227 "LeftUpVectorBar": "⥘",
228 "LeftVectorBar": "⥒",
229 "Leftarrow": "⇐",
230 "Leftrightarrow": "⇔",
231 "Lleftarrow": "⇚",
232 "Longleftarrow": "⟸",
233 "Longleftrightarrow": "⟺",
234 "Longmapsfrom": "⟽",
235 "Longmapsto": "⟾",
236 "Longrightarrow": "⟹",
237 "Lparen": "⦅",
238 "Lsh": "↰",
239 "MapsDown": "↧",
240 "MapsUp": "↥",
241 "Mapsfrom": "⤆",
242 "Mapsto": "⤇",
243 "Micro": "µ",
244 "Nearrow": "⇗",
245 "NestedGreaterGreater": "⪢",
246 "NestedLessLess": "⪡",
247 "NotGreaterLess": "≹",
248 "NotGreaterTilde": "≵",
249 "NotLessTilde": "≴",
250 "Nwarrow": "⇖",
251 "Omega": "Ω",
252 "Phi": "Φ",
253 "Pi": "Π",
254 "Proportion": "∷",
255 "Psi": "Ψ",
256 "Qoppa": "Ϙ",
257 "RHD": "▶",
258 "RIGHTCIRCLE": "◗",
259 "RIGHTcircle": "◑",
260 "Rbag": "⟆",
261 "Re": "ℜ",
262 "RightArrowBar": "⇥",
263 "RightDownTeeVector": "⥝",
264 "RightDownVectorBar": "⥕",
265 "RightTeeVector": "⥛",
266 "RightTriangleBar": "⧐",
267 "RightUpTeeVector": "⥜",
268 "RightUpVectorBar": "⥔",
269 "RightVectorBar": "⥓",
270 "Rightarrow": "⇒",
271 "Rparen": "⦆",
272 "Rrightarrow": "⇛",
273 "Rsh": "↱",
274 "S": "§",
275 "Same": "⩶",
276 "Sampi": "Ϡ",
277 "Searrow": "⇘",
278 "Sigma": "Σ",
279 "Square": "☐",
280 "Stigma": "Ϛ",
281 "Subset": "⋐",
282 "Sun": "☉",
283 "Supset": "⋑",
284 "Swarrow": "⇙",
285 "Theta": "Θ",
286 "Top": "⫪",
287 "UpArrowBar": "⤒",
288 "Uparrow": "⇑",
289 "Updownarrow": "⇕",
290 "Upsilon": "Υ",
291 "VDash": "⊫",
292 "VERT": "⦀",
293 "Vdash": "⊩",
294 "Vert": "‖",
295 "Vvdash": "⊪",
296 "XBox": "☒",
297 "Xi": "Ξ",
298 "Yup": "⅄",
299 "_": "_",
300 "aleph": "א",
301 "alpha": "α",
302 "amalg": "⨿",
303 "anchor": "⚓",
304 "angle": "∠",
305 "approx": "≈",
306 "approxeq": "≊",
307 "aquarius": "♒",
308 "arg": "arg",
309 "aries": "♈",
310 "arrowbullet": "➢",
311 "ast": "∗",
312 "asymp": "≍",
313 "backepsilon": "϶",
314 "backprime": "‵",
315 "backsim": "∽",
316 "backsimeq": "⋍",
317 "backslash": "",
318 "ballotx": "✗",
319 "barin": "⋶",
320 "barleftharpoon": "⥫",
321 "barrightharpoon": "⥭",
322 "barwedge": "⊼",
323 "because": "∵",
324 "beta": "β",
325 "beth": "ב",
326 "between": "≬",
327 "bigcap": "∩",
328 "bigcup": "∪",
329 "biginterleave": "⫼",
330 "bigodot": "⨀",
331 "bigoplus": "⨁",
332 "bigotimes": "⨂",
333 "bigsqcap": "⨅",
334 "bigsqcup": "⨆",
335 "bigstar": "★",
336 "bigtriangledown": "▽",
337 "bigtriangleup": "△",
338 "biguplus": "⨄",
339 "bigvee": "∨",
340 "bigwedge": "∧",
341 "bij": "⤖",
342 "biohazard": "☣",
343 "blacklozenge": "⧫",
344 "blacksmiley": "☻",
345 "blacksquare": "■",
346 "blacktriangledown": "▾",
347 "blacktriangleleft": "◂",
348 "blacktriangleright": "▸",
349 "blacktriangleup": "▴",
350 "bot": "⊥",
351 "bowtie": "⋈",
352 "boxast": "⧆",
353 "boxbar": "◫",
354 "boxbox": "⧈",
355 "boxbslash": "⧅",
356 "boxcircle": "⧇",
357 "boxdot": "⊡",
358 "boxminus": "⊟",
359 "boxplus": "⊞",
360 "boxslash": "⧄",
361 "boxtimes": "⊠",
362 "bullet": "•",
363 "bumpeq": "≏",
364 "cancer": "♋",
365 "cap": "∩",
366 "capricornus": "♑",
367 "capwedge": "⩄",
368 "cat": "⁀",
369 "cdot": "·",
370 "cdots": "⋯",
371 "cent": "¢",
372 "checkmark": "✓",
373 "chi": "χ",
374 "circ": "∘",
375 "circeq": "≗",
376 "circlearrowleft": "↺",
377 "circlearrowright": "↻",
378 "circledR": "®",
379 "circledast": "⊛",
380 "circledbslash": "⦸",
381 "circledcirc": "⊚",
382 "circleddash": "⊝",
383 "circledgtr": "⧁",
384 "circledless": "⧀",
385 "clubsuit": "♣",
386 "colon": ":",
387 "coloneq": "≔",
388 "complement": "∁",
389 "cong": "≅",
390 "coprod": "∐",
391 "corresponds": "≙",
392 "cup": "∪",
393 "curlyeqprec": "⋞",
394 "curlyeqsucc": "⋟",
395 "curlyvee": "⋎",
396 "curlywedge": "⋏",
397 "curvearrowleft": "↶",
398 "curvearrowright": "↷",
399 "dagger": "†",
400 "daleth": "ד",
401 "dashleftarrow": "⇠",
402 "dashrightarrow": "⇢",
403 "dashv": "⊣",
404 "ddagger": "‡",
405 "delta": "δ",
406 "diameter": "∅",
407 "diamond": "⋄",
408 "diamondsuit": "♢",
409 "digamma": "ϝ",
410 "div": "÷",
411 "divideontimes": "⋇",
412 "dlsh": "↲",
413 "dot\\bigvee": "⩒",
414 "dot\\cap": "⩀",
415 "dot\\cup": "⊍",
416 "dot\\lor": "⩒",
417 "dot\\vee": "⩒",
418 "doteq": "≐",
419 "dotplus": "∔",
420 "dots": "…",
421 "doublebarwedge": "⩞",
422 "downarrow": "↓",
423 "downdownarrows": "⇊",
424 "downdownharpoons": "⥥",
425 "downharpoonleft": "⇃",
426 "downharpoonright": "⇂",
427 "downuparrows": "⇵",
428 "downupharpoons": "⥯",
429 "drsh": "↳",
430 "dsub": "⩤",
431 "earth": "♁",
432 "eighthnote": "♪",
433 "ell": "ℓ",
434 "emptyset": "∅",
435 "epsilon": "ϵ",
436 "eqcirc": "≖",
437 "eqcolon": "∹",
438 "eqsim": "≂",
439 "eqslantgtr": "⪖",
440 "eqslantless": "⪕",
441 "equiv": "≡",
442 "eta": "η",
443 "eth": "ð",
444 "exists": "∃",
445 "fallingdotseq": "≒",
446 "fcmp": "⨾",
447 "female": "♀",
448 "ffun": "⇻",
449 "finj": "⤕",
450 "fint": "⨏",
451 "flat": "♭",
452 "footnotesize": "",
453 "forall": "∀",
454 "fourth": "⁗",
455 "frown": "⌢",
456 "frownie": "☹",
457 "gamma": "γ",
458 "ge": ">",
459 "gemini": "♊",
460 "geq": "≥",
461 "geqq": "≧",
462 "geqslant": "⩾",
463 "gg": "≫",
464 "ggcurly": "⪼",
465 "ggg": "⋙",
466 "gimel": "ג",
467 "gnapprox": "⪊",
468 "gneq": "⪈",
469 "gneqq": "≩",
470 "gnsim": "⋧",
471 "gtrapprox": "⪆",
472 "gtrdot": "⋗",
473 "gtreqless": "⋛",
474 "gtreqqless": "⪌",
475 "gtrless": "≷",
476 "gtrsim": "≳",
477 "hash": "⋕",
478 "heartsuit": "♡",
479 "hookleftarrow": "↩",
480 "hookrightarrow": "↪",
481 "hslash": "ℏ",
482 "iddots": "⋰",
483 "iff": "⟺",
484 "iiiint": "⨌",
485 "iiint": "∭",
486 "iint": "∬",
487 "imath": "ı",
488 "implies": "⟹",
489 "in": "∈",
490 "infty": "∞",
491 "int": "∫",
492 "intercal": "⊺",
493 "interleave": "⫴",
494 "invamp": "⅋",
495 "invdiameter": "⍉",
496 "invneg": "⌐",
497 "iota": "ι",
498 "jmath": "ȷ",
499 "jupiter": "♃",
500 "kappa": "κ",
501 "koppa": "ϟ",
502 "lambda": "λ",
503 "land": "∧",
504 "lang": "⟪",
505 "langle": "⟨",
506 "large": "",
507 "lblot": "⦉",
508 "lbrace": "{",
509 "lbrack": "[",
510 "lceil": "⌈",
511 "ldots": "…",
512 "le": "<",
513 "leadsto": "⤳",
514 "leftarrow": "←",
515 "leftarrowtail": "↢",
516 "leftarrowtriangle": "⇽",
517 "leftbarharpoon": "⥪",
518 "leftharpoondown": "↽",
519 "leftharpoonup": "↼",
520 "leftleftarrows": "⇇",
521 "leftleftharpoons": "⥢",
522 "leftmoon": "☾",
523 "leftrightarrow": "↔",
524 "leftrightarrows": "⇆",
525 "leftrightarrowtriangle": "⇿",
526 "leftrightharpoon": "⥊",
527 "leftrightharpoondown": "⥐",
528 "leftrightharpoons": "⇋",
529 "leftrightharpoonup": "⥎",
530 "leftrightsquigarrow": "↭",
531 "leftslice": "⪦",
532 "leftsquigarrow": "⇜",
533 "leftthreetimes": "⋋",
534 "leftupdownharpoon": "⥑",
535 "leo": "♌",
536 "leq": "≤",
537 "leqq": "≦",
538 "leqslant": "⩽",
539 "lessapprox": "⪅",
540 "lessdot": "⋖",
541 "lesseqgtr": "⋚",
542 "lesseqqgtr": "⪋",
543 "lessgtr": "≶",
544 "lessim": "≲",
545 "lesssim": "≲",
546 "lfloor": "⌊",
547 "lgroup": "⟮",
548 "lhd": "◁",
549 "libra": "♎",
550 "lightning": "↯",
551 "limg": "⦇",
552 "ll": "≪",
553 "llbracket": "⟦",
554 "llcorner": "⌞",
555 "llcurly": "⪻",
556 "lll": "⋘",
557 "lnapprox": "⪉",
558 "lneq": "⪇",
559 "lneqq": "≨",
560 "lnot": "¬",
561 "lnsim": "⋦",
562 "longleftarrow": "⟵",
563 "longleftrightarrow": "⟷",
564 "longmapsfrom": "⟻",
565 "longmapsto": "⟼",
566 "longrightarrow": "⟶",
567 "looparrowleft": "↫",
568 "looparrowright": "↬",
569 "lor": "∨",
570 "lozenge": "◊",
571 "lrcorner": "⌟",
572 "ltimes": "⋉",
573 "male": "♂",
574 "maltese": "✠",
575 "mapsfrom": "↤",
576 "mapsto": "↦",
577 "measuredangle": "∡",
578 "medbullet": "⚫",
579 "medcirc": "⚪",
580 "mercury": "☿",
581 "mho": "℧",
582 "mid": "∣",
583 "mlcp": "⫛",
584 "mod": " mod ",
585 "models": "⊧",
586 "mp": "∓",
587 "mu": "μ",
588 "multimap": "⊸",
589 "multimapboth": "⧟",
590 "multimapdotbothA": "⊶",
591 "multimapdotbothB": "⊷",
592 "multimapinv": "⟜",
593 "nLeftarrow": "⇍",
594 "nLeftrightarrow": "⇎",
595 "nRightarrow": "⇏",
596 "nVDash": "⊯",
597 "nVdash": "⊮",
598 "nabla": "∇",
599 "napprox": "≉",
600 "natural": "♮",
601 "ncong": "≇",
602 "nearrow": "↗",
603 "neg": "¬",
604 "neptune": "♆",
605 "neq": "≠",
606 "nequiv": "≢",
607 "nexists": "∄",
608 "ngeq": "≱",
609 "ngtr": "≯",
610 "ni": "∋",
611 "nleftarrow": "↚",
612 "nleftrightarrow": "↮",
613 "nleq": "≰",
614 "nless": "≮",
615 "nmid": "∤",
616 "nni": "∌",
617 "normalsize": "",
618 "not\\in": "∉",
619 "not\\ni": "∌",
620 "not\\preceq": "⋠",
621 "not\\subset": "⊄",
622 "not\\subseteq": "⊈",
623 "not\\succeq": "⋡",
624 "not\\supset": "⊅",
625 "not\\supseteq": "⊉",
626 "not\\trianglelefteq": "⋬",
627 "not\\trianglerighteq": "⋭",
628 "not\\vartriangleleft": "⋪",
629 "not\\vartriangleright": "⋫",
630 "notasymp": "≭",
631 "notbackslash": "⍀",
632 "notin": "∉",
633 "notslash": "⌿",
634 "nparallel": "∦",
635 "nprec": "⊀",
636 "npreceq": "⋠",
637 "nrightarrow": "↛",
638 "nsim": "≁",
639 "nsimeq": "≄",
640 "nsqsubseteq": "⋢",
641 "nsqsupseteq": "⋣",
642 "nsubset": "⊄",
643 "nsubseteq": "⊈",
644 "nsucc": "⊁",
645 "nsucceq": "⋡",
646 "nsupset": "⊅",
647 "nsupseteq": "⊉",
648 "ntriangleleft": "⋪",
649 "ntrianglelefteq": "⋬",
650 "ntriangleright": "⋫",
651 "ntrianglerighteq": "⋭",
652 "nu": "ν",
653 "nvDash": "⊭",
654 "nvdash": "⊬",
655 "nwarrow": "↖",
656 "odot": "⊙",
657 "oiiint": "∰",
658 "oiint": "∯",
659 "oint": "∮",
660 "ointctrclockwise": "∳",
661 "omega": "ω",
662 "ominus": "⊖",
663 "oplus": "⊕",
664 "oslash": "⊘",
665 "otimes": "⊗",
666 "over": "/",
667 "overbrace": "⏞",
668 "overleftrightarrow": "x⃡",
669 "overparen": "⏜",
670 "overset?=": "≟",
671 "overset{?}{=}": "≟",
672 "overset{\\operatorname{def}}{=}": "≝",
673 "parallel": "∥",
674 "partial": "∂",
675 "pencil": "✎",
676 "perp": "⊥",
677 "pfun": "⇸",
678 "phi": "ϕ",
679 "pi": "π",
680 "pinj": "⤔",
681 "pisces": "♓",
682 "pitchfork": "⋔",
683 "pluto": "♇",
684 "pm": "±",
685 "pointright": "☞",
686 "pounds": "£",
687 "prec": "≺",
688 "precapprox": "⪷",
689 "preccurlyeq": "≼",
690 "preceq": "⪯",
691 "preceqq": "⪳",
692 "precnapprox": "⪹",
693 "precnsim": "⋨",
694 "precsim": "≾",
695 "prime": "′",
696 "prod": "∏",
697 "propto": "∝",
698 "psi": "ψ",
699 "psur": "⤀",
700 "qoppa": "ϙ",
701 "quad": " ",
702 "quarternote": "♩",
703 "radiation": "☢",
704 "rang": "⟫",
705 "rangle": "⟩",
706 "rarr": "→",
707 "rblot": "⦊",
708 "rbrace": "}",
709 "rbrack": "]",
710 "rceil": "⌉",
711 "recycle": "♻",
712 "rfloor": "⌋",
713 "rgroup": "⟯",
714 "rhd": "▷",
715 "rho": "ρ",
716 "rightangle": "∟",
717 "rightarrow": "→",
718 "rightarrowtail": "↣",
719 "rightarrowtriangle": "⇾",
720 "rightbarharpoon": "⥬",
721 "rightharpoondown": "⇁",
722 "rightharpoonup": "⇀",
723 "rightleftarrows": "⇄",
724 "rightleftharpoon": "⥋",
725 "rightleftharpoons": "⇌",
726 "rightmoon": "☽",
727 "rightrightarrows": "⇉",
728 "rightrightharpoons": "⥤",
729 "rightslice": "⪧",
730 "rightsquigarrow": "⇝",
731 "rightthreetimes": "⋌",
732 "rightupdownharpoon": "⥏",
733 "rimg": "⦈",
734 "risingdotseq": "≓",
735 "rrbracket": "⟧",
736 "rsub": "⩥",
737 "rtimes": "⋊",
738 "sagittarius": "♐",
739 "sampi": "ϡ",
740 "saturn": "♄",
741 "scorpio": "♏",
742 "scriptsize": "",
743 "searrow": "↘",
744 "second": "″",
745 "setminus": "⧵",
746 "sharp": "♯",
747 "sigma": "σ",
748 "sim": "∼",
749 "simeq": "≃",
750 "sixteenthnote": "♬",
751 "skull": "☠",
752 "slash": "∕",
753 "small": "",
754 "smallsetminus": "∖",
755 "smalltriangledown": "▿",
756 "smalltriangleleft": "◃",
757 "smalltriangleright": "▹",
758 "smalltriangleup": "▵",
759 "smile": "⌣",
760 "smiley": "☺",
761 "spadesuit": "♠",
762 "spddot": "¨",
763 "sphat": "^",
764 "sphericalangle": "∢",
765 "spot": "⦁",
766 "sptilde": "~",
767 "sqcap": "⊓",
768 "sqcup": "⊔",
769 "sqint": "⨖",
770 "sqrt": "√", # ∛ ∜ - partly special handling below
771 "sqrt[3]": "∛",
772 "sqrt[4]": "∜",
773 "sqsubset": "⊏",
774 "sqsubseteq": "⊑",
775 "sqsupset": "⊐",
776 "sqsupseteq": "⊒",
777 "square": "□",
778 "sslash": "⫽",
779 "star": "⋆",
780 "steaming": "☕",
781 "stigma": "ϛ",
782 "strictfi": "⥼",
783 "strictif": "⥽",
784 "subset": "⊂",
785 "subseteq": "⊆",
786 "subseteqq": "⫅",
787 "subsetneq": "⊊",
788 "subsetneqq": "⫋",
789 "succ": "≻",
790 "succapprox": "⪸",
791 "succcurlyeq": "≽",
792 "succeq": "⪰",
793 "succeqq": "⪴",
794 "succnapprox": "⪺",
795 "succnsim": "⋩",
796 "succsim": "≿",
797 "sum": "∑",
798 "sun": "☼",
799 "supset": "⊃",
800 "supseteq": "⊇",
801 "supseteqq": "⫆",
802 "supsetneq": "⊋",
803 "supsetneqq": "⫌",
804 "swarrow": "↙",
805 "swords": "⚔",
806 "talloblong": "⫾",
807 "tau": "τ",
808 "taurus": "♉",
809 "tcohm": "Ω",
810 "textbackslash": "\\",
811 "textbar": "|",
812 "textbullet": "•",
813 "textgreater": ">",
814 "textless": "<",
815 "textprime": "′",
816 "therefore": "∴",
817 "theta": "θ",
818 "third": "‴",
819 "times": "×",
820 "tiny": "",
821 "to": "→",
822 "top": "⊤",
823 "triangle": "∆",
824 "trianglelefteq": "⊴",
825 "triangleq": "≜",
826 "trianglerighteq": "⊵",
827 "twoheadleftarrow": "↞",
828 "twoheadrightarrow": "↠",
829 "twonotes": "♫",
830 "ulcorner": "⌜",
831 "underbar": " ̱",
832 "underbrace": "⏟",
833 "underleftarrow": "x⃮",
834 "underline": " ̲",
835 "underparen": "⏝",
836 "underrightarrow": "x⃯",
837 "uparrow": "↑",
838 "updownarrow": "↕",
839 "updownarrows": "⇅",
840 "updownharpoons": "⥮",
841 "upharpoonleft": "↿",
842 "upharpoonright": "↾",
843 "uplus": "⊎",
844 "upsilon": "υ",
845 "upuparrows": "⇈",
846 "upupharpoons": "⥣",
847 "uranus": "♅",
848 "urcorner": "⌝",
849 "utilde": " ̰",
850 "vDash": "⊨",
851 "varbeta": "β",
852 "varclubsuit": "♧",
853 "vardiamondsuit": "♦",
854 "varepsilon": "ε",
855 "varheartsuit": "♥",
856 "varkappa": "ϰ",
857 "varnothing": "∅",
858 "varointclockwise": "∲",
859 "varphi": "φ",
860 "varpi": "ϖ",
861 "varprod": "⨉",
862 "varrho": "ϱ",
863 "varsigma": "ς",
864 "varspadesuit": "♤",
865 "vartheta": "θ",
866 "vartriangleleft": "⊲",
867 "vartriangleright": "⊳",
868 "vdash": "⊢",
869 "vdots": "⋮",
870 "vee": "∨",
871 "veebar": "⊻",
872 "vert": "|",
873 "virgo": "♍",
874 "warning": "⚠",
875 "wasylozenge": "⌑",
876 "wedge": "∧",
877 "widehat=": "≙",
878 "widehat{=}": "≙",
879 "wp": "℘",
880 "wr": "≀",
881 "xi": "ξ",
882 "yen": "¥",
883 "yinyang": "☯",
884 "zcmp": "⨟",
885 "zeta": "ζ",
886 "zhide": "⧹",
887 "zpipe": "⨠",
888 "zproject": "⨡",
889 "|": "‖",
890 # Accents XXX these really should be handled specially with diacritics
891 # after argument
892 "acute": "́",
893 "bar": "̄",
894 "breve": "̆",
895 "check": "̌",
896 "ddddot": "⃜",
897 "dddot": "⃛",
898 "ddot": "̈",
899 "ddots": "⋱",
900 "dot": "̇",
901 "grave": "̀",
902 "hat": "̂",
903 "lvec": "⃐",
904 "mathring": "̊",
905 "not": "̸",
906 "overline": "◌̅",
907 "tilde": "̃",
908 "vec": "⃑",
909 # Some ignored operators
910 "bigl": "",
911 "bigr": "",
912 "left": "",
913 "right": "",
914 "style": "",
915 "textstyle": "",
916 "mathrm": "",
917}
919mathcal_map: dict[str, str] = {
920 "A": "𝒜",
921 "B": "ℬ",
922 "C": "𝒞",
923 "D": "𝒟",
924 "E": "ℰ",
925 "F": "ℱ",
926 "G": "𝒢",
927 "H": "ℋ",
928 "I": "ℐ",
929 "J": "𝒥",
930 "K": "𝒦",
931 "L": "ℒ",
932 "M": "ℳ",
933 "N": "𝒩",
934 "O": "𝒪",
935 "P": "𝒫",
936 "Q": "𝒬",
937 "R": "ℛ",
938 "S": "𝒮",
939 "T": "𝒯",
940 "U": "𝒰",
941 "V": "𝒱",
942 "W": "𝒲",
943 "X": "𝒳",
944 "Y": "𝒴",
945 "Z": "𝒵",
946 "a": "𝒶",
947 "b": "𝒷",
948 "c": "𝒸",
949 "d": "𝒹",
950 "e": "ℯ",
951 "f": "𝒻",
952 "g": "ℊ",
953 "h": "𝒽",
954 "i": "𝒾",
955 "j": "𝒿",
956 "k": "𝓀",
957 "l": "𝓁",
958 "m": "𝓂",
959 "n": "𝓃",
960 "o": "ℴ",
961 "p": "𝓅",
962 "q": "𝓆",
963 "r": "𝓇",
964 "s": "𝓈",
965 "t": "𝓉",
966 "u": "𝓊",
967 "v": "𝓋",
968 "w": "𝓌",
969 "x": "𝓍",
970 "y": "𝓎",
971 "z": "𝓏",
972}
974mathfrak_map: dict[str, str] = {
975 "A": "𝔄",
976 "B": "𝔅",
977 "C": "ℭ",
978 "D": "𝔇",
979 "E": "𝔈",
980 "F": "𝔉",
981 "G": "𝔊",
982 "H": "ℌ",
983 "J": "𝔍",
984 "K": "𝔎",
985 "L": "𝔏",
986 "M": "𝔐",
987 "N": "𝔑",
988 "O": "𝔒",
989 "P": "𝔓",
990 "Q": "𝔔",
991 "S": "𝔖",
992 "T": "𝔗",
993 "U": "𝔘",
994 "V": "𝔙",
995 "W": "𝔚",
996 "X": "𝔛",
997 "Y": "𝔜",
998 "Z": "ℨ",
999}
1001mathbb_map: dict[str, str] = {
1002 "A": "𝔸",
1003 "B": "𝔹",
1004 "C": "ℂ",
1005 "D": "𝔻",
1006 "E": "𝔼",
1007 "F": "𝔽",
1008 "G": "𝔾",
1009 "H": "ℍ",
1010 "I": "𝕀",
1011 "J": "𝕁",
1012 "K": "𝕂",
1013 "L": "𝕃",
1014 "M": "𝕄",
1015 "N": "ℕ",
1016 "O": "𝕆",
1017 "P": "ℙ",
1018 "Q": "ℚ",
1019 "R": "ℝ",
1020 "S": "𝕊",
1021 "T": "𝕋",
1022 "U": "𝕌",
1023 "V": "𝕍",
1024 "W": "𝕎",
1025 "X": "𝕏",
1026 "Y": "𝕐",
1027 "Z": "ℤ",
1028 "a": "𝕒",
1029 "b": "𝕓",
1030 "c": "𝕔",
1031 "d": "𝕕",
1032 "e": "𝕖",
1033 "f": "𝕗",
1034 "g": "𝕘",
1035 "h": "𝕙",
1036 "i": "𝕚",
1037 "j": "𝕛",
1038 "k": "𝕜",
1039 "l": "𝕝",
1040 "m": "𝕞",
1041 "n": "𝕟",
1042 "o": "𝕠",
1043 "p": "𝕡",
1044 "q": "𝕢",
1045 "r": "𝕣",
1046 "s": "𝕤",
1047 "t": "𝕥",
1048 "u": "𝕦",
1049 "v": "𝕧",
1050 "w": "𝕨",
1051 "x": "𝕩",
1052 "y": "𝕪",
1053 "z": "𝕫",
1054 "pi": "ℼ",
1055 "gamma": "ℽ",
1056 "Gamma": "ℾ",
1057 "Pi": "ℿ",
1058 "Sigma": "⅀",
1059 "0": "𝟘",
1060 "1": "𝟙",
1061 "2": "𝟚",
1062 "3": "𝟛",
1063 "4": "𝟜",
1064 "5": "𝟝",
1065 "6": "𝟞",
1066 "7": "𝟟",
1067 "8": "𝟠",
1068 "9": "𝟡",
1069}
1072def mathcal_fn(text: str) -> str:
1073 return "".join(mathcal_map.get(x, x) for x in text)
1076def mathfrak_fn(text: str) -> str:
1077 return "".join(mathfrak_map.get(x, x) for x in text)
1080def mathbb_fn(text: str) -> str:
1081 return "".join(mathbb_map.get(x, x) for x in text)
1084def to_math(text: str) -> str:
1085 """Converts a mathematical formula to ASCII."""
1086 # print("to_math: {!r}".format(text))
1087 magic_vec: list[str] = []
1089 def expand(text: str) -> str:
1090 while True:
1091 orig = text
1092 # formatting with {:c} converts input into character
1093 text = re.sub(
1094 r"[{:c}-{:c}]".format(MAGIC_FIRST, MAGIC_LAST),
1095 lambda m: magic_vec[ord(m.group(0)) - MAGIC_FIRST],
1096 text,
1097 )
1098 if text == orig:
1099 break
1100 return text
1102 def recurse(text: str) -> str:
1103 def math_magic(
1104 text: str, left: str, right: str, fn: Callable[[str], str]
1105 ) -> str:
1106 regexp_str = r"{}([^{}{}]+){}".format(
1107 re.escape(left),
1108 re.escape(left),
1109 re.escape(right),
1110 re.escape(right),
1111 )
1112 regexp = re.compile(regexp_str)
1114 def repl(m: re.Match) -> str:
1115 magic = chr(MAGIC_FIRST + len(magic_vec))
1116 t = fn(m.group(1)).strip()
1117 magic_vec.append(t)
1118 return magic
1120 while True:
1121 orig = text
1122 text = re.sub(regexp, repl, text)
1123 if text == orig:
1124 break
1125 return text
1127 def expand_group(v: str) -> str:
1128 fn: Optional[Callable[[str], str]] = None
1129 if re.match(r"\\mathcal\b", v):
1130 fn = mathcal_fn
1131 v = v[8:].strip()
1132 elif re.match(r"\\mathfrak\b", v):
1133 fn = mathfrak_fn
1134 v = v[9:].strip()
1135 elif re.match(r"\\mathbb\b", v):
1136 fn = mathbb_fn
1137 v = v[7:]
1138 elif re.match(r"\\(begin|end)\b", v): 1138 ↛ 1139line 1138 didn't jump to line 1139 because the condition on line 1138 was never true
1139 v = "" # Skip
1140 elif re.match(r"\\text\b", v): 1140 ↛ 1141line 1140 didn't jump to line 1141 because the condition on line 1140 was never true
1141 v = v[5:]
1142 elif re.match(r"\\pmod\b", v): 1142 ↛ 1143line 1142 didn't jump to line 1143 because the condition on line 1142 was never true
1143 v = v[5:].strip()
1144 v = "(mod " + expand_group(v) + ")"
1145 elif re.match(r"\\sqrt\[", v): 1145 ↛ 1146line 1145 didn't jump to line 1146 because the condition on line 1145 was never true
1146 a = v[6:-1].strip()
1147 if a == "2":
1148 v = "√"
1149 elif a == "3":
1150 v = "∛"
1151 elif a == "4":
1152 v = "∜"
1153 else:
1154 v = to_superscript(a) + "√"
1155 elif re.match(r"\\sqrt($|[0-9]|\b)", v): 1155 ↛ 1156line 1155 didn't jump to line 1156 because the condition on line 1155 was never true
1156 v = "√"
1157 elif re.match(r"\\(frac|binom)($|[0-9]|\b)", v):
1158 m = re.match(
1159 r"\\(frac|binom)\s*(\\[a-zA-Z]+|\\.|.)\s*"
1160 r"(\\[a-zA-Z]+|\\.|.)$",
1161 v,
1162 )
1163 if not m: 1163 ↛ 1164line 1163 didn't jump to line 1164 because the condition on line 1163 was never true
1164 print("MATH FRAC/BINOM ERROR: {!r}".format(v))
1165 return v
1166 op, a, b = m.groups()
1167 a = expand_group(a).strip()
1168 b = expand_group(b).strip()
1169 if len(a) > 1:
1170 a = "(" + a + ")"
1171 if len(b) > 1:
1172 b = "(" + b + ")"
1173 if op == "frac": 1173 ↛ 1175line 1173 didn't jump to line 1175 because the condition on line 1173 was always true
1174 v = a + "/" + b
1175 elif op == "binom":
1176 v = "binom({}, {})".format(a, b)
1177 else:
1178 # Should never get here
1179 v = "{}({})".format(op, v)
1180 elif v.startswith("_"):
1181 fn = to_subscript
1182 v = v[1:]
1183 elif v.startswith("^"):
1184 fn = to_superscript
1185 v = v[1:]
1186 if v.startswith("\\"):
1187 mapped = math_map.get(v[1:].strip())
1188 if mapped is None:
1189 if v[1:].strip().isalnum(): 1189 ↛ 1192line 1189 didn't jump to line 1192 because the condition on line 1189 was always true
1190 v = " " + v[1:].strip() + " "
1191 else:
1192 v = v[1:].strip()
1193 else:
1194 v = mapped
1195 elif v.isspace() or v in ("&",): # Ignore certain special chars 1195 ↛ 1196line 1195 didn't jump to line 1196 because the condition on line 1195 was never true
1196 v = ""
1197 if fn is not None:
1198 v = expand(v)
1199 v = fn(v)
1200 v = expand(v)
1201 return v
1203 parts: list[str] = []
1204 while True:
1205 orig = text
1206 text = math_magic(text, "{", "}", recurse)
1207 if text == orig:
1208 break
1209 for m in re.finditer(
1210 r"\s+|"
1211 r"\\frac\s*(\\[a-zA-Z]+|\\.|.)\s*"
1212 r"(\\dot\\(bigvee|cup|cap|lor|vee)|"
1213 r"\\not\\(subset|supset|subseteq|supseteq|in|ni|"
1214 r"preceq|succeq|vartrianglelefteq|"
1215 r"vartrianglerighteq|trianglelefteq|"
1216 r"trianglerighteq)|"
1217 r"\\widehat\{=\}|\\widehat=|"
1218 r"\\overset\{?\}\{=\}|"
1219 r"\\overset\?=|"
1220 r"\\overset\{\\operatorname\{def\}\}\{=\}|"
1221 r"\\[a-zA-Z]+|\\.|.)|"
1222 r"(\\(mathcal|mathfrak|mathbb|text|begin|end|pmod)"
1223 r"\b\s*|"
1224 r"\\sqrt\b(\[\d+\])?)?"
1225 r"[_^]?(\\[a-zA-Z]+\s*|\\.|\w+|.)",
1226 text,
1227 ):
1228 v = m.group(0).strip()
1229 if not v:
1230 continue
1231 v = expand_group(v)
1232 if v: 1232 ↛ 1209line 1232 didn't jump to line 1209 because the condition on line 1232 was always true
1233 if (
1234 parts and parts[-1][-1].isalpha() and v[0] in "0123456789"
1235 ) or (
1236 parts
1237 and parts[-1][-1] in "0123456789"
1238 and v[0] in "0123456789"
1239 ):
1240 v = " " + v
1241 parts.append(v)
1243 text = "".join(parts)
1244 return text
1246 text = recurse(text)
1247 # print("math text final: {!r}".format(text))
1248 return text
1251def bold_follows(parts: list[str], i: int) -> bool:
1252 """Checks if there is a bold (''') in parts after parts[i]. We allow
1253 intervening italics ('')."""
1254 parts = parts[i + 1 :]
1255 for p in parts: 1255 ↛ 1260line 1255 didn't jump to line 1260 because the loop on line 1255 didn't complete
1256 if not p.startswith("''"):
1257 continue
1258 if p.startswith("'''"): 1258 ↛ 1255line 1258 didn't jump to line 1255 because the condition on line 1258 was always true
1259 return True
1260 return False
1263def remove_italic_and_bold(text: str) -> str:
1264 """Based on token_iter in wikitextprocessor"""
1265 assert isinstance(text, str)
1266 lines = re.split(r"(\n+)", text) # Lines and separators
1267 parts_re = re.compile(r"(''+)")
1268 new_text_parts = []
1269 for line in lines:
1270 parts = balance_apostrophes(re.split(parts_re, line))
1271 state = 0 # 1=in italic 2=in bold 3=in both
1272 for i, part in enumerate(parts):
1273 if part.startswith("''"):
1274 # This is a bold/italic part. Scan the rest of the line
1275 # to determine how it should be interpreted if there are
1276 # more than two apostrophes.
1277 if part.startswith("'''''"):
1278 if state == 1: # in italic
1279 part = part[5:]
1280 state = 2
1281 elif state == 2: # in bold
1282 part = part[5:]
1283 state = 1
1284 elif state == 3: # in both
1285 state = 0
1286 part = part[5:]
1287 else: # in nothing
1288 part = part[5:]
1289 state = 3
1290 elif part.startswith("'''"):
1291 if state == 1: # in italic
1292 if bold_follows(parts, i): 1292 ↛ 1296line 1292 didn't jump to line 1296 because the condition on line 1292 was always true
1293 part = part[3:]
1294 state = 3
1295 else:
1296 part = part[2:]
1297 state = 0
1298 elif state == 2: # in bold
1299 part = part[3:]
1300 state = 0
1301 elif state == 3: # in both
1302 part = part[3:]
1303 state = 1
1304 else: # in nothing
1305 part = part[3:]
1306 state = 2
1307 elif part.startswith("''"): 1307 ↛ 1320line 1307 didn't jump to line 1320 because the condition on line 1307 was always true
1308 if state == 1: # in italic
1309 part = part[2:]
1310 state = 0
1311 elif state == 2: # in bold
1312 part = part[2:]
1313 state = 3
1314 elif state == 3: # in both
1315 part = part[2:]
1316 state = 2
1317 else: # in nothing
1318 part = part[2:]
1319 state = 1
1320 if part: 1320 ↛ 1321line 1320 didn't jump to line 1321 because the condition on line 1320 was never true
1321 new_text_parts.append(part)
1322 continue
1323 new_text_parts.append(part)
1324 new_text_parts.append("\n")
1325 new_text_parts = new_text_parts[:-1] # remove last \n
1326 return "".join(new_text_parts)
1329# regex to find File/Image link attributes that would mean an image
1330# is *not* inline
1331NOT_INLINE_IMG_RE = re.compile(r"\|\s*(right|left|center|thumb|frame)\s*\|")
1334URL_STARTS_RE = re.compile(
1335 r"({})".format(r"|".join(URL_STARTS)), flags=re.IGNORECASE
1336)
1338IMAGE_LINK_RE: Optional[re.Pattern] = None
1341def remove_invisible_markup(title: str) -> str:
1342 """Remove omitted content before text and link extraction."""
1343 # remove nowiki tag returned from `Wtp.node_to_html()`
1344 title = re.sub(r"<nowiki\s*/>", "", title)
1346 # Remove any remaining templates
1347 # title = re.sub(r"\{\{[^}]+\}\}", "", title)
1349 # Remove tables, which can contain other tables
1350 prev = ""
1351 while title != prev:
1352 prev = title
1353 title = re.sub(
1354 r"\{\|((?!\{\|)(?!\|\}).)*\|\}",
1355 "\n",
1356 title,
1357 flags=re.DOTALL,
1358 )
1359 # title = re.sub(r"(?s)\{\|.*?\|\}", "\n", title)
1360 # Remove second reference tags (<ref name="ref_name"/>)
1361 title = re.sub(r"<ref\s+name=\"[^\"]+\"\s*/>", "", title)
1362 # Remove references (<ref>...</ref>).
1363 title = re.sub(r"(?is)<ref\b\s*[^>/]*?>\s*.*?</ref\s*>", "", title)
1364 # Replace <span>...</span> by stripped content without newlines
1365 title = re.sub(
1366 r"(?is)<span\b\s*[^>]*?>(.*?)\s*</span\s*>",
1367 lambda m: re.sub(r"\s+", " ", m.group(1)),
1368 title,
1369 )
1370 # Replace <br/> by comma space (it is used to express alternatives in some
1371 # declensions)
1372 title = re.sub(r"(?si)\s*<br\s*/?>\n*", "\n", title)
1373 # Remove divs with floatright class (generated e.g. by {{ja-kanji|...}})
1374 title = re.sub(
1375 r'(?si)<div\b[^>]*?\bclass="[^"]*?\bfloatright\b[^>]*?>'
1376 r"((<div\b(<div\b.*?</div\s*>|.)*?</div>)|.)*?"
1377 r"</div\s*>",
1378 "",
1379 title,
1380 )
1381 # Remove divs with float: attribute
1382 title = re.sub(
1383 r'(?si)<div\b[^>]*?\bstyle="[^"]*?\bfloat:[^>]*?>'
1384 r"((<div\b(<div\b.*?</div\s*>|.)*?</div>)|.)*?"
1385 r"</div\s*>",
1386 "",
1387 title,
1388 )
1389 # Remove <sup> with previewonly class (generated e.g. by {{taxlink|...}})
1390 title = re.sub(
1391 r'(?si)<sup\b[^>]*?\bclass="[^"<>]*?'
1392 r"\bpreviewonly\b[^>]*?>"
1393 r".+?</sup\s*>",
1394 "",
1395 title,
1396 )
1397 # Remove <strong class="error">...</strong>
1398 title = re.sub(
1399 r'(?si)<strong\b[^>]*?\bclass="[^"]*?\berror\b[^>]*?>'
1400 r".+?</strong\s*>",
1401 "",
1402 title,
1403 )
1404 return title
1407def clean_value(
1408 wxr: WiktextractContext, title: str, no_strip=False, no_html_strip=False
1409) -> str:
1410 """Cleans a title or value into a normal string. This should basically
1411 remove any Wikimedia formatting from it: HTML tags, templates, links,
1412 emphasis, etc. This will also merge multiple whitespaces into one
1413 normal space and will remove any surrounding whitespace."""
1414 assert isinstance(wxr, WiktextractContext)
1415 assert isinstance(title, str)
1417 global IMAGE_LINK_RE
1418 if IMAGE_LINK_RE is None:
1419 image_link_prefixes = wxr.wtp.namespace_prefixes(
1420 wxr.wtp.NAMESPACE_DATA["File"]["id"], suffix=""
1421 )
1422 IMAGE_LINK_RE = re.compile(
1423 rf"(?:\s*{'|'.join(image_link_prefixes)})\s*:", re.IGNORECASE
1424 )
1426 def repl_1(m: re.Match) -> str:
1427 return clean_value(wxr, m.group(1), no_strip=True)
1429 def repl_exturl(m: re.Match) -> str:
1430 args = re.split(r"\s+", m.group(1))
1431 i = 0
1432 while i < len(args) - 1:
1433 if not URL_STARTS_RE.match(args[i]):
1434 break
1435 i += 1
1436 return " ".join(args[i:])
1438 def repl_link(m: re.Match) -> str:
1439 before_colon = m.group(1)
1440 after_colon = m.group(3)
1441 if (
1442 before_colon is not None
1443 and IMAGE_LINK_RE.match(before_colon) is not None
1444 ):
1445 return ""
1446 if before_colon is not None and before_colon.strip(": ") in ("w", "s"):
1447 # Wikipedia or Wikisource link
1448 v = after_colon.split("|")[0]
1449 else:
1450 v = m.group(0).strip("[]").split("|")[0]
1451 return clean_value(wxr, v, no_strip=True)
1453 def repl_link_bars(m: re.Match) -> str:
1454 link = m.group(1)
1455 if IMAGE_LINK_RE.match(link) is not None:
1456 # Handle File / Image / Fichier 'links' here.
1457 if NOT_INLINE_IMG_RE.match(m.group(0)) is None and "alt" in m.group(
1458 0
1459 ):
1460 # This image should be inline, so let's print its alt text
1461 alt_m = re.search(r"\|\s*alt\s*=([^]|]+)(\||\]\])", m.group(0))
1462 if alt_m is not None: 1462 ↛ 1464line 1462 didn't jump to line 1464 because the condition on line 1462 was always true
1463 return "[Alt: " + alt_m.group(1) + "]"
1464 return ""
1465 # m.group(5) is always the last matching group because you can
1466 # only access the last matched group; the indexes don't 'grow'
1467 return clean_value(wxr, m.group(3) or "", no_strip=True)
1469 def repl_1_sup(m: re.Match) -> str:
1470 return to_superscript(clean_value(wxr, m.group(1)))
1472 def repl_1_sub(m: re.Match) -> str:
1473 return to_subscript(clean_value(wxr, m.group(1)))
1475 def repl_1_chem(m: re.Match) -> str:
1476 return to_chem(clean_value(wxr, m.group(1)))
1478 def repl_1_math(m: re.Match) -> str:
1479 v = to_math(m.group(1))
1480 # print("to_math:", ascii(v))
1481 return v
1483 def repl_1_syntaxhighlight(m: re.Match) -> str:
1484 # Content is preformatted
1485 return "`" + m.group(1).strip() + "`"
1487 title = remove_invisible_markup(title)
1488 # Change <div> and </div> to newlines. Ditto for tr, li, table, dl, ul, ol
1489 title = re.sub(r"(?si)</?(div|tr|li|table|dl|ul|ol)\b[^>]*>", "\n", title)
1490 # Change <dt>, <dd>, </dt> and </dd> into newlines;
1491 # these generate new rows/lines.
1492 title = re.sub(r"(?i)</?d[dt]\s*>", "\n", title)
1493 # Change <td> </td> to spaces. Ditto for th.
1494 title = re.sub(r"(?si)</?(td|th)\b[^>]*>", " ", title)
1495 # Change <sup> ... </sup> to ^
1496 title = re.sub(r"(?si)<sup\b[^>]*>\s*</sup\s*>", "", title)
1497 title = re.sub(r"(?si)<sup\b[^>]*>(.*?)</sup\s*>", repl_1_sup, title)
1498 # Change <sub> ... </sub> to _
1499 title = re.sub(r"(?si)<sub\b[^>]*>\s*</sub\s*>", "", title)
1500 title = re.sub(r"(?si)<sub\b[^>]*>(.*?)</sub\s*>", repl_1_sub, title)
1501 # Change <chem> ... </chem> using subscripts for digits
1502 title = re.sub(r"(?si)<chem\b[^>]*>(.*?)</chem\s*>", repl_1_chem, title)
1503 # Change <math> ... </math> using special formatting.
1504 title = re.sub(r"(?si)<math\b[^>]*>(.*?)</math\s*>", repl_1_math, title)
1505 # Change <syntaxhighlight> ... </syntaxhighlight> using special formatting.
1506 title = re.sub(
1507 r"(?si)<syntaxhighlight\b[^>]*>(.*?)" r"</syntaxhighlight\s*>",
1508 repl_1_syntaxhighlight,
1509 title,
1510 )
1511 # Remove any remaining HTML tags.
1512 if not no_html_strip:
1513 title = re.sub(r"(?s)<[/!a-zA-Z][^>]*>", "", title)
1514 title = re.sub(r"(?s)</[^>]+>", "", title)
1515 else:
1516 # Strip <noinclude/> anyway
1517 title = re.sub(r"(?si)<noinclude\s*/\s*>", "", title)
1518 # Replace [...]
1519 title = re.sub(r"(?s)\[\s*\.\.\.\s*\]", "…", title)
1520 # Remove http links in superscript
1521 title = re.sub(r"\^\(\[?(https?:)?//[^]()]+\]?\)", "", title)
1522 # Remove any edit links to local pages
1523 title = re.sub(r"\[//[^]\s]+\s+edit\s*\]", "", title)
1524 # Replace links by their text
1526 category_ns_data: NamespaceDataEntry
1527 # XXX "Category" -> config variable for portability
1528 category_ns_data = wxr.wtp.NAMESPACE_DATA.get("Category", {}) # type: ignore[typeddict-item]
1529 # Fail if we received empty dict from .get()
1530 category_ns_names = {"Category", category_ns_data["name"]} | set(
1531 category_ns_data["aliases"]
1532 )
1533 category_names_pattern = rf"(?:{'|'.join(category_ns_names)})"
1534 while True:
1535 # Links may be nested, so keep replacing until there is no more change.
1536 orig = title
1537 title = re.sub(
1538 rf"(?si)\s*\[\[\s*{category_names_pattern}\s*:\s*([^]]+?)\s*\]\]",
1539 "",
1540 title,
1541 )
1542 title = re.sub(
1543 r"(?s)\[\[\s*:?([^]|#<>:&]+?)(#[^][|<>]*?)?\]\]", repl_1, title
1544 )
1545 title = re.sub(
1546 r"(?s)\[\[\s*(([\w\d]+)\s*:)?(\s*[^][#|<>]+?)"
1547 r"(#[^][|]*?)?\|?\]\]",
1548 repl_link,
1549 title,
1550 )
1551 title = re.sub(
1552 # [[ (...) |
1553 r"(?s)\[\[\s*([^][|<>]+?)\s*"
1554 # (|((...OR[...])+))*|]]
1555 r"(\|(([^][|]|\[[^]]*\])+?))*\|*\]\]",
1556 repl_link_bars,
1557 title,
1558 )
1559 if title == orig:
1560 break
1561 # Replace remaining HTML links by the URL.
1562 while True:
1563 orig = title
1564 title = re.sub(
1565 r"\[\s*((https?:|mailto:)?//([^][]+?))\s*\]", repl_exturl, title
1566 )
1567 if title == orig:
1568 break
1570 # Remove italic and bold
1571 title = remove_italic_and_bold(title)
1573 # Replace HTML entities
1574 title = html.unescape(title)
1575 title = title.replace("\xa0", " ") # nbsp
1576 # Remove left-to-right and right-to-left, zero-with characters
1577 title = re.sub(r"[\u200e\u200f\u200b\u200d\u200c\ufeff]", "", title)
1578 # Replace whitespace sequences by a single space.
1579 # https://en.wikipedia.org/wiki/En_(typography)
1580 title = re.sub(r"[ \t\r\u2002]+", " ", title)
1581 title = re.sub(r" *\n+", "\n", title)
1582 # Eliminate spaces around ellipsis in brackets
1583 title = re.sub(r"\[\s*…\s*\]", "[…]", title)
1585 # This unicode quote seems to be used instead of apostrophe quite randomly
1586 # (about 4% of apostrophes in English entries, some in Finnish entries).
1587 # title = re.sub("\u2019", "'", title) # Note: no r"..." here!
1588 # Replace strange unicode quotes with normal quotes
1589 # title = re.sub(r"”", '"', title)
1590 # Replace unicode long dash by normal dash
1591 # title = re.sub(r"–", "-", title)
1593 # Remove whitespace before periods and commas etc
1594 # XXX we might re-enable this, now trying without as it is removing some
1595 # instances where we would want to leave the space
1596 # title = re.sub(r" ([.,;:!?)])", repl_1, title)
1597 # Strip surrounding whitespace.
1598 if not no_strip:
1599 title = title.strip()
1600 # Normalize different ways of writing accents into the NFC canonical form
1601 title = unicodedata.normalize("NFC", title)
1602 return title
1605def clean_template_args(
1606 wxr: WiktextractContext,
1607 ht: Union[TemplateArgs, TemplateParameters],
1608 no_strip=False,
1609) -> dict[Union[str, int], str]:
1610 """Cleans all values in a template argument dictionary and returns the
1611 cleaned dictionary."""
1612 assert isinstance(wxr, WiktextractContext)
1613 assert isinstance(ht, dict)
1614 return {
1615 clean_value(wxr, str(k), no_html_strip=True): clean_value(
1616 wxr, str(v), no_strip=no_strip, no_html_strip=True
1617 )
1618 for k, v in ht.items()
1619 }