Coverage for src/wiktextract/clean.py: 92%

328 statements  

« prev     ^ index     » next       coverage.py v7.17.0, created at 2026-10-23 06:29 +0000

1# This file contains code to clean Wiktionary annotations from a string and to 

2# produce plain text from it, typically for glossary entries but this is also 

3# called for various other data to produce clean strings. 

4# 

5# This file also contains code for cleaning qualifiers for the "tags" field. 

6# 

7# Copyright (c) 2018-2022 Tatu Ylonen. See file LICENSE and https://ylonen.org 

8 

9import html 

10import re 

11import unicodedata 

12from typing import Callable, Optional, Union 

13 

14from wikitextprocessor.common import MAGIC_FIRST, MAGIC_LAST, URL_STARTS 

15from wikitextprocessor.core import NamespaceDataEntry, TemplateArgs 

16from wikitextprocessor.parser import TemplateParameters, balance_apostrophes 

17 

18from .wxr_context import WiktextractContext 

19 

20###################################################################### 

21# Cleaning values into plain text. 

22###################################################################### 

23 

24superscript_ht: dict[str, str] = { 

25 "0": "⁰", 

26 "1": "¹", 

27 "2": "²", 

28 "3": "³", 

29 "4": "⁴", 

30 "5": "⁵", 

31 "6": "⁶", 

32 "7": "⁷", 

33 "8": "⁸", 

34 "9": "⁹", 

35 "+": "⁺", 

36 "-": "⁻", 

37 "−": "⁻", 

38 "‐": "⁻", 

39 "–": "⁻", 

40 "—": "⁻", 

41 "一": "⁻", 

42 "=": "⁼", 

43 "(": "⁽", 

44 ")": "⁾", 

45 "A": "ᴬ", 

46 "B": "ᴮ", 

47 "D": "ᴰ", 

48 "E": "ᴱ", 

49 "G": "ᴳ", 

50 "H": "ᴴ", 

51 "I": "ᴵ", 

52 "J": "ᴶ", 

53 "K": "ᴷ", 

54 "L": "ᴸ", 

55 "M": "ᴹ", 

56 "N": "ᴺ", 

57 "O": "ᴼ", 

58 "P": "ᴾ", 

59 "R": "ᴿ", 

60 "T": "ᵀ", 

61 "U": "ᵁ", 

62 "V": "ⱽ", 

63 "W": "ᵂ", 

64 "a": "ᵃ", 

65 "b": "ᵇ", 

66 "c": "ᶜ", 

67 "d": "ᵈ", 

68 "e": "ᵉ", 

69 "f": "ᶠ", 

70 "g": "ᵍ", 

71 "h": "ʰ", 

72 "i": "ⁱ", 

73 "j": "ʲ", 

74 "k": "ᵏ", 

75 "l": "ˡ", 

76 "m": "ᵐ", 

77 "n": "ⁿ", 

78 "o": "ᵒ", 

79 "p": "ᵖ", 

80 "r": "ʳ", 

81 "s": "ˢ", 

82 "t": "ᵗ", 

83 "u": "ᵘ", 

84 "v": "ᵛ", 

85 "w": "ʷ", 

86 "x": "ˣ", 

87 "y": "ʸ", 

88 "z": "ᶻ", 

89 "β": "ᵝ", 

90 "γ": "ᵞ", 

91 "δ": "ᵟ", 

92 "θ": "ᶿ", 

93 "ι": "ᶥ", 

94 "φ": "ᵠ", 

95 "χ": "ᵡ", 

96 "∞": "\u2002᪲", # This is a KLUDGE 

97} 

98 

99subscript_ht: dict[str, str] = { 

100 "0": "₀", 

101 "1": "₁", 

102 "2": "₂", 

103 "3": "₃", 

104 "4": "₄", 

105 "5": "₅", 

106 "6": "₆", 

107 "7": "₇", 

108 "8": "₈", 

109 "9": "₉", 

110 "+": "₊", 

111 "-": "₋", 

112 "−": "₋", 

113 "=": "₌", 

114 "(": "₍", 

115 ")": "₎", 

116 "a": "ₐ", 

117 "e": "ₑ", 

118 "h": "ₕ", 

119 "i": "ᵢ", 

120 "j": "ⱼ", 

121 "k": "ₖ", 

122 "l": "ₗ", 

123 "m": "ₘ", 

124 "n": "ₙ", 

125 "o": "ₒ", 

126 "p": "ₚ", 

127 "r": "ᵣ", 

128 "s": "ₛ", 

129 "t": "ₜ", 

130 "u": "ᵤ", 

131 "v": "ᵥ", 

132 "x": "ₓ", 

133 "ə": "ₔ", 

134 "ρ": "ᵨ", 

135 "φ": "ᵩ", 

136 "χ": "ᵪ", 

137} 

138 

139 

140def to_superscript(text: str) -> str: 

141 "Converts text to superscript." 

142 if not text: 

143 return "" 

144 if all(x in superscript_ht for x in text): 

145 return "".join(superscript_ht[x] for x in text) 

146 if len(text) == 1: 

147 return "^" + text 

148 return "^({})".format(text) 

149 

150 

151def to_subscript(text: str) -> str: 

152 """Converts text to subscript.""" 

153 if not text: 153 ↛ 154line 153 didn't jump to line 154 because the condition on line 153 was never true

154 return "" 

155 if all(x in subscript_ht for x in text): 

156 return "".join(subscript_ht[x] for x in text) 

157 if len(text) == 1: 157 ↛ 159line 157 didn't jump to line 159 because the condition on line 157 was always true

158 return "_" + text 

159 return "_({})".format(text) 

160 

161 

162def to_chem(text: str) -> str: 

163 """Converts text to chemical formula, making digits subscript.""" 

164 return "".join(to_subscript(x) if x.isdigit() else x for x in text) 

165 

166 

167# Mapping from Latex names to Unicode characters/strings. This is the 

168# default mapping (some cases are handled specially in the code). 

169math_map: dict[str, str] = { 

170 # XXX should probably change greek characters to non-slanted ones? 

171 "AC": "∿", 

172 "APLcomment": "⍝", 

173 "APLdownarrowbox": "⍗", 

174 "APLinput": "⍞", 

175 "APLinv": "⌹", 

176 "APLleftarrowbox": "⍇", 

177 "APLlog": "⍟", 

178 "APLrightarrowbox": "⍈", 

179 "APLuparrowbox": "⍐", 

180 "Angstroem": "Å", 

181 "Bot": "⫫", 

182 "Box": "□", 

183 "Bumpeq": "≎", 

184 "CIRCLE": "●", 

185 "Cap": "⋒", 

186 "CapitalDifferentialD": "ⅅ", 

187 "CheckedBox": "☑", 

188 "Circle": "○", 

189 "Coloneqq": "⩴", 

190 "ComplexI": "ⅈ", 

191 "ComplexJ": "ⅉ", 

192 "Cup": "⋓", 

193 "Delta": "Δ", 

194 "Diamond": "◇", 

195 "Diamondblack": "◆", 

196 "Diamonddot": "⟐", 

197 "DifferentialD": "ⅆ", 

198 "Digamma": "Ϝ", 

199 "Doteq": "≑", 

200 "DownArrowBar": "⤓", 

201 "DownLeftTeeVector": "⥞", 

202 "DownLeftVectorBar": "⥖", 

203 "DownRightTeeVector": "⥟", 

204 "DownRightVectorBar": "⥗", 

205 "Downarrow": "⇓", 

206 "Equal": "⩵", 

207 "Euler": "Ɛ", 

208 "ExponentialE": "ⅇ", 

209 "ExponetialE": "ⅇ", 

210 "Finv": "Ⅎ", 

211 "Gamma": "Γ", 

212 "Im": "ℑ", 

213 "Join": "⨝", 

214 "Koppa": "Ϟ", 

215 "LEFTCIRCLE": "◖", 

216 "LEFTcircle": "◐", 

217 "LHD": "◀", 

218 "LVec": "x⃖", 

219 "Lambda": "Λ", 

220 "Lbag": "⟅", 

221 "LeftArrowBar": "⇤", 

222 "LeftDownTeeVector": "⥡", 

223 "LeftDownVectorBar": "⥙", 

224 "LeftTeeVector": "⥚", 

225 "LeftTriangleBar": "⧏", 

226 "LeftUpTeeVector": "⥠", 

227 "LeftUpVectorBar": "⥘", 

228 "LeftVectorBar": "⥒", 

229 "Leftarrow": "⇐", 

230 "Leftrightarrow": "⇔", 

231 "Lleftarrow": "⇚", 

232 "Longleftarrow": "⟸", 

233 "Longleftrightarrow": "⟺", 

234 "Longmapsfrom": "⟽", 

235 "Longmapsto": "⟾", 

236 "Longrightarrow": "⟹", 

237 "Lparen": "⦅", 

238 "Lsh": "↰", 

239 "MapsDown": "↧", 

240 "MapsUp": "↥", 

241 "Mapsfrom": "⤆", 

242 "Mapsto": "⤇", 

243 "Micro": "µ", 

244 "Nearrow": "⇗", 

245 "NestedGreaterGreater": "⪢", 

246 "NestedLessLess": "⪡", 

247 "NotGreaterLess": "≹", 

248 "NotGreaterTilde": "≵", 

249 "NotLessTilde": "≴", 

250 "Nwarrow": "⇖", 

251 "Omega": "Ω", 

252 "Phi": "Φ", 

253 "Pi": "Π", 

254 "Proportion": "∷", 

255 "Psi": "Ψ", 

256 "Qoppa": "Ϙ", 

257 "RHD": "▶", 

258 "RIGHTCIRCLE": "◗", 

259 "RIGHTcircle": "◑", 

260 "Rbag": "⟆", 

261 "Re": "ℜ", 

262 "RightArrowBar": "⇥", 

263 "RightDownTeeVector": "⥝", 

264 "RightDownVectorBar": "⥕", 

265 "RightTeeVector": "⥛", 

266 "RightTriangleBar": "⧐", 

267 "RightUpTeeVector": "⥜", 

268 "RightUpVectorBar": "⥔", 

269 "RightVectorBar": "⥓", 

270 "Rightarrow": "⇒", 

271 "Rparen": "⦆", 

272 "Rrightarrow": "⇛", 

273 "Rsh": "↱", 

274 "S": "§", 

275 "Same": "⩶", 

276 "Sampi": "Ϡ", 

277 "Searrow": "⇘", 

278 "Sigma": "Σ", 

279 "Square": "☐", 

280 "Stigma": "Ϛ", 

281 "Subset": "⋐", 

282 "Sun": "☉", 

283 "Supset": "⋑", 

284 "Swarrow": "⇙", 

285 "Theta": "Θ", 

286 "Top": "⫪", 

287 "UpArrowBar": "⤒", 

288 "Uparrow": "⇑", 

289 "Updownarrow": "⇕", 

290 "Upsilon": "Υ", 

291 "VDash": "⊫", 

292 "VERT": "⦀", 

293 "Vdash": "⊩", 

294 "Vert": "‖", 

295 "Vvdash": "⊪", 

296 "XBox": "☒", 

297 "Xi": "Ξ", 

298 "Yup": "⅄", 

299 "_": "_", 

300 "aleph": "א", 

301 "alpha": "α", 

302 "amalg": "⨿", 

303 "anchor": "⚓", 

304 "angle": "∠", 

305 "approx": "≈", 

306 "approxeq": "≊", 

307 "aquarius": "♒", 

308 "arg": "arg", 

309 "aries": "♈", 

310 "arrowbullet": "➢", 

311 "ast": "∗", 

312 "asymp": "≍", 

313 "backepsilon": "϶", 

314 "backprime": "‵", 

315 "backsim": "∽", 

316 "backsimeq": "⋍", 

317 "backslash": "", 

318 "ballotx": "✗", 

319 "barin": "⋶", 

320 "barleftharpoon": "⥫", 

321 "barrightharpoon": "⥭", 

322 "barwedge": "⊼", 

323 "because": "∵", 

324 "beta": "β", 

325 "beth": "ב", 

326 "between": "≬", 

327 "bigcap": "∩", 

328 "bigcup": "∪", 

329 "biginterleave": "⫼", 

330 "bigodot": "⨀", 

331 "bigoplus": "⨁", 

332 "bigotimes": "⨂", 

333 "bigsqcap": "⨅", 

334 "bigsqcup": "⨆", 

335 "bigstar": "★", 

336 "bigtriangledown": "▽", 

337 "bigtriangleup": "△", 

338 "biguplus": "⨄", 

339 "bigvee": "∨", 

340 "bigwedge": "∧", 

341 "bij": "⤖", 

342 "biohazard": "☣", 

343 "blacklozenge": "⧫", 

344 "blacksmiley": "☻", 

345 "blacksquare": "■", 

346 "blacktriangledown": "▾", 

347 "blacktriangleleft": "◂", 

348 "blacktriangleright": "▸", 

349 "blacktriangleup": "▴", 

350 "bot": "⊥", 

351 "bowtie": "⋈", 

352 "boxast": "⧆", 

353 "boxbar": "◫", 

354 "boxbox": "⧈", 

355 "boxbslash": "⧅", 

356 "boxcircle": "⧇", 

357 "boxdot": "⊡", 

358 "boxminus": "⊟", 

359 "boxplus": "⊞", 

360 "boxslash": "⧄", 

361 "boxtimes": "⊠", 

362 "bullet": "•", 

363 "bumpeq": "≏", 

364 "cancer": "♋", 

365 "cap": "∩", 

366 "capricornus": "♑", 

367 "capwedge": "⩄", 

368 "cat": "⁀", 

369 "cdot": "·", 

370 "cdots": "⋯", 

371 "cent": "¢", 

372 "checkmark": "✓", 

373 "chi": "χ", 

374 "circ": "∘", 

375 "circeq": "≗", 

376 "circlearrowleft": "↺", 

377 "circlearrowright": "↻", 

378 "circledR": "®", 

379 "circledast": "⊛", 

380 "circledbslash": "⦸", 

381 "circledcirc": "⊚", 

382 "circleddash": "⊝", 

383 "circledgtr": "⧁", 

384 "circledless": "⧀", 

385 "clubsuit": "♣", 

386 "colon": ":", 

387 "coloneq": "≔", 

388 "complement": "∁", 

389 "cong": "≅", 

390 "coprod": "∐", 

391 "corresponds": "≙", 

392 "cup": "∪", 

393 "curlyeqprec": "⋞", 

394 "curlyeqsucc": "⋟", 

395 "curlyvee": "⋎", 

396 "curlywedge": "⋏", 

397 "curvearrowleft": "↶", 

398 "curvearrowright": "↷", 

399 "dagger": "†", 

400 "daleth": "ד", 

401 "dashleftarrow": "⇠", 

402 "dashrightarrow": "⇢", 

403 "dashv": "⊣", 

404 "ddagger": "‡", 

405 "delta": "δ", 

406 "diameter": "∅", 

407 "diamond": "⋄", 

408 "diamondsuit": "♢", 

409 "digamma": "ϝ", 

410 "div": "÷", 

411 "divideontimes": "⋇", 

412 "dlsh": "↲", 

413 "dot\\bigvee": "⩒", 

414 "dot\\cap": "⩀", 

415 "dot\\cup": "⊍", 

416 "dot\\lor": "⩒", 

417 "dot\\vee": "⩒", 

418 "doteq": "≐", 

419 "dotplus": "∔", 

420 "dots": "…", 

421 "doublebarwedge": "⩞", 

422 "downarrow": "↓", 

423 "downdownarrows": "⇊", 

424 "downdownharpoons": "⥥", 

425 "downharpoonleft": "⇃", 

426 "downharpoonright": "⇂", 

427 "downuparrows": "⇵", 

428 "downupharpoons": "⥯", 

429 "drsh": "↳", 

430 "dsub": "⩤", 

431 "earth": "♁", 

432 "eighthnote": "♪", 

433 "ell": "ℓ", 

434 "emptyset": "∅", 

435 "epsilon": "ϵ", 

436 "eqcirc": "≖", 

437 "eqcolon": "∹", 

438 "eqsim": "≂", 

439 "eqslantgtr": "⪖", 

440 "eqslantless": "⪕", 

441 "equiv": "≡", 

442 "eta": "η", 

443 "eth": "ð", 

444 "exists": "∃", 

445 "fallingdotseq": "≒", 

446 "fcmp": "⨾", 

447 "female": "♀", 

448 "ffun": "⇻", 

449 "finj": "⤕", 

450 "fint": "⨏", 

451 "flat": "♭", 

452 "footnotesize": "", 

453 "forall": "∀", 

454 "fourth": "⁗", 

455 "frown": "⌢", 

456 "frownie": "☹", 

457 "gamma": "γ", 

458 "ge": ">", 

459 "gemini": "♊", 

460 "geq": "≥", 

461 "geqq": "≧", 

462 "geqslant": "⩾", 

463 "gg": "≫", 

464 "ggcurly": "⪼", 

465 "ggg": "⋙", 

466 "gimel": "ג", 

467 "gnapprox": "⪊", 

468 "gneq": "⪈", 

469 "gneqq": "≩", 

470 "gnsim": "⋧", 

471 "gtrapprox": "⪆", 

472 "gtrdot": "⋗", 

473 "gtreqless": "⋛", 

474 "gtreqqless": "⪌", 

475 "gtrless": "≷", 

476 "gtrsim": "≳", 

477 "hash": "⋕", 

478 "heartsuit": "♡", 

479 "hookleftarrow": "↩", 

480 "hookrightarrow": "↪", 

481 "hslash": "ℏ", 

482 "iddots": "⋰", 

483 "iff": "⟺", 

484 "iiiint": "⨌", 

485 "iiint": "∭", 

486 "iint": "∬", 

487 "imath": "ı", 

488 "implies": "⟹", 

489 "in": "∈", 

490 "infty": "∞", 

491 "int": "∫", 

492 "intercal": "⊺", 

493 "interleave": "⫴", 

494 "invamp": "⅋", 

495 "invdiameter": "⍉", 

496 "invneg": "⌐", 

497 "iota": "ι", 

498 "jmath": "ȷ", 

499 "jupiter": "♃", 

500 "kappa": "κ", 

501 "koppa": "ϟ", 

502 "lambda": "λ", 

503 "land": "∧", 

504 "lang": "⟪", 

505 "langle": "⟨", 

506 "large": "", 

507 "lblot": "⦉", 

508 "lbrace": "{", 

509 "lbrack": "[", 

510 "lceil": "⌈", 

511 "ldots": "…", 

512 "le": "<", 

513 "leadsto": "⤳", 

514 "leftarrow": "←", 

515 "leftarrowtail": "↢", 

516 "leftarrowtriangle": "⇽", 

517 "leftbarharpoon": "⥪", 

518 "leftharpoondown": "↽", 

519 "leftharpoonup": "↼", 

520 "leftleftarrows": "⇇", 

521 "leftleftharpoons": "⥢", 

522 "leftmoon": "☾", 

523 "leftrightarrow": "↔", 

524 "leftrightarrows": "⇆", 

525 "leftrightarrowtriangle": "⇿", 

526 "leftrightharpoon": "⥊", 

527 "leftrightharpoondown": "⥐", 

528 "leftrightharpoons": "⇋", 

529 "leftrightharpoonup": "⥎", 

530 "leftrightsquigarrow": "↭", 

531 "leftslice": "⪦", 

532 "leftsquigarrow": "⇜", 

533 "leftthreetimes": "⋋", 

534 "leftupdownharpoon": "⥑", 

535 "leo": "♌", 

536 "leq": "≤", 

537 "leqq": "≦", 

538 "leqslant": "⩽", 

539 "lessapprox": "⪅", 

540 "lessdot": "⋖", 

541 "lesseqgtr": "⋚", 

542 "lesseqqgtr": "⪋", 

543 "lessgtr": "≶", 

544 "lessim": "≲", 

545 "lesssim": "≲", 

546 "lfloor": "⌊", 

547 "lgroup": "⟮", 

548 "lhd": "◁", 

549 "libra": "♎", 

550 "lightning": "↯", 

551 "limg": "⦇", 

552 "ll": "≪", 

553 "llbracket": "⟦", 

554 "llcorner": "⌞", 

555 "llcurly": "⪻", 

556 "lll": "⋘", 

557 "lnapprox": "⪉", 

558 "lneq": "⪇", 

559 "lneqq": "≨", 

560 "lnot": "¬", 

561 "lnsim": "⋦", 

562 "longleftarrow": "⟵", 

563 "longleftrightarrow": "⟷", 

564 "longmapsfrom": "⟻", 

565 "longmapsto": "⟼", 

566 "longrightarrow": "⟶", 

567 "looparrowleft": "↫", 

568 "looparrowright": "↬", 

569 "lor": "∨", 

570 "lozenge": "◊", 

571 "lrcorner": "⌟", 

572 "ltimes": "⋉", 

573 "male": "♂", 

574 "maltese": "✠", 

575 "mapsfrom": "↤", 

576 "mapsto": "↦", 

577 "measuredangle": "∡", 

578 "medbullet": "⚫", 

579 "medcirc": "⚪", 

580 "mercury": "☿", 

581 "mho": "℧", 

582 "mid": "∣", 

583 "mlcp": "⫛", 

584 "mod": " mod ", 

585 "models": "⊧", 

586 "mp": "∓", 

587 "mu": "μ", 

588 "multimap": "⊸", 

589 "multimapboth": "⧟", 

590 "multimapdotbothA": "⊶", 

591 "multimapdotbothB": "⊷", 

592 "multimapinv": "⟜", 

593 "nLeftarrow": "⇍", 

594 "nLeftrightarrow": "⇎", 

595 "nRightarrow": "⇏", 

596 "nVDash": "⊯", 

597 "nVdash": "⊮", 

598 "nabla": "∇", 

599 "napprox": "≉", 

600 "natural": "♮", 

601 "ncong": "≇", 

602 "nearrow": "↗", 

603 "neg": "¬", 

604 "neptune": "♆", 

605 "neq": "≠", 

606 "nequiv": "≢", 

607 "nexists": "∄", 

608 "ngeq": "≱", 

609 "ngtr": "≯", 

610 "ni": "∋", 

611 "nleftarrow": "↚", 

612 "nleftrightarrow": "↮", 

613 "nleq": "≰", 

614 "nless": "≮", 

615 "nmid": "∤", 

616 "nni": "∌", 

617 "normalsize": "", 

618 "not\\in": "∉", 

619 "not\\ni": "∌", 

620 "not\\preceq": "⋠", 

621 "not\\subset": "⊄", 

622 "not\\subseteq": "⊈", 

623 "not\\succeq": "⋡", 

624 "not\\supset": "⊅", 

625 "not\\supseteq": "⊉", 

626 "not\\trianglelefteq": "⋬", 

627 "not\\trianglerighteq": "⋭", 

628 "not\\vartriangleleft": "⋪", 

629 "not\\vartriangleright": "⋫", 

630 "notasymp": "≭", 

631 "notbackslash": "⍀", 

632 "notin": "∉", 

633 "notslash": "⌿", 

634 "nparallel": "∦", 

635 "nprec": "⊀", 

636 "npreceq": "⋠", 

637 "nrightarrow": "↛", 

638 "nsim": "≁", 

639 "nsimeq": "≄", 

640 "nsqsubseteq": "⋢", 

641 "nsqsupseteq": "⋣", 

642 "nsubset": "⊄", 

643 "nsubseteq": "⊈", 

644 "nsucc": "⊁", 

645 "nsucceq": "⋡", 

646 "nsupset": "⊅", 

647 "nsupseteq": "⊉", 

648 "ntriangleleft": "⋪", 

649 "ntrianglelefteq": "⋬", 

650 "ntriangleright": "⋫", 

651 "ntrianglerighteq": "⋭", 

652 "nu": "ν", 

653 "nvDash": "⊭", 

654 "nvdash": "⊬", 

655 "nwarrow": "↖", 

656 "odot": "⊙", 

657 "oiiint": "∰", 

658 "oiint": "∯", 

659 "oint": "∮", 

660 "ointctrclockwise": "∳", 

661 "omega": "ω", 

662 "ominus": "⊖", 

663 "oplus": "⊕", 

664 "oslash": "⊘", 

665 "otimes": "⊗", 

666 "over": "/", 

667 "overbrace": "⏞", 

668 "overleftrightarrow": "x⃡", 

669 "overparen": "⏜", 

670 "overset?=": "≟", 

671 "overset{?}{=}": "≟", 

672 "overset{\\operatorname{def}}{=}": "≝", 

673 "parallel": "∥", 

674 "partial": "∂", 

675 "pencil": "✎", 

676 "perp": "⊥", 

677 "pfun": "⇸", 

678 "phi": "ϕ", 

679 "pi": "π", 

680 "pinj": "⤔", 

681 "pisces": "♓", 

682 "pitchfork": "⋔", 

683 "pluto": "♇", 

684 "pm": "±", 

685 "pointright": "☞", 

686 "pounds": "£", 

687 "prec": "≺", 

688 "precapprox": "⪷", 

689 "preccurlyeq": "≼", 

690 "preceq": "⪯", 

691 "preceqq": "⪳", 

692 "precnapprox": "⪹", 

693 "precnsim": "⋨", 

694 "precsim": "≾", 

695 "prime": "′", 

696 "prod": "∏", 

697 "propto": "∝", 

698 "psi": "ψ", 

699 "psur": "⤀", 

700 "qoppa": "ϙ", 

701 "quad": " ", 

702 "quarternote": "♩", 

703 "radiation": "☢", 

704 "rang": "⟫", 

705 "rangle": "⟩", 

706 "rarr": "→", 

707 "rblot": "⦊", 

708 "rbrace": "}", 

709 "rbrack": "]", 

710 "rceil": "⌉", 

711 "recycle": "♻", 

712 "rfloor": "⌋", 

713 "rgroup": "⟯", 

714 "rhd": "▷", 

715 "rho": "ρ", 

716 "rightangle": "∟", 

717 "rightarrow": "→", 

718 "rightarrowtail": "↣", 

719 "rightarrowtriangle": "⇾", 

720 "rightbarharpoon": "⥬", 

721 "rightharpoondown": "⇁", 

722 "rightharpoonup": "⇀", 

723 "rightleftarrows": "⇄", 

724 "rightleftharpoon": "⥋", 

725 "rightleftharpoons": "⇌", 

726 "rightmoon": "☽", 

727 "rightrightarrows": "⇉", 

728 "rightrightharpoons": "⥤", 

729 "rightslice": "⪧", 

730 "rightsquigarrow": "⇝", 

731 "rightthreetimes": "⋌", 

732 "rightupdownharpoon": "⥏", 

733 "rimg": "⦈", 

734 "risingdotseq": "≓", 

735 "rrbracket": "⟧", 

736 "rsub": "⩥", 

737 "rtimes": "⋊", 

738 "sagittarius": "♐", 

739 "sampi": "ϡ", 

740 "saturn": "♄", 

741 "scorpio": "♏", 

742 "scriptsize": "", 

743 "searrow": "↘", 

744 "second": "″", 

745 "setminus": "⧵", 

746 "sharp": "♯", 

747 "sigma": "σ", 

748 "sim": "∼", 

749 "simeq": "≃", 

750 "sixteenthnote": "♬", 

751 "skull": "☠", 

752 "slash": "∕", 

753 "small": "", 

754 "smallsetminus": "∖", 

755 "smalltriangledown": "▿", 

756 "smalltriangleleft": "◃", 

757 "smalltriangleright": "▹", 

758 "smalltriangleup": "▵", 

759 "smile": "⌣", 

760 "smiley": "☺", 

761 "spadesuit": "♠", 

762 "spddot": "¨", 

763 "sphat": "^", 

764 "sphericalangle": "∢", 

765 "spot": "⦁", 

766 "sptilde": "~", 

767 "sqcap": "⊓", 

768 "sqcup": "⊔", 

769 "sqint": "⨖", 

770 "sqrt": "√", # ∛ ∜ - partly special handling below 

771 "sqrt[3]": "∛", 

772 "sqrt[4]": "∜", 

773 "sqsubset": "⊏", 

774 "sqsubseteq": "⊑", 

775 "sqsupset": "⊐", 

776 "sqsupseteq": "⊒", 

777 "square": "□", 

778 "sslash": "⫽", 

779 "star": "⋆", 

780 "steaming": "☕", 

781 "stigma": "ϛ", 

782 "strictfi": "⥼", 

783 "strictif": "⥽", 

784 "subset": "⊂", 

785 "subseteq": "⊆", 

786 "subseteqq": "⫅", 

787 "subsetneq": "⊊", 

788 "subsetneqq": "⫋", 

789 "succ": "≻", 

790 "succapprox": "⪸", 

791 "succcurlyeq": "≽", 

792 "succeq": "⪰", 

793 "succeqq": "⪴", 

794 "succnapprox": "⪺", 

795 "succnsim": "⋩", 

796 "succsim": "≿", 

797 "sum": "∑", 

798 "sun": "☼", 

799 "supset": "⊃", 

800 "supseteq": "⊇", 

801 "supseteqq": "⫆", 

802 "supsetneq": "⊋", 

803 "supsetneqq": "⫌", 

804 "swarrow": "↙", 

805 "swords": "⚔", 

806 "talloblong": "⫾", 

807 "tau": "τ", 

808 "taurus": "♉", 

809 "tcohm": "Ω", 

810 "textbackslash": "\\", 

811 "textbar": "|", 

812 "textbullet": "•", 

813 "textgreater": ">", 

814 "textless": "<", 

815 "textprime": "′", 

816 "therefore": "∴", 

817 "theta": "θ", 

818 "third": "‴", 

819 "times": "×", 

820 "tiny": "", 

821 "to": "→", 

822 "top": "⊤", 

823 "triangle": "∆", 

824 "trianglelefteq": "⊴", 

825 "triangleq": "≜", 

826 "trianglerighteq": "⊵", 

827 "twoheadleftarrow": "↞", 

828 "twoheadrightarrow": "↠", 

829 "twonotes": "♫", 

830 "ulcorner": "⌜", 

831 "underbar": " ̱", 

832 "underbrace": "⏟", 

833 "underleftarrow": "x⃮", 

834 "underline": " ̲", 

835 "underparen": "⏝", 

836 "underrightarrow": "x⃯", 

837 "uparrow": "↑", 

838 "updownarrow": "↕", 

839 "updownarrows": "⇅", 

840 "updownharpoons": "⥮", 

841 "upharpoonleft": "↿", 

842 "upharpoonright": "↾", 

843 "uplus": "⊎", 

844 "upsilon": "υ", 

845 "upuparrows": "⇈", 

846 "upupharpoons": "⥣", 

847 "uranus": "♅", 

848 "urcorner": "⌝", 

849 "utilde": " ̰", 

850 "vDash": "⊨", 

851 "varbeta": "β", 

852 "varclubsuit": "♧", 

853 "vardiamondsuit": "♦", 

854 "varepsilon": "ε", 

855 "varheartsuit": "♥", 

856 "varkappa": "ϰ", 

857 "varnothing": "∅", 

858 "varointclockwise": "∲", 

859 "varphi": "φ", 

860 "varpi": "ϖ", 

861 "varprod": "⨉", 

862 "varrho": "ϱ", 

863 "varsigma": "ς", 

864 "varspadesuit": "♤", 

865 "vartheta": "θ", 

866 "vartriangleleft": "⊲", 

867 "vartriangleright": "⊳", 

868 "vdash": "⊢", 

869 "vdots": "⋮", 

870 "vee": "∨", 

871 "veebar": "⊻", 

872 "vert": "|", 

873 "virgo": "♍", 

874 "warning": "⚠", 

875 "wasylozenge": "⌑", 

876 "wedge": "∧", 

877 "widehat=": "≙", 

878 "widehat{=}": "≙", 

879 "wp": "℘", 

880 "wr": "≀", 

881 "xi": "ξ", 

882 "yen": "¥", 

883 "yinyang": "☯", 

884 "zcmp": "⨟", 

885 "zeta": "ζ", 

886 "zhide": "⧹", 

887 "zpipe": "⨠", 

888 "zproject": "⨡", 

889 "|": "‖", 

890 # Accents XXX these really should be handled specially with diacritics 

891 # after argument 

892 "acute": "́", 

893 "bar": "̄", 

894 "breve": "̆", 

895 "check": "̌", 

896 "ddddot": "⃜", 

897 "dddot": "⃛", 

898 "ddot": "̈", 

899 "ddots": "⋱", 

900 "dot": "̇", 

901 "grave": "̀", 

902 "hat": "̂", 

903 "lvec": "⃐", 

904 "mathring": "̊", 

905 "not": "̸", 

906 "overline": "◌̅", 

907 "tilde": "̃", 

908 "vec": "⃑", 

909 # Some ignored operators 

910 "bigl": "", 

911 "bigr": "", 

912 "left": "", 

913 "right": "", 

914 "style": "", 

915 "textstyle": "", 

916 "mathrm": "", 

917} 

918 

919mathcal_map: dict[str, str] = { 

920 "A": "𝒜", 

921 "B": "ℬ", 

922 "C": "𝒞", 

923 "D": "𝒟", 

924 "E": "ℰ", 

925 "F": "ℱ", 

926 "G": "𝒢", 

927 "H": "ℋ", 

928 "I": "ℐ", 

929 "J": "𝒥", 

930 "K": "𝒦", 

931 "L": "ℒ", 

932 "M": "ℳ", 

933 "N": "𝒩", 

934 "O": "𝒪", 

935 "P": "𝒫", 

936 "Q": "𝒬", 

937 "R": "ℛ", 

938 "S": "𝒮", 

939 "T": "𝒯", 

940 "U": "𝒰", 

941 "V": "𝒱", 

942 "W": "𝒲", 

943 "X": "𝒳", 

944 "Y": "𝒴", 

945 "Z": "𝒵", 

946 "a": "𝒶", 

947 "b": "𝒷", 

948 "c": "𝒸", 

949 "d": "𝒹", 

950 "e": "ℯ", 

951 "f": "𝒻", 

952 "g": "ℊ", 

953 "h": "𝒽", 

954 "i": "𝒾", 

955 "j": "𝒿", 

956 "k": "𝓀", 

957 "l": "𝓁", 

958 "m": "𝓂", 

959 "n": "𝓃", 

960 "o": "ℴ", 

961 "p": "𝓅", 

962 "q": "𝓆", 

963 "r": "𝓇", 

964 "s": "𝓈", 

965 "t": "𝓉", 

966 "u": "𝓊", 

967 "v": "𝓋", 

968 "w": "𝓌", 

969 "x": "𝓍", 

970 "y": "𝓎", 

971 "z": "𝓏", 

972} 

973 

974mathfrak_map: dict[str, str] = { 

975 "A": "𝔄", 

976 "B": "𝔅", 

977 "C": "ℭ", 

978 "D": "𝔇", 

979 "E": "𝔈", 

980 "F": "𝔉", 

981 "G": "𝔊", 

982 "H": "ℌ", 

983 "J": "𝔍", 

984 "K": "𝔎", 

985 "L": "𝔏", 

986 "M": "𝔐", 

987 "N": "𝔑", 

988 "O": "𝔒", 

989 "P": "𝔓", 

990 "Q": "𝔔", 

991 "S": "𝔖", 

992 "T": "𝔗", 

993 "U": "𝔘", 

994 "V": "𝔙", 

995 "W": "𝔚", 

996 "X": "𝔛", 

997 "Y": "𝔜", 

998 "Z": "ℨ", 

999} 

1000 

1001mathbb_map: dict[str, str] = { 

1002 "A": "𝔸", 

1003 "B": "𝔹", 

1004 "C": "ℂ", 

1005 "D": "𝔻", 

1006 "E": "𝔼", 

1007 "F": "𝔽", 

1008 "G": "𝔾", 

1009 "H": "ℍ", 

1010 "I": "𝕀", 

1011 "J": "𝕁", 

1012 "K": "𝕂", 

1013 "L": "𝕃", 

1014 "M": "𝕄", 

1015 "N": "ℕ", 

1016 "O": "𝕆", 

1017 "P": "ℙ", 

1018 "Q": "ℚ", 

1019 "R": "ℝ", 

1020 "S": "𝕊", 

1021 "T": "𝕋", 

1022 "U": "𝕌", 

1023 "V": "𝕍", 

1024 "W": "𝕎", 

1025 "X": "𝕏", 

1026 "Y": "𝕐", 

1027 "Z": "ℤ", 

1028 "a": "𝕒", 

1029 "b": "𝕓", 

1030 "c": "𝕔", 

1031 "d": "𝕕", 

1032 "e": "𝕖", 

1033 "f": "𝕗", 

1034 "g": "𝕘", 

1035 "h": "𝕙", 

1036 "i": "𝕚", 

1037 "j": "𝕛", 

1038 "k": "𝕜", 

1039 "l": "𝕝", 

1040 "m": "𝕞", 

1041 "n": "𝕟", 

1042 "o": "𝕠", 

1043 "p": "𝕡", 

1044 "q": "𝕢", 

1045 "r": "𝕣", 

1046 "s": "𝕤", 

1047 "t": "𝕥", 

1048 "u": "𝕦", 

1049 "v": "𝕧", 

1050 "w": "𝕨", 

1051 "x": "𝕩", 

1052 "y": "𝕪", 

1053 "z": "𝕫", 

1054 "pi": "ℼ", 

1055 "gamma": "ℽ", 

1056 "Gamma": "ℾ", 

1057 "Pi": "ℿ", 

1058 "Sigma": "⅀", 

1059 "0": "𝟘", 

1060 "1": "𝟙", 

1061 "2": "𝟚", 

1062 "3": "𝟛", 

1063 "4": "𝟜", 

1064 "5": "𝟝", 

1065 "6": "𝟞", 

1066 "7": "𝟟", 

1067 "8": "𝟠", 

1068 "9": "𝟡", 

1069} 

1070 

1071 

1072def mathcal_fn(text: str) -> str: 

1073 return "".join(mathcal_map.get(x, x) for x in text) 

1074 

1075 

1076def mathfrak_fn(text: str) -> str: 

1077 return "".join(mathfrak_map.get(x, x) for x in text) 

1078 

1079 

1080def mathbb_fn(text: str) -> str: 

1081 return "".join(mathbb_map.get(x, x) for x in text) 

1082 

1083 

1084def to_math(text: str) -> str: 

1085 """Converts a mathematical formula to ASCII.""" 

1086 # print("to_math: {!r}".format(text)) 

1087 magic_vec: list[str] = [] 

1088 

1089 def expand(text: str) -> str: 

1090 while True: 

1091 orig = text 

1092 # formatting with {:c} converts input into character 

1093 text = re.sub( 

1094 r"[{:c}-{:c}]".format(MAGIC_FIRST, MAGIC_LAST), 

1095 lambda m: magic_vec[ord(m.group(0)) - MAGIC_FIRST], 

1096 text, 

1097 ) 

1098 if text == orig: 

1099 break 

1100 return text 

1101 

1102 def recurse(text: str) -> str: 

1103 def math_magic( 

1104 text: str, left: str, right: str, fn: Callable[[str], str] 

1105 ) -> str: 

1106 regexp_str = r"{}([^{}{}]+){}".format( 

1107 re.escape(left), 

1108 re.escape(left), 

1109 re.escape(right), 

1110 re.escape(right), 

1111 ) 

1112 regexp = re.compile(regexp_str) 

1113 

1114 def repl(m: re.Match) -> str: 

1115 magic = chr(MAGIC_FIRST + len(magic_vec)) 

1116 t = fn(m.group(1)).strip() 

1117 magic_vec.append(t) 

1118 return magic 

1119 

1120 while True: 

1121 orig = text 

1122 text = re.sub(regexp, repl, text) 

1123 if text == orig: 

1124 break 

1125 return text 

1126 

1127 def expand_group(v: str) -> str: 

1128 fn: Optional[Callable[[str], str]] = None 

1129 if re.match(r"\\mathcal\b", v): 

1130 fn = mathcal_fn 

1131 v = v[8:].strip() 

1132 elif re.match(r"\\mathfrak\b", v): 

1133 fn = mathfrak_fn 

1134 v = v[9:].strip() 

1135 elif re.match(r"\\mathbb\b", v): 

1136 fn = mathbb_fn 

1137 v = v[7:] 

1138 elif re.match(r"\\(begin|end)\b", v): 1138 ↛ 1139line 1138 didn't jump to line 1139 because the condition on line 1138 was never true

1139 v = "" # Skip 

1140 elif re.match(r"\\text\b", v): 1140 ↛ 1141line 1140 didn't jump to line 1141 because the condition on line 1140 was never true

1141 v = v[5:] 

1142 elif re.match(r"\\pmod\b", v): 1142 ↛ 1143line 1142 didn't jump to line 1143 because the condition on line 1142 was never true

1143 v = v[5:].strip() 

1144 v = "(mod " + expand_group(v) + ")" 

1145 elif re.match(r"\\sqrt($|[0-9\[]|\b)", v): 

1146 m = re.match(r"\\sqrt\s*(?:\[([^\]]*)\])?(.*)$", v, re.DOTALL) 

1147 assert m is not None 

1148 index, radicand = m.groups() 

1149 index = (index or "").strip() 

1150 if index in ("", "2"): 

1151 root = "√" 

1152 elif index == "3": 

1153 root = "∛" 

1154 elif index == "4": 

1155 root = "∜" 

1156 else: 

1157 root = to_superscript(index) + "√" 

1158 # Keep the radicand after the root sign, in parentheses unless 

1159 # it is a single symbol or number. 

1160 radicand = expand_group(radicand.strip()).strip() 

1161 if len(radicand) > 1 and not radicand.isalnum(): 

1162 radicand = "(" + radicand + ")" 

1163 v = root + radicand 

1164 elif re.match(r"\\(frac|binom)($|[0-9]|\b)", v): 

1165 m = re.match( 

1166 r"\\(frac|binom)\s*(\\[a-zA-Z]+|\\.|.)\s*" 

1167 r"(\\[a-zA-Z]+|\\.|.)$", 

1168 v, 

1169 ) 

1170 if not m: 1170 ↛ 1171line 1170 didn't jump to line 1171 because the condition on line 1170 was never true

1171 print("MATH FRAC/BINOM ERROR: {!r}".format(v)) 

1172 return v 

1173 op, a, b = m.groups() 

1174 a = expand_group(a).strip() 

1175 b = expand_group(b).strip() 

1176 if len(a) > 1: 

1177 a = "(" + a + ")" 

1178 if len(b) > 1: 

1179 b = "(" + b + ")" 

1180 if op == "frac": 1180 ↛ 1182line 1180 didn't jump to line 1182 because the condition on line 1180 was always true

1181 v = a + "/" + b 

1182 elif op == "binom": 

1183 v = "binom({}, {})".format(a, b) 

1184 else: 

1185 # Should never get here 

1186 v = "{}({})".format(op, v) 

1187 elif v.startswith("_"): 

1188 fn = to_subscript 

1189 v = v[1:] 

1190 elif v.startswith("^"): 

1191 fn = to_superscript 

1192 v = v[1:] 

1193 if v.startswith("\\"): 

1194 mapped = math_map.get(v[1:].strip()) 

1195 if mapped is None: 

1196 if v[1:].strip().isalnum(): 1196 ↛ 1199line 1196 didn't jump to line 1199 because the condition on line 1196 was always true

1197 v = " " + v[1:].strip() + " " 

1198 else: 

1199 v = v[1:].strip() 

1200 else: 

1201 v = mapped 

1202 elif v.isspace() or v in ("&",): # Ignore certain special chars 1202 ↛ 1203line 1202 didn't jump to line 1203 because the condition on line 1202 was never true

1203 v = "" 

1204 if fn is not None: 

1205 v = expand(v) 

1206 v = fn(v) 

1207 v = expand(v) 

1208 return v 

1209 

1210 parts: list[str] = [] 

1211 while True: 

1212 orig = text 

1213 text = math_magic(text, "{", "}", recurse) 

1214 if text == orig: 

1215 break 

1216 for m in re.finditer( 

1217 r"\s+|" 

1218 r"\\frac\s*(\\[a-zA-Z]+|\\.|.)\s*" 

1219 r"(\\dot\\(bigvee|cup|cap|lor|vee)|" 

1220 r"\\not\\(subset|supset|subseteq|supseteq|in|ni|" 

1221 r"preceq|succeq|vartrianglelefteq|" 

1222 r"vartrianglerighteq|trianglelefteq|" 

1223 r"trianglerighteq)|" 

1224 r"\\widehat\{=\}|\\widehat=|" 

1225 r"\\overset\{?\}\{=\}|" 

1226 r"\\overset\?=|" 

1227 r"\\overset\{\\operatorname\{def\}\}\{=\}|" 

1228 r"\\[a-zA-Z]+|\\.|.)|" 

1229 r"(\\(mathcal|mathfrak|mathbb|text|begin|end|pmod)" 

1230 r"\b\s*|" 

1231 r"\\sqrt\b(\[\d+\])?)?" 

1232 r"[_^]?(\\[a-zA-Z]+\s*|\\.|\w+|.)", 

1233 text, 

1234 ): 

1235 v = m.group(0).strip() 

1236 if not v: 

1237 continue 

1238 v = expand_group(v) 

1239 if v: 1239 ↛ 1216line 1239 didn't jump to line 1216 because the condition on line 1239 was always true

1240 if ( 

1241 parts and parts[-1][-1].isalpha() and v[0] in "0123456789" 

1242 ) or ( 

1243 parts 

1244 and parts[-1][-1] in "0123456789" 

1245 and v[0] in "0123456789" 

1246 ): 

1247 v = " " + v 

1248 parts.append(v) 

1249 

1250 text = "".join(parts) 

1251 return text 

1252 

1253 text = recurse(text) 

1254 # print("math text final: {!r}".format(text)) 

1255 return text 

1256 

1257 

1258def bold_follows(parts: list[str], i: int) -> bool: 

1259 """Checks if there is a bold (''') in parts after parts[i]. We allow 

1260 intervening italics ('').""" 

1261 parts = parts[i + 1 :] 

1262 for p in parts: 1262 ↛ 1267line 1262 didn't jump to line 1267 because the loop on line 1262 didn't complete

1263 if not p.startswith("''"): 

1264 continue 

1265 if p.startswith("'''"): 1265 ↛ 1262line 1265 didn't jump to line 1262 because the condition on line 1265 was always true

1266 return True 

1267 return False 

1268 

1269 

1270def remove_italic_and_bold(text: str) -> str: 

1271 """Based on token_iter in wikitextprocessor""" 

1272 assert isinstance(text, str) 

1273 lines = re.split(r"(\n+)", text) # Lines and separators 

1274 parts_re = re.compile(r"(''+)") 

1275 new_text_parts = [] 

1276 for line in lines: 

1277 parts = balance_apostrophes(re.split(parts_re, line)) 

1278 state = 0 # 1=in italic 2=in bold 3=in both 

1279 for i, part in enumerate(parts): 

1280 if part.startswith("''"): 

1281 # This is a bold/italic part. Scan the rest of the line 

1282 # to determine how it should be interpreted if there are 

1283 # more than two apostrophes. 

1284 if part.startswith("'''''"): 

1285 if state == 1: # in italic 

1286 part = part[5:] 

1287 state = 2 

1288 elif state == 2: # in bold 

1289 part = part[5:] 

1290 state = 1 

1291 elif state == 3: # in both 

1292 state = 0 

1293 part = part[5:] 

1294 else: # in nothing 

1295 part = part[5:] 

1296 state = 3 

1297 elif part.startswith("'''"): 

1298 if state == 1: # in italic 

1299 if bold_follows(parts, i): 1299 ↛ 1303line 1299 didn't jump to line 1303 because the condition on line 1299 was always true

1300 part = part[3:] 

1301 state = 3 

1302 else: 

1303 part = part[2:] 

1304 state = 0 

1305 elif state == 2: # in bold 

1306 part = part[3:] 

1307 state = 0 

1308 elif state == 3: # in both 

1309 part = part[3:] 

1310 state = 1 

1311 else: # in nothing 

1312 part = part[3:] 

1313 state = 2 

1314 elif part.startswith("''"): 1314 ↛ 1327line 1314 didn't jump to line 1327 because the condition on line 1314 was always true

1315 if state == 1: # in italic 

1316 part = part[2:] 

1317 state = 0 

1318 elif state == 2: # in bold 

1319 part = part[2:] 

1320 state = 3 

1321 elif state == 3: # in both 

1322 part = part[2:] 

1323 state = 2 

1324 else: # in nothing 

1325 part = part[2:] 

1326 state = 1 

1327 if part: 1327 ↛ 1328line 1327 didn't jump to line 1328 because the condition on line 1327 was never true

1328 new_text_parts.append(part) 

1329 continue 

1330 new_text_parts.append(part) 

1331 new_text_parts.append("\n") 

1332 new_text_parts = new_text_parts[:-1] # remove last \n 

1333 return "".join(new_text_parts) 

1334 

1335 

1336# regex to find File/Image link attributes that would mean an image 

1337# is *not* inline 

1338NOT_INLINE_IMG_RE = re.compile(r"\|\s*(right|left|center|thumb|frame)\s*\|") 

1339 

1340 

1341URL_STARTS_RE = re.compile( 

1342 r"({})".format(r"|".join(URL_STARTS)), flags=re.IGNORECASE 

1343) 

1344 

1345IMAGE_LINK_RE: Optional[re.Pattern] = None 

1346 

1347 

1348def remove_invisible_markup(title: str) -> str: 

1349 """Remove omitted content before text and link extraction.""" 

1350 # remove nowiki tag returned from `Wtp.node_to_html()` 

1351 title = re.sub(r"<nowiki\s*/>", "", title) 

1352 

1353 # Remove any remaining templates 

1354 # title = re.sub(r"\{\{[^}]+\}\}", "", title) 

1355 

1356 # Remove tables, which can contain other tables 

1357 prev = "" 

1358 while title != prev: 

1359 prev = title 

1360 title = re.sub( 

1361 r"\{\|((?!\{\|)(?!\|\}).)*\|\}", 

1362 "\n", 

1363 title, 

1364 flags=re.DOTALL, 

1365 ) 

1366 # title = re.sub(r"(?s)\{\|.*?\|\}", "\n", title) 

1367 # Remove second reference tags (<ref name="ref_name"/>) 

1368 title = re.sub(r"<ref\s+name=\"[^\"]+\"\s*/>", "", title) 

1369 # Remove references (<ref>...</ref>). 

1370 title = re.sub(r"(?is)<ref\b\s*[^>/]*?>\s*.*?</ref\s*>", "", title) 

1371 # Replace <span>...</span> by stripped content without newlines 

1372 title = re.sub( 

1373 r"(?is)<span\b\s*[^>]*?>(.*?)\s*</span\s*>", 

1374 lambda m: re.sub(r"\s+", " ", m.group(1)), 

1375 title, 

1376 ) 

1377 # Replace <br/> by comma space (it is used to express alternatives in some 

1378 # declensions) 

1379 title = re.sub(r"(?si)\s*<br\s*/?>\n*", "\n", title) 

1380 # Remove divs with floatright class (generated e.g. by {{ja-kanji|...}}) 

1381 title = re.sub( 

1382 r'(?si)<div\b[^>]*?\bclass="[^"]*?\bfloatright\b[^>]*?>' 

1383 r"((<div\b(<div\b.*?</div\s*>|.)*?</div>)|.)*?" 

1384 r"</div\s*>", 

1385 "", 

1386 title, 

1387 ) 

1388 # Remove divs with float: attribute 

1389 title = re.sub( 

1390 r'(?si)<div\b[^>]*?\bstyle="[^"]*?\bfloat:[^>]*?>' 

1391 r"((<div\b(<div\b.*?</div\s*>|.)*?</div>)|.)*?" 

1392 r"</div\s*>", 

1393 "", 

1394 title, 

1395 ) 

1396 # Remove <sup> with previewonly class (generated e.g. by {{taxlink|...}}) 

1397 title = re.sub( 

1398 r'(?si)<sup\b[^>]*?\bclass="[^"<>]*?' 

1399 r"\bpreviewonly\b[^>]*?>" 

1400 r".+?</sup\s*>", 

1401 "", 

1402 title, 

1403 ) 

1404 # Remove <strong class="error">...</strong> 

1405 title = re.sub( 

1406 r'(?si)<strong\b[^>]*?\bclass="[^"]*?\berror\b[^>]*?>' 

1407 r".+?</strong\s*>", 

1408 "", 

1409 title, 

1410 ) 

1411 return title 

1412 

1413 

1414def clean_value( 

1415 wxr: WiktextractContext, title: str, no_strip=False, no_html_strip=False 

1416) -> str: 

1417 """Cleans a title or value into a normal string. This should basically 

1418 remove any Wikimedia formatting from it: HTML tags, templates, links, 

1419 emphasis, etc. This will also merge multiple whitespaces into one 

1420 normal space and will remove any surrounding whitespace.""" 

1421 assert isinstance(wxr, WiktextractContext) 

1422 assert isinstance(title, str) 

1423 

1424 global IMAGE_LINK_RE 

1425 if IMAGE_LINK_RE is None: 

1426 image_link_prefixes = wxr.wtp.namespace_prefixes( 

1427 wxr.wtp.NAMESPACE_DATA["File"]["id"], suffix="" 

1428 ) 

1429 IMAGE_LINK_RE = re.compile( 

1430 rf"(?:\s*{'|'.join(image_link_prefixes)})\s*:", re.IGNORECASE 

1431 ) 

1432 

1433 def repl_1(m: re.Match) -> str: 

1434 return clean_value(wxr, m.group(1), no_strip=True) 

1435 

1436 def repl_exturl(m: re.Match) -> str: 

1437 args = re.split(r"\s+", m.group(1)) 

1438 i = 0 

1439 while i < len(args) - 1: 

1440 if not URL_STARTS_RE.match(args[i]): 

1441 break 

1442 i += 1 

1443 return " ".join(args[i:]) 

1444 

1445 def repl_link(m: re.Match) -> str: 

1446 before_colon = m.group(1) 

1447 after_colon = m.group(3) 

1448 if ( 

1449 before_colon is not None 

1450 and IMAGE_LINK_RE.match(before_colon) is not None 

1451 ): 

1452 return "" 

1453 if before_colon is not None and before_colon.strip(": ") in ("w", "s"): 

1454 # Wikipedia or Wikisource link 

1455 v = after_colon.split("|")[0] 

1456 else: 

1457 v = m.group(0).strip("[]").split("|")[0] 

1458 return clean_value(wxr, v, no_strip=True) 

1459 

1460 def repl_link_bars(m: re.Match) -> str: 

1461 link = m.group(1) 

1462 if IMAGE_LINK_RE.match(link) is not None: 

1463 # Handle File / Image / Fichier 'links' here. 

1464 if NOT_INLINE_IMG_RE.match(m.group(0)) is None and "alt" in m.group( 

1465 0 

1466 ): 

1467 # This image should be inline, so let's print its alt text 

1468 alt_m = re.search(r"\|\s*alt\s*=([^]|]+)(\||\]\])", m.group(0)) 

1469 if alt_m is not None: 1469 ↛ 1471line 1469 didn't jump to line 1471 because the condition on line 1469 was always true

1470 return "[Alt: " + alt_m.group(1) + "]" 

1471 return "" 

1472 # m.group(5) is always the last matching group because you can 

1473 # only access the last matched group; the indexes don't 'grow' 

1474 return clean_value(wxr, m.group(3) or "", no_strip=True) 

1475 

1476 def repl_1_sup(m: re.Match) -> str: 

1477 return to_superscript(clean_value(wxr, m.group(1))) 

1478 

1479 def repl_1_sub(m: re.Match) -> str: 

1480 return to_subscript(clean_value(wxr, m.group(1))) 

1481 

1482 def repl_1_chem(m: re.Match) -> str: 

1483 return to_chem(clean_value(wxr, m.group(1))) 

1484 

1485 def repl_1_math(m: re.Match) -> str: 

1486 v = to_math(m.group(1)) 

1487 # print("to_math:", ascii(v)) 

1488 return v 

1489 

1490 def repl_1_syntaxhighlight(m: re.Match) -> str: 

1491 # Content is preformatted 

1492 return "`" + m.group(1).strip() + "`" 

1493 

1494 title = remove_invisible_markup(title) 

1495 # Change <div> and </div> to newlines. Ditto for tr, li, table, dl, ul, ol 

1496 title = re.sub(r"(?si)</?(div|tr|li|table|dl|ul|ol)\b[^>]*>", "\n", title) 

1497 # Change <dt>, <dd>, </dt> and </dd> into newlines; 

1498 # these generate new rows/lines. 

1499 title = re.sub(r"(?i)</?d[dt]\s*>", "\n", title) 

1500 # Change <td> </td> to spaces. Ditto for th. 

1501 title = re.sub(r"(?si)</?(td|th)\b[^>]*>", " ", title) 

1502 # Change <sup> ... </sup> to ^ 

1503 title = re.sub(r"(?si)<sup\b[^>]*>\s*</sup\s*>", "", title) 

1504 title = re.sub(r"(?si)<sup\b[^>]*>(.*?)</sup\s*>", repl_1_sup, title) 

1505 # Change <sub> ... </sub> to _ 

1506 title = re.sub(r"(?si)<sub\b[^>]*>\s*</sub\s*>", "", title) 

1507 title = re.sub(r"(?si)<sub\b[^>]*>(.*?)</sub\s*>", repl_1_sub, title) 

1508 # Change <chem> ... </chem> using subscripts for digits 

1509 title = re.sub(r"(?si)<chem\b[^>]*>(.*?)</chem\s*>", repl_1_chem, title) 

1510 # Change <math> ... </math> using special formatting. 

1511 title = re.sub(r"(?si)<math\b[^>]*>(.*?)</math\s*>", repl_1_math, title) 

1512 # Change <syntaxhighlight> ... </syntaxhighlight> using special formatting. 

1513 title = re.sub( 

1514 r"(?si)<syntaxhighlight\b[^>]*>(.*?)" r"</syntaxhighlight\s*>", 

1515 repl_1_syntaxhighlight, 

1516 title, 

1517 ) 

1518 # Remove any remaining HTML tags. 

1519 if not no_html_strip: 

1520 title = re.sub(r"(?s)<[/!a-zA-Z][^>]*>", "", title) 

1521 title = re.sub(r"(?s)</[^>]+>", "", title) 

1522 else: 

1523 # Strip <noinclude/> anyway 

1524 title = re.sub(r"(?si)<noinclude\s*/\s*>", "", title) 

1525 # Replace [...] 

1526 title = re.sub(r"(?s)\[\s*\.\.\.\s*\]", "…", title) 

1527 # Remove http links in superscript 

1528 title = re.sub(r"\^\(\[?(https?:)?//[^]()]+\]?\)", "", title) 

1529 # Remove any edit links to local pages 

1530 title = re.sub(r"\[//[^]\s]+\s+edit\s*\]", "", title) 

1531 # Replace links by their text 

1532 

1533 category_ns_data: NamespaceDataEntry 

1534 # XXX "Category" -> config variable for portability 

1535 category_ns_data = wxr.wtp.NAMESPACE_DATA.get("Category", {}) # type: ignore[typeddict-item] 

1536 # Fail if we received empty dict from .get() 

1537 category_ns_names = {"Category", category_ns_data["name"]} | set( 

1538 category_ns_data["aliases"] 

1539 ) 

1540 category_names_pattern = rf"(?:{'|'.join(category_ns_names)})" 

1541 while True: 

1542 # Links may be nested, so keep replacing until there is no more change. 

1543 orig = title 

1544 title = re.sub( 

1545 rf"(?si)\s*\[\[\s*{category_names_pattern}\s*:\s*([^]]+?)\s*\]\]", 

1546 "", 

1547 title, 

1548 ) 

1549 title = re.sub( 

1550 r"(?s)\[\[\s*:?([^]|#<>:&]+?)(#[^][|<>]*?)?\]\]", repl_1, title 

1551 ) 

1552 title = re.sub( 

1553 r"(?s)\[\[\s*(([\w\d]+)\s*:)?(\s*[^][#|<>]+?)" 

1554 r"(#[^][|]*?)?\|?\]\]", 

1555 repl_link, 

1556 title, 

1557 ) 

1558 title = re.sub( 

1559 # [[ (...) | 

1560 r"(?s)\[\[\s*([^][|<>]+?)\s*" 

1561 # (|((...OR[...])+))*|]] 

1562 r"(\|(([^][|]|\[[^]]*\])+?))*\|*\]\]", 

1563 repl_link_bars, 

1564 title, 

1565 ) 

1566 if title == orig: 

1567 break 

1568 # Replace remaining HTML links by the URL. 

1569 while True: 

1570 orig = title 

1571 title = re.sub( 

1572 r"\[\s*((https?:|mailto:)?//([^][]+?))\s*\]", repl_exturl, title 

1573 ) 

1574 if title == orig: 

1575 break 

1576 

1577 # Remove italic and bold 

1578 title = remove_italic_and_bold(title) 

1579 

1580 # Replace HTML entities 

1581 title = html.unescape(title) 

1582 title = title.replace("\xa0", " ") # nbsp 

1583 # Remove left-to-right and right-to-left, zero-with characters 

1584 title = re.sub(r"[\u200e\u200f\u200b\u200d\u200c\ufeff]", "", title) 

1585 # Replace whitespace sequences by a single space. 

1586 # https://en.wikipedia.org/wiki/En_(typography) 

1587 title = re.sub(r"[ \t\r\u2002]+", " ", title) 

1588 title = re.sub(r" *\n+", "\n", title) 

1589 # Eliminate spaces around ellipsis in brackets 

1590 title = re.sub(r"\[\s*…\s*\]", "[…]", title) 

1591 

1592 # This unicode quote seems to be used instead of apostrophe quite randomly 

1593 # (about 4% of apostrophes in English entries, some in Finnish entries). 

1594 # title = re.sub("\u2019", "'", title) # Note: no r"..." here! 

1595 # Replace strange unicode quotes with normal quotes 

1596 # title = re.sub(r"”", '"', title) 

1597 # Replace unicode long dash by normal dash 

1598 # title = re.sub(r"–", "-", title) 

1599 

1600 # Remove whitespace before periods and commas etc 

1601 # XXX we might re-enable this, now trying without as it is removing some 

1602 # instances where we would want to leave the space 

1603 # title = re.sub(r" ([.,;:!?)])", repl_1, title) 

1604 # Strip surrounding whitespace. 

1605 if not no_strip: 

1606 title = title.strip() 

1607 # Normalize different ways of writing accents into the NFC canonical form 

1608 title = unicodedata.normalize("NFC", title) 

1609 return title 

1610 

1611 

1612def clean_template_args( 

1613 wxr: WiktextractContext, 

1614 ht: Union[TemplateArgs, TemplateParameters], 

1615 no_strip=False, 

1616) -> dict[Union[str, int], str]: 

1617 """Cleans all values in a template argument dictionary and returns the 

1618 cleaned dictionary.""" 

1619 assert isinstance(wxr, WiktextractContext) 

1620 assert isinstance(ht, dict) 

1621 return { 

1622 clean_value(wxr, str(k), no_html_strip=True): clean_value( 

1623 wxr, str(v), no_strip=no_strip, no_html_strip=True 

1624 ) 

1625 for k, v in ht.items() 

1626 }