Coverage for src/wiktextract/wiktionary.py: 75%

1# Wiktionary parser for extracting a lexicon and various other information

2# from wiktionary. This file contains code to uncompress the Wiktionary

3# dump file and to separate it into individual pages.

7import atexit

8import io

9import json

10import os

11import re

12import tarfile

13import tempfile

14import time

15from concurrent.futures import ProcessPoolExecutor

16from copy import deepcopy

17from multiprocessing import current_process, get_all_start_methods, get_context

18from pathlib import Path

19from traceback import format_exc

20from typing import TextIO

22from wikitextprocessor import Page

23from wikitextprocessor.core import CollatedErrorReturnData, ErrorMessageData

24from wikitextprocessor.dumpparser import process_dump

26from .import_utils import import_extractor_module

27from .page import parse_page

28from .thesaurus import (

29 emit_words_in_thesaurus,

30 extract_thesaurus_data,

31 thesaurus_linkage_number,

32)

33from .wxr_context import WiktextractContext

34from .wxr_logging import logger

37def page_handler(

38 page: Page,

39) -> tuple[list[dict[str, str]], CollatedErrorReturnData]:

40 # Make sure there are no newlines or other strange characters in the

41 # title. They could cause security problems at several post-processing

42 # steps.

43 # We've given the page_handler function an extra wxr attribute previously.

44 # This should never cause an exception, and if it does, we want it to.

46 # Helps debug extraction hangs. This writes the path of each file being

47 # processed into /tmp/wiktextract*/wiktextract-*. Once a hang

48 # has been observed, these files contain page(s) that hang. They should

49 # be checked before aborting the process, as an interrupt might delete them.

50 with tempfile.TemporaryDirectory(prefix="wiktextract") as tmpdirname:

51 debug_path = "{}/wiktextract-{}".format(tmpdirname, os.getpid())

52 with open(debug_path, "w", encoding="utf-8") as f:

53 f.write(page.title + "\n")

55 worker_wxr.wtp.start_page(page.title)

56 try:

57 title = re.sub(r"[\s\000-\037]+", " ", page.title)

58 title = title.strip()

59 if page.redirect_to is not None:

60 page_data = [

61 {

62 "title": title,

63 "redirect": page.redirect_to,

64 "pos": "hard-redirect",

65 }

66 ]

67 else:

68 # XXX Sign gloss pages?

69 start_t = time.time()

70 page_data = parse_page(worker_wxr, title, page.body) # type: ignore[arg-type]

71 dur = time.time() - start_t

72 if dur > 100: 72 ↛ 73line 72 didn't jump to line 73 because the condition on line 72 was never true

73 logger.warning(

74 "====== WARNING: PARSING PAGE TOOK {:.1f}s: {}".format(

75 dur, title

76 )

77 )

79 return page_data, worker_wxr.wtp.to_return()

80 except Exception:

81 worker_wxr.wtp.error(

82 f'=== EXCEPTION while parsing page "{page.title}" '

83 f"in process {current_process().name}",

84 format_exc(),

85 "page_handler_exception",

86 )

87 return [], worker_wxr.wtp.to_return()

90def parse_wiktionary(

91 wxr: WiktextractContext,

92 dump_path: str,

93 num_processes: int | None,

94 phase1_only: bool,

95 namespace_ids: set[int],

96 out_f: TextIO,

97 human_readable: bool = False,

98 override_folders: list[str] | list[Path] | None = None,

99 skip_extract_dump: bool = False,

100 save_pages_path: str | Path | None = None,

101) -> None:

102 """Parses Wiktionary from the dump file ``path`` (which should point

103 to a "enwiktionary-<date>-pages-articles.xml.bz2" file. This

104 calls `word_cb(data)` for all words defined for languages in `languages`."""

105 capture_language_codes = wxr.config.capture_language_codes

106 if capture_language_codes is not None: 106 ↛ 111line 106 didn't jump to line 111 because the condition on line 106 was always true

107 assert isinstance(capture_language_codes, (list, tuple, set))

108 for x in capture_language_codes:

109 assert isinstance(x, str)

110

111 logger.info("First phase - extracting templates, macros, and pages")

112 if override_folders is not None: 112 ↛ 113line 112 didn't jump to line 113 because the condition on line 112 was never true

113 override_folders = [Path(folder) for folder in override_folders]

114 if save_pages_path is not None: 114 ↛ 115line 114 didn't jump to line 115 because the condition on line 114 was never true

115 save_pages_path = Path(save_pages_path)

116

117 analyze_template_mod = import_extractor_module(

118 wxr.wtp.lang_code, "analyze_template"

119 )

120 process_dump(

121 wxr.wtp,

122 dump_path,

123 namespace_ids,

124 override_folders,

125 skip_extract_dump,

126 save_pages_path,

127 analyze_template_mod.analyze_template

128 if analyze_template_mod is not None

129 else None,

130 )

131

132 if not phase1_only: 132 ↛ exitline 132 didn't return from function 'parse_wiktionary' because the condition on line 132 was always true

133 reprocess_wiktionary(wxr, num_processes, out_f, human_readable)

134

135

136def sanitize_and_write_json_data(

137 data: dict, out_f: TextIO, human_readable: bool

138) -> None:

139 if out_f is not None: 139 ↛ exitline 139 didn't return from function 'sanitize_and_write_json_data' because the condition on line 139 was always true

140 if human_readable: 140 ↛ 141line 140 didn't jump to line 141 because the condition on line 140 was never true

141 out_json = json.dumps(

142 data, indent=2, sort_keys=True, ensure_ascii=False

143 )

144 else:

145 out_json = json.dumps(data, ensure_ascii=False)

146 # Sanitize output for weird Unicode punctuation that breaks json

147 # on other systems and applications.

148 out_json = out_json.replace("\u2028", "")

149 out_f.write(out_json)

150 out_f.write("\n")

151

152

153def estimate_progress(

154 processed_pages: int, all_pages: int, start_time: float, last_time: float

155) -> float:

156 current_time = time.time()

157 processed_pages += 1

158 if current_time - last_time > 1:

159 remaining_pages = all_pages - processed_pages

160 estimate_seconds = (

161 (current_time - start_time) / processed_pages * remaining_pages

162 )

163 logger.info(

164 " ... {}/{} pages ({:.1%}) processed, "

165 "{:02d}:{:02d}:{:02d} remaining".format(

166 processed_pages,

167 all_pages,

168 processed_pages / all_pages,

169 int(estimate_seconds / 3600),

170 int(estimate_seconds / 60 % 60),

171 int(estimate_seconds % 60),

172 )

173 )

174 last_time = current_time

175 return last_time

176

177

178def check_error(

179 wxr: WiktextractContext,

180 dt: dict,

181 word: str | None,

182 lang: str | None,

183 pos: str | None,

184 msg: str,

185 called_from: str | None = None,

186) -> None:

187 """Formats and outputs an error message about data format checks."""

188 if called_from is None: 188 ↛ 191line 188 didn't jump to line 191 because the condition on line 188 was always true

189 called_from = "wiktionary/179/20240425"

190 else:

191 called_from = "wiktionary/179/20240425" + called_from

192 # msg += ": " + json.dumps(dt, sort_keys=True, ensure_ascii=False)

193 prefix = word or ""

194 if lang: 194 ↛ 196line 194 didn't jump to line 196 because the condition on line 194 was always true

195 prefix += "/" + lang

196 if pos: 196 ↛ 198line 196 didn't jump to line 198 because the condition on line 196 was always true

197 prefix += "/" + pos

198 if prefix: 198 ↛ 200line 198 didn't jump to line 200 because the condition on line 198 was always true

199 msg = prefix + ": " + msg

200 print(msg)

201 config = wxr.config

202 if len(config.debugs) > 100000: # Avoid excessive size 202 ↛ 203line 202 didn't jump to line 203 because the condition on line 202 was never true

203 return

204 error_data: ErrorMessageData = {

205 "msg": msg,

206 "trace": "",

207 "title": word,

208 "section": lang,

209 "subsection": pos,

210 "called_from": called_from,

211 "path": tuple(),

212 }

213 config.debugs.append(error_data)

214

215

216def check_tags(

217 wxr: WiktextractContext,

218 dt: dict,

219 word: str,

220 lang: str,

221 pos: str,

222 item: dict,

223) -> None:

224 assert isinstance(item, dict)

225 tags = item.get("tags")

226 if tags is None:

227 return

228 if not isinstance(tags, (list, tuple)): 228 ↛ 229line 228 didn't jump to line 229 because the condition on line 228 was never true

229 check_error(

230 wxr,

231 dt,

232 word,

233 lang,

234 pos,

235 '"tags" field value must be a list of strings: {}'.format(

236 repr(tags)

237 ),

238 )

239 return

240 for tag in tags:

241 if not isinstance(tag, str): 241 ↛ 242line 241 didn't jump to line 242 because the condition on line 241 was never true

242 check_error(

243 wxr,

244 dt,

245 word,

246 lang,

247 pos,

248 '"tags" field should only contain strings: {}'.format(

249 repr(tag)

250 ),

251 )

252 continue

253 # XXX enable the following later (currently too many bogus tags in

254 # non-English editions). Tag values should be standardized across

255 # editions, except for uppercase tags (e.g., regional variants).

256 if wxr.wtp.lang_code in ("en",): # Check edition 256 ↛ 240line 256 didn't jump to line 240 because the condition on line 256 was always true

257 from .tags import uppercase_tags, valid_tags

258

259 if tag not in valid_tags and tag not in uppercase_tags: 259 ↛ 260line 259 didn't jump to line 260 because the condition on line 259 was never true

260 if len(tag) > 0 and tag[0].isupper():

261 check_error(

262 wxr,

263 dt,

264 word,

265 lang,

266 pos,

267 f"invalid uppercase tag {tag} not in or uppercase_tags",

268 called_from="uppercase_tags",

269 )

270 else:

271 check_error(

272 wxr,

273 dt,

274 word,

275 lang,

276 pos,

277 f"invalid tag {tag} not in valid_tags "

278 "or uppercase_tags",

279 )

280

281

282def check_str_fields(

283 wxr: WiktextractContext,

284 dt: dict,

285 word: str,

286 lang: str,

287 pos: str,

288 item: dict,

289 fields: list[str],

290 mandatory: bool = False,

291 empty_ok: bool = False,

292) -> None:

293 """Checks that each of the listed fields contains a non-empty string.

294 Non-existent fields are ok unless ``mandatory`` is True."""

295 assert isinstance(item, dict)

296 for field in fields:

297 v = item.get(field)

298 if field not in item:

299 if mandatory:

300 check_error(

301 wxr,

302 dt,

303 word,

304 lang,

305 pos,

306 "{!r} is missing and should be a{} string: {}".format(

307 field,

308 "" if empty_ok else " non-empty",

309 json.dumps(item, sort_keys=True, ensure_ascii=False),

310 ),

311 )

312 continue

313 if not isinstance(v, str): 313 ↛ 314line 313 didn't jump to line 314 because the condition on line 313 was never true

314 check_error(

315 wxr,

316 dt,

317 word,

318 lang,

319 pos,

320 "{!r} should be a{} string: {}".format(

321 field,

322 "" if empty_ok else " non-empty",

323 json.dumps(item, sort_keys=True, ensure_ascii=False),

324 ),

325 )

326 if not v and not empty_ok:

327 check_error(

328 wxr,

329 dt,

330 word,

331 lang,

332 pos,

333 "{!r} should contain a non-empty string: {}".format(

334 field, json.dumps(item, sort_keys=True, ensure_ascii=False)

335 ),

336 )

337

338

339def check_dict_list_fields(

340 wxr: WiktextractContext,

341 dt: dict,

342 word: str,

343 lang: str,

344 pos: str,

345 item: dict,

346 fields: list[str],

347) -> bool:

348 """Checks that each listed field, if present, is a list of dicts."""

349 assert isinstance(item, dict)

350 for field in fields:

351 lst = item.get(field)

352 if lst is None:

353 continue

354 if not isinstance(lst, (list, tuple)): 354 ↛ 355line 354 didn't jump to line 355 because the condition on line 354 was never true

355 check_error(

356 wxr,

357 dt,

358 word,

359 lang,

360 pos,

361 "{!r} should be a list of dicts: {}".format(

362 field, json.dumps(lst, sort_keys=True, ensure_ascii=False)

363 ),

364 )

365 return False

366 for x in lst:

367 if not isinstance(x, dict): 367 ↛ 368line 367 didn't jump to line 368 because the condition on line 367 was never true

368 check_error(

369 wxr,

370 dt,

371 word,

372 lang,

373 pos,

374 "{!r} should be a list of dicts: {}".format(

375 field, json.dumps(x, sort_keys=True, ensure_ascii=False)

376 ),

377 )

378 return False

379 return True

380

381

382def check_str_list_fields(

383 wxr: WiktextractContext,

384 dt: dict,

385 word: str,

386 lang: str,

387 pos: str,

388 item: dict,

389 fields: list[str],

390) -> None:

391 """Checks that each of the listed fields contains a list of non-empty

392 strings or is not present."""

393 assert isinstance(item, dict)

394 for field in fields:

395 lst = item.get(field)

396 if lst is None:

397 continue

398 if not isinstance(lst, (list, tuple)): 398 ↛ 399line 398 didn't jump to line 399 because the condition on line 398 was never true

399 check_error(

400 wxr,

401 dt,

402 word,

403 lang,

404 pos,

405 "{!r} should be a list of dicts: {}".format(

406 field, json.dumps(item, sort_keys=True)

407 ),

408 )

409 continue

410 for x in lst:

411 if not isinstance(x, str) or not x: 411 ↛ 412line 411 didn't jump to line 412 because the condition on line 411 was never true

412 check_error(

413 wxr,

414 dt,

415 word,

416 lang,

417 pos,

418 "{!r} should be a list of non-empty strings: {}".format(

419 field, json.dumps(item, sort_keys=True)

420 ),

421 )

422 break

423

424

425def check_json_data(wxr: WiktextractContext, dt: dict) -> None:

426 """Performs some basic checks on the generated data."""

427 word = dt.get("word", dt.get("title"))

428 if word is None: 428 ↛ 429line 428 didn't jump to line 429 because the condition on line 428 was never true

429 check_error(

430 wxr,

431 dt,

432 None,

433 None,

434 None,

435 'missing "word" or "title" field in data',

436 )

437 return

438 if "title" in dt:

439 return # redirect pages don't have following fields

440 lang = dt.get("lang")

441 if not lang: 441 ↛ 442line 441 didn't jump to line 442 because the condition on line 441 was never true

442 check_error(wxr, dt, word, None, None, 'missing "lang" field in data')

443 return

444 pos = dt.get("pos")

445 if not pos: 445 ↛ 446line 445 didn't jump to line 446 because the condition on line 445 was never true

446 check_error(wxr, dt, word, lang, pos, 'missing "pos" field in data')

447 return

448 if not dt.get("lang_code"): 448 ↛ 449line 448 didn't jump to line 449 because the condition on line 448 was never true

449 check_error(

450 wxr, dt, word, lang, pos, 'missing "lang_code" field in data'

451 )

452 check_tags(wxr, dt, word, lang, pos, dt)

453 check_str_fields(wxr, dt, word, lang, pos, dt, ["etymology_text"])

454 num = dt.get("etymology_number")

455 if num is not None and not isinstance(num, str): 455 ↛ 456line 455 didn't jump to line 456 because the condition on line 455 was never true

456 check_error(

457 wxr,

458 dt,

459 word,

460 lang,

461 pos,

462 '"etymology_number" has been changed to str',

463 )

464 # Check that certain fields, if present, contain lists of dicts

465 if not check_dict_list_fields( 465 ↛ 491line 465 didn't jump to line 491 because the condition on line 465 was never true

466 wxr,

467 dt,

468 word,

469 lang,

470 pos,

471 dt,

472 [

473 "forms",

474 "senses",

475 "synonyms",

476 "antonyms",

477 "hypernyms",

478 "holonyms",

479 "meronyms",

480 "coordinate_terms",

481 "derived",

482 "related",

483 "sounds",

484 "translations",

485 "descendants",

486 "etymology_templates",

487 "head_templates",

488 "inflection_templates",

489 ],

490 ):

491 return # Avoid further processing because it would cause type errors

492 # Check the "forms" field

493 forms = dt.get("forms") or []

494 for form in forms:

495 check_tags(wxr, dt, word, lang, pos, form)

496 tags = dt.get("tags")

497 if not isinstance(tags, (list, tuple)) or "table-tags" not in tags: 497 ↛ 494line 497 didn't jump to line 494 because the condition on line 497 was always true

498 check_str_fields(

499 wxr, dt, word, lang, pos, form, ["form"], mandatory=True

500 )

501 check_str_list_fields(

502 wxr,

503 dt,

504 word,

505 lang,

506 pos,

507 dt,

508 ["categories", "topics", "wikidata", "wikipedia"],

509 )

510 # Check the "senses" field

511 senses = dt.get("senses") or []

512 if not senses: 512 ↛ 513line 512 didn't jump to line 513 because the condition on line 512 was never true

513 check_error(

514 wxr,

515 dt,

516 word,

517 lang,

518 pos,

519 'missing "senses" in data (must have at least one '

520 'sense, add empty sense with "no-gloss" tag if none '

521 "otherwise available)",

522 )

523 return

524 for sense in senses:

525 check_str_list_fields(

526 wxr, dt, word, lang, pos, sense, ["glosses", "raw_glosses"]

527 )

528 # Extra check: should have no-gloss tag if no glosses

529 for field in ("glosses", "raw_glosses"):

530 glosses = sense.get(field) or []

531 if ( 531 ↛ 536line 531 didn't jump to line 536 because the condition on line 531 was never true

532 not glosses

533 and isinstance(sense.get("tags"), str)

534 and "no-gloss" not in sense.get("tags", "").split()

535 ):

536 check_error(

537 wxr,

538 dt,

539 word,

540 lang,

541 pos,

542 "{!r} should have at least one gloss or "

543 '"no-gloss" in "tags"'.format(field),

544 )

545 continue

546 check_tags(wxr, dt, word, lang, pos, sense)

547 check_str_list_fields(

548 wxr,

549 dt,

550 word,

551 lang,

552 pos,

553 sense,

554 ["categories", "topics", "wikidata", "wikipedia"],

555 )

556 check_str_fields(

557 wxr, dt, word, lang, pos, sense, ["english"]

558 ) # DEPRECATED in favor of "translation"

559 check_str_fields(wxr, dt, word, lang, pos, sense, ["translation"])

560 if not check_dict_list_fields( 560 ↛ 580line 560 didn't jump to line 580 because the condition on line 560 was never true

561 wxr,

562 dt,

563 word,

564 lang,

565 pos,

566 sense,

567 [

568 "alt_of",

569 "form_of",

570 "synonyms",

571 "antonyms",

572 "hypernyms",

573 "holonyms",

574 "meronyms",

575 "coordinate_terms",

576 "derived",

577 "related",

578 ],

579 ):

580 continue

581 for field in ("alt_of", "form_of"):

582 lst = sense.get(field)

583 if lst is None:

584 continue

585 for item in lst:

586 check_str_fields(

587 wxr, dt, word, lang, pos, item, ["word"], mandatory=True

588 )

589 check_str_fields(

590 wxr, dt, word, lang, pos, item, ["extra"], mandatory=False

591 )

592

593 for field in (

594 "synonyms",

595 "antonyms",

596 "hypernyms",

597 "holonyms",

598 "meronyms",

599 "coordinate_terms",

600 "derived",

601 "related",

602 ):

603 lst = sense.get(field)

604 if lst is None:

605 continue

606 for item in lst:

607 check_str_fields(

608 wxr, dt, word, lang, pos, item, ["word"], mandatory=True

609 )

610 check_tags(wxr, dt, word, lang, pos, item)

611 check_str_fields(

612 wxr,

613 dt,

614 word,

615 lang,

616 pos,

617 item,

618 [

619 "english", # DEPRECATED in favor of "translation"

620 "translation",

621 "roman",

622 "sense",

623 "taxonomic",

624 ],

625 mandatory=False,

626 empty_ok=True,

627 )

628 check_str_list_fields(

629 wxr, dt, word, lang, pos, item, ["topics"]

630 )

631 # Check the "sounds" field

632 # We will permit having any number of different types (ipa, enpr, etc)

633 # in the same sound entry or in different sound entries.

634 sounds = dt.get("sounds", [])

635 for item in sounds:

636 check_str_fields(

637 wxr,

638 dt,

639 word,

640 lang,

641 pos,

642 item,

643 ["ipa", "enpr", "audio", "ogg_url", "mp3_url", "audio-ipa", "text"],

644 )

645 check_tags(wxr, dt, word, lang, pos, item)

646 check_str_list_fields(

647 wxr, dt, word, lang, pos, item, ["homophones", "hyphenation"]

648 )

649 # Check the "translations" field

650 translations = dt.get("translations") or []

651 for item in translations:

652 check_str_fields(

653 wxr, dt, word, lang, pos, item, ["word"], mandatory=True

654 )

655 check_tags(wxr, dt, word, lang, pos, item)

656 check_str_fields(

657 wxr,

658 dt,

659 word,

660 lang,

661 pos,

662 item,

663 [

664 "alt",

665 "code", # DEPRECATED for "lang_code"

666 "lang_code",

667 "english", # DEPRECATED in favor of "translation"

668 "translation",

669 "lang",

670 "note",

671 "roman",

672 "sense",

673 "taxonomic",

674 ],

675 )

676 if not item.get("lang_code") and not item.get("lang"): 676 ↛ 677line 676 didn't jump to line 677 because the condition on line 676 was never true

677 check_error(

678 wxr,

679 dt,

680 word,

681 lang,

682 pos,

683 '"translations" items must contain at least one '

684 'of "lang_code" and "lang" (normally both): {}'.format(

685 json.dumps(item, sort_keys=True, ensure_ascii=False)

686 ),

687 )

688 # Check the "etymology_templates", "head_templates", and

689 # "inflection_templates" fields

690 for field in [

691 "etymology_templates",

692 "head_templates",

693 "inflection_templates",

694 ]:

695 lst = dt.get(field)

696 if lst is None:

697 continue

698 for item in lst:

699 check_str_fields(

700 wxr, dt, word, lang, pos, item, ["name"], mandatory=True

701 )

702 check_str_fields(

703 wxr,

704 dt,

705 word,

706 lang,

707 pos,

708 item,

709 ["expansion"],

710 # empty_ok=True because there are some templates

711 # that generate empty expansions.

712 mandatory=False,

713 empty_ok=True,

714 )

715 args = item.get("args")

716 if args is None: 716 ↛ 717line 716 didn't jump to line 717 because the condition on line 716 was never true

717 continue

718 if not isinstance(args, dict): 718 ↛ 719line 718 didn't jump to line 719 because the condition on line 718 was never true

719 check_error(

720 wxr,

721 dt,

722 word,

723 lang,

724 pos,

725 '{!r} item "args" value must be a dict: {}'.format(

726 field, json.dumps(args, sort_keys=True)

727 ),

728 )

729 continue

730 for k, v in args.items():

731 if not isinstance(k, str) or not isinstance(v, str): 731 ↛ 732line 731 didn't jump to line 732 because the condition on line 731 was never true

732 check_error(

733 wxr,

734 dt,

735 word,

736 lang,

737 pos,

738 '{!r} item "args" must be a dict with '

739 "string keys and values: {}".format(

740 field, json.dumps(args, sort_keys=True)

741 ),

742 )

743 continue

744 # Check the "descendants" field

745 descendants = dt.get("descendants") or []

746 for item in descendants:

747 check_str_fields(wxr, dt, word, lang, pos, item, ["text"])

748 depth = item.get("depth")

749 if depth is not None and not isinstance(depth, int): 749 ↛ 750line 749 didn't jump to line 750 because the condition on line 749 was never true

750 check_error(

751 wxr,

752 dt,

753 word,

754 lang,

755 pos,

756 '"descentants" field "depth" must be an int',

757 )

758 check_dict_list_fields(wxr, dt, word, lang, pos, item, ["templates"])

759 # XXX should check that they are valid templates, perhaps turn

760 # template checking code above into a function

761

762

763def init_worker(wxr: WiktextractContext) -> None:

764 global worker_wxr

765 worker_wxr = wxr

766 worker_wxr.reconnect_databases()

767 atexit.register(worker_wxr.remove_unpicklable_objects)

768

769

770def reprocess_wiktionary(

771 wxr: WiktextractContext,

772 num_processes: int | None,

773 out_f: TextIO,

774 human_readable: bool = False,

775 search_pattern: str | None = None,

776) -> None:

777 """Reprocesses the Wiktionary from the sqlite db."""

778 logger.info("Second phase - processing pages")

779

780 # Extract thesaurus data. This iterates over thesaurus pages,

781 # but is very fast.

782 if ( 782 ↛ 788line 782 didn't jump to line 788 because the condition on line 782 was always true

783 wxr.config.extract_thesaurus_pages

784 and thesaurus_linkage_number(wxr.thesaurus_db_conn) == 0 # type: ignore[arg-type]

785 ):

786 extract_thesaurus_data(wxr, num_processes)

787

788 emitted = set()

789 process_ns_ids: list[int] = list(

790 {

791 wxr.wtp.NAMESPACE_DATA.get(ns, {}).get("id", 0) # type: ignore[call-overload]

792 for ns in wxr.config.extract_ns_names

793 }

794 )

795 start_time = time.time()

796 last_time = start_time

797 all_page_nums = wxr.wtp.saved_page_nums(

798 process_ns_ids, True, "wikitext", search_pattern

799 )

800 wxr.remove_unpicklable_objects()

801 with ProcessPoolExecutor(

802 max_workers=num_processes,

803 mp_context=get_context(

804 "forkserver" if "forkserver" in get_all_start_methods() else "spawn"

805 ),

806 initializer=init_worker,

807 initargs=(deepcopy(wxr),),

808 ) as executor:

809 wxr.reconnect_databases()

810 for processed_pages, (page_data, wtp_stats) in enumerate(

811 executor.map(

812 page_handler,

813 wxr.wtp.get_all_pages(

814 process_ns_ids, True, "wikitext", search_pattern

815 ),

816 chunksize=100, # default is 1 too slow

817 )

818 ):

819 wxr.config.merge_return(wtp_stats)

820 for dt in page_data:

821 check_json_data(wxr, dt)

822 sanitize_and_write_json_data(dt, out_f, human_readable)

823 word = dt.get("word")

824 lang_code = dt.get("lang_code")

825 pos = dt.get("pos")

826 if word and lang_code and pos:

827 emitted.add((word, lang_code, pos))

828 last_time = estimate_progress(

829 processed_pages, all_page_nums, start_time, last_time

830 )

831

832 if wxr.config.dump_file_lang_code == "en": 832 ↛ 834line 832 didn't jump to line 834 because the condition on line 832 was always true

833 emit_words_in_thesaurus(wxr, emitted, out_f, human_readable)

834 logger.info("Reprocessing wiktionary complete")

835

836

837def process_ns_page_title(page: Page, ns_name: str) -> tuple[str, str]:

838 text: str = page.body if page.body is not None else page.redirect_to # type: ignore[assignment]

839 title = page.title[page.title.find(":") + 1 :]

840 title = re.sub(r"(^|/)\.($|/)", r"\1__dotdot__\2", title)

841 title = re.sub(r"(^|/)\.\.($|/)", r"\1__dotdot__\2", title)

842 title = title.replace("//", "__slashslash__")

843 title = re.sub(r"^/", r"__slash__", title)

844 title = re.sub(r"/$", r"__slash__", title)

845 title = ns_name + "/" + title

846 return title, text

847

848

849def extract_namespace(

850 wxr: WiktextractContext, namespace: str, path: str

851) -> None:

852 """Extracts all pages in the given namespace and writes them to a .tar

853 file with the given path."""

854 logger.info(

855 f"Extracting pages from namespace {namespace} to tar file {path}"

856 )

857 ns_id: int = wxr.wtp.NAMESPACE_DATA.get(namespace, {}).get("id") # type: ignore[assignment, call-overload]

858 t = time.time()

859 with tarfile.open(path, "w") as tarf:

860 for page in wxr.wtp.get_all_pages([ns_id]):

861 title, text = process_ns_page_title(page, namespace)

862 text = text.encode("utf-8")

863 f = io.BytesIO(text)

864 title += ".txt"

865 ti = tarfile.TarInfo(name=title)

866 ti.size = len(text)

867 # According to documentation, TarInfo.mtime can be int, float,

868 # or even None in newer versions, but mypy can't tell because

869 # it's not annotated and assumes it can only be int

870 ti.mtime = t # type: ignore[assignment]

871 ti.uid = 0

872 ti.gid = 0

873 ti.type = tarfile.REGTYPE

874 tarf.addfile(ti, f)