<commit_before>"""Extract all the page ids to a csv file."""

import csv

import mwxml

from .. import utils


def configure_subparsers(subparsers):
    """Configure the subparsers."""
    parser = subparsers.add_parser(
        'extract-page-ids',
        help='''Extract the page ids from the text.''',
    )

    parser.set_defaults(func=main)


def main(dump: mwxml.Dump,
         features_output_h,
         stats_output_h,
         args):
    """Main function that parses the arguments and writes the output."""
    print(args)

    with features_output_h:
        csvwriter = csv.writer(features_output_h)

        for mw_page in dump:
            utils.log('Analyzing', mw_page.title)
            csvwriter.writerow((mw_page.id, mw_page.title))<commit_msg>pages-id-extractor needs the project name<commit_after>"""Extract all the page ids to a csv file."""

import csv

import mwxml

from .. import utils


def configure_subparsers(subparsers):
    """Configure the subparsers."""
    parser = subparsers.add_parser(
        'extract-page-ids',
        help='''Extract the page ids from the text.''',
    )
    parser = parser.add_argument(
        '--project', '-p',
        required=True,
        help='''Porject name (en, it, ...)''',
    )

    parser.set_defaults(func=main)


def main(dump: mwxml.Dump,
         features_output_h,
         stats_output_h,
         args):
    """Main function that parses the arguments and writes the output."""
    print(args)

    project = args.project

    with features_output_h:
        csvwriter = csv.writer(features_output_h)

        for mw_page in dump:
            utils.log('Analyzing', mw_page.title)
            csvwriter.writerow((project, mw_page.id, mw_page.title))
{'fraction_non_alphanumeric': 0.11502890173410404, 'fraction_numerical': 0.0, 'mean_word_length': 2.667372881355932, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '515431', 'n_tokens_mistral': 558, 'n_tokens_neox': 537, 'n_words': 137}