<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This script adds a missing references section to pages.

It goes over multiple pages, searches for pages where <references />
is missing although a <ref> tag is present, and in that case adds a new
references section.

These command line parameters can be used to specify which pages to work on:

&params;

    -xml          Retrieve information from a local XML dump (pages-articles
                  or pages-meta-current, see https://download.wikimedia.org).
                  Argument can also be given as "-xml:filename".

    -namespace:n  Number or name of namespace to process. The parameter can be
                  used multiple times. It works in combination with all other
                  parameters, except for the -start parameter. If you e.g.
                  want to iterate over all categories starting at M, use
                  -start:Category:M.

    -always       Don't prompt you for each replacement.

    -quiet        Use this option to get less output

If neither a page title nor a page generator is given, it takes all pages from
the default maintenance category.

It is strongly recommended not to run this script over the entire article
namespace (using the -start) parameter, as that would consume too much
bandwidth. Instead, use the -xml parameter, or use another way to generate
a list of affected articles
"""
#
# (C) Pywikibot team, 2007-2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, unicode_literals

import re

from functools import partial

import pywikibot

from pywikibot import i18n, pagegenerators, textlib, Bot
from pywikibot.pagegenerators import (
    XMLDumpPageGenerator,
)

# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {
    '&params;': pagegenerators.parameterHelp,
}

# References sections are usually placed before further reading / external
# link sections. This dictionary defines these sections, sorted by priority.
# For example, on an English wiki, the script would place the "References"
# section in front of the "Further reading" section, if that existed.
# Otherwise, it would try to put it in front of the "External links" section,
# or if that fails, the "See also" section, etc.
placeBeforeSections = {
    'ar': [              # no explicit policy on where to put the references
        u'وصلات خارجية',
        u'انظر أيضا',
        u'ملاحظات'
    ],
    'ca': [
        u'Bibliografia',
        u'Bibliografia complementària',
        u'Vegeu també',
        u'Enllaços externs',
        u'Enllaços',
    ],
    'cs': [
        u'Externí odkazy',
        u'Poznámky',
    ],
    'da': [              # no explicit policy on where to put the references
        u'Eksterne links'
    ],
    'de': [              # no explicit policy on where to put the references
        u'Literatur',
        u'Weblinks',
        u'Siehe auch',
        u'Weblink',      # bad, but common singular form of Weblinks
    ],
    'dsb': [
        u'Nožki',
    ],
    'en': [              # no explicit policy on where to put the references
        u'Further reading',
        u'External links',
        u'See also',
        u'Notes'
    ],
    'ru': [
        u'Ссылки',
        u'Литература',
    ],
    'eo': [
        u'Eksteraj ligiloj',
        u'Ekstera ligilo',
        u'Eksteraj ligoj',
        u'Ekstera ligo',
        u'Rete'
    ],
    'es': [
        u'Enlaces externos',
        u'Véase también',
        u'Notas',
    ],
    'fa': [
        u'پیوند به بیرون',
        u'پانویس',
        u'جستارهای وابسته'
    ],
    'fi': [
        u'Kirjallisuutta',
        u'Aiheesta muualla',
        u'Ulkoiset linkit',
        u'Linkkejä',
    ],
    'fr': [
        u'Liens externes',
        u'Lien externe',
        u'Voir aussi',
        u'Notes'
    ],
    'he': [
        u'ראו גם',
        u'לקריאה נוספת',
        u'קישורים חיצוניים',
        u'הערות שוליים',
    ],
    'hsb': [
        u'Nóžki',
    ],
    'hu': [
        u'Külső hivatkozások',
        u'Lásd még',
    ],
    'it': [
        u'Bibliografia',
        u'Voci correlate',
        u'Altri progetti',
        u'Collegamenti esterni',
        u'Vedi anche',
    ],
    'ja': [
        u'関連項目',
        u'参考文献',
        u'外部リンク',
    ],
    'ko': [              # no explicit policy on where to put the references
        u'외부 링크',
        u'외부링크',
        u'바깥 고리',
        u'바깥고리',
        u'바깥 링크',
        u'바깥링크'
        u'외부 고리',
        u'외부고리'
    ],
    'lt': [              # no explicit policy on where to put the references
        u'Nuorodos'
    ],
    'nl': [              # no explicit policy on where to put the references
        u'Literatuur',
        u'Zie ook',
        u'Externe verwijzingen',
        u'Externe verwijzing',
    ],
    'pdc': [
        u'Beweisunge',
        u'Quelle unn Literatur',
        u'Gwelle',
        u'Gwuelle',
        u'Auswenniche Gleecher',
        u'Gewebbgleecher',
        u'Guckt mol aa',
        u'Seh aa',
    ],
    'pl': [
        u'Źródła',
        u'Bibliografia',
        u'Zobacz też',
        u'Linki zewnętrzne',
    ],
    'pt': [
        u'Ligações externas',
        u'Veja também',
        u'Ver também',
        u'Notas',
    ],
    'sk': [
        u'Pozri aj',
    ],
    'sr': [
        'Даље читање',
        'Спољашње везе',
        'Види још',
        'Напомене',
    ],
    'szl': [
        u'Przipisy',
        u'Připisy',
    ],
    'th': [
        u'อ่านเพิ่มเติม',
        u'แหล่งข้อมูลอื่น',
        u'ดูเพิ่ม',
        u'หมายเหตุ',
    ],
    'zh': [
        u'外部链接',
        u'外部連结',
        u'外部連結',
        u'外部连接',
    ],
}

# Titles of sections where a reference tag would fit into.
# The first title should be the preferred one: It's the one that
# will be used when a new section has to be created.
# Except for the first, others are tested as regexes.
referencesSections = {
    'ar': [             # not sure about which ones are preferred.
        u'مراجع',
        u'المراجع',
        u'مصادر',
        u'المصادر',
        u'مراجع ومصادر',
        u'مصادر ومراجع',
        u'المراجع والمصادر',
        u'المصادر والمراجع',
    ],
    'ca': [
        u'Referències',
    ],
    'cs': [
        u'Reference',
        u'Poznámky',
    ],
    'da': [
        u'Noter',
    ],
    'de': [             # see [[de:WP:REF]]
        u'Einzelnachweise',
        u'Anmerkungen',
        u'Belege',
        u'Endnoten',
        u'Fußnoten',
        u'Fuß-/Endnoten',
        u'Quellen',
        u'Quellenangaben',
    ],
    'dsb': [
        u'Nožki',
    ],
    'en': [             # not sure about which ones are preferred.
        u'References',
        u'Footnotes',
        u'Notes',
    ],
    'ru': [
        u'Примечания',
        u'Сноски',
        u'Источники',
    ],
    'eo': [
        u'Referencoj',
    ],
    'es': [
        u'Referencias',
        u'Notas',
    ],
    'fa': [
        u'منابع',
        u'منبع'
    ],
    'fi': [
        u'Lähteet',
        u'Viitteet',
    ],
    'fr': [             # [[fr:Aide:Note]]
        'Notes et références',
        'Notes? et r[ée]f[ée]rences?',
        'R[ée]f[ée]rences?',
        'Notes?',
        'Sources?',
    ],
    'he': [
        u'הערות שוליים',
    ],
    'hsb': [
        u'Nóžki',
    ],
    'hu': [
        u'Források és jegyzetek',
        u'Források',
        u'Jegyzetek',
        u'Hivatkozások',
        u'Megjegyzések',
    ],
    'is': [
        u'Heimildir',
        u'Tilvísanir',
    ],
    'it': [
        u'Note',
        u'Riferimenti',
    ],
    'ja': [
        u'脚注',
        u'脚注欄',
        u'脚注・出典',
        u'出典',
        u'注釈',
        u'註',
    ],
    'ko': [
        u'주석',
        u'각주'
        u'주석 및 참고 자료'
        u'주석 및 참고자료',
        u'주석 및 참고 출처'
    ],
    'lt': [             # not sure about which ones are preferred.
        u'Šaltiniai',
        u'Literatūra',
    ],
    'nl': [             # not sure about which ones are preferred.
        u'Voetnoten',
        u'Voetnoot',
        u'Referenties',
        u'Noten',
        u'Bronvermelding',
    ],
    'pdc': [
        u'Aamarrickunge',
    ],
    'pl': [
        u'Przypisy',
        u'Uwagi',
    ],
    'pt': [
        u'Referências',
    ],
    'sk': [
        u'Referencie',
    ],
    'sr': [
        'Референце',
    ],
    'szl': [
        u'Przipisy',
        u'Připisy',
    ],
    'th': [
        u'อ้างอิง',
        u'เชิงอรรถ',
        u'หมายเหตุ',
    ],
    'zh': [
        u'參考資料',
        u'参考资料',
        u'參考文獻',
        u'参考文献',
        u'資料來源',
        u'资料来源',
    ],
}

# Templates which include a <references /> tag. If there is no such template
# on your wiki, you don't have to enter anything here.
referencesTemplates = {
    'wikipedia': {
        'ar': ['Reflist', 'مراجع', 'ثبت المراجع', 'ثبت_المراجع',
               'بداية المراجع', 'نهاية المراجع', 'المراجع'],
        'be': [u'Зноскі', u'Примечания', u'Reflist', u'Спіс заўваг',
               u'Заўвагі'],
        'be-tarask': [u'Зноскі'],
        'ca': [u'Referències', u'Reflist', u'Listaref', u'Referència',
               u'Referencies', u'Referències2',
               u'Amaga', u'Amaga ref', u'Amaga Ref', u'Amaga Ref2', u'Apèndix'],
        'da': [u'Reflist'],
        'dsb': [u'Referency'],
        'en': [u'Reflist', u'Refs', u'FootnotesSmall', u'Reference',
               u'Ref-list', u'Reference list', u'References-small', u'Reflink',
               u'Footnotes', u'FootnotesSmall'],
        'eo': [u'Referencoj'],
        'es': ['Listaref', 'Reflist', 'muchasref'],
        'fa': [u'Reflist', u'Refs', u'FootnotesSmall', u'Reference',
               u'پانویس', u'پانویس‌ها ', u'پانویس ۲', u'پانویس۲',
               u'فهرست منابع'],
        'fi': [u'Viitteet', u'Reflist'],
        'fr': [u'Références', u'Notes', u'References', u'Reflist'],
        'he': [u'הערות שוליים', u'הערה'],
        'hsb': [u'Referency'],
        'hu': [u'reflist', u'források', u'references', u'megjegyzések'],
        'is': [u'reflist'],
        'it': [u'References'],
        'ja': [u'Reflist', u'脚注リスト'],
        'ko': [u'주석', u'Reflist'],
        'lt': [u'Reflist', u'Ref', u'Litref'],
        'nl': [u'Reflist', u'Refs', u'FootnotesSmall', u'Reference',
               u'Ref-list', u'Reference list', u'References-small', u'Reflink',
               u'Referenties', u'Bron', u'Bronnen/noten/referenties', u'Bron2',
               u'Bron3', u'ref', u'references', u'appendix',
               u'Noot', u'FootnotesSmall'],
        'pl': [u'Przypisy', u'Przypisy-lista', u'Uwagi'],
        'pt': [u'Notas', u'ref-section', u'Referências', u'Reflist'],
        'ru': [u'Reflist', u'Ref-list', u'Refs', u'Sources',
               u'Примечания', u'Список примечаний',
               u'Сноска', u'Сноски'],
        'sr': ['Reflist'],
        'szl': [u'Przipisy', u'Připisy'],
        'th': [u'รายการอ้างอิง'],
        'zh': [u'Reflist', u'RefFoot', u'NoteFoot'],
    },
}

# Text to be added instead of the <references /> tag.
# Define this only if required by your wiki.
referencesSubstitute = {
    'wikipedia': {
        'ar': u'{{مراجع}}',
        'be': u'{{зноскі}}',
        'da': u'{{reflist}}',
        'dsb': u'{{referency}}',
        'fa': u'{{پانویس}}',
        'fi': u'{{viitteet}}',
        'fr': u'{{références}}',
        'he': u'{{הערות שוליים}}',
        'hsb': u'{{referency}}',<commit_msg>noreferences.py: Localise name of template reflist on serbian language<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This script adds a missing references section to pages.

It goes over multiple pages, searches for pages where <references />
is missing although a <ref> tag is present, and in that case adds a new
references section.

These command line parameters can be used to specify which pages to work on:

&params;

    -xml          Retrieve information from a local XML dump (pages-articles
                  or pages-meta-current, see https://download.wikimedia.org).
                  Argument can also be given as "-xml:filename".

    -namespace:n  Number or name of namespace to process. The parameter can be
                  used multiple times. It works in combination with all other
                  parameters, except for the -start parameter. If you e.g.
                  want to iterate over all categories starting at M, use
                  -start:Category:M.

    -always       Don't prompt you for each replacement.

    -quiet        Use this option to get less output

If neither a page title nor a page generator is given, it takes all pages from
the default maintenance category.

It is strongly recommended not to run this script over the entire article
namespace (using the -start) parameter, as that would consume too much
bandwidth. Instead, use the -xml parameter, or use another way to generate
a list of affected articles
"""
#
# (C) Pywikibot team, 2007-2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, unicode_literals

import re

from functools import partial

import pywikibot

from pywikibot import i18n, pagegenerators, textlib, Bot
from pywikibot.pagegenerators import (
    XMLDumpPageGenerator,
)

# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {
    '&params;': pagegenerators.parameterHelp,
}

# References sections are usually placed before further reading / external
# link sections. This dictionary defines these sections, sorted by priority.
# For example, on an English wiki, the script would place the "References"
# section in front of the "Further reading" section, if that existed.
# Otherwise, it would try to put it in front of the "External links" section,
# or if that fails, the "See also" section, etc.
placeBeforeSections = {
    'ar': [              # no explicit policy on where to put the references
        u'وصلات خارجية',
        u'انظر أيضا',
        u'ملاحظات'
    ],
    'ca': [
        u'Bibliografia',
        u'Bibliografia complementària',
        u'Vegeu també',
        u'Enllaços externs',
        u'Enllaços',
    ],
    'cs': [
        u'Externí odkazy',
        u'Poznámky',
    ],
    'da': [              # no explicit policy on where to put the references
        u'Eksterne links'
    ],
    'de': [              # no explicit policy on where to put the references
        u'Literatur',
        u'Weblinks',
        u'Siehe auch',
        u'Weblink',      # bad, but common singular form of Weblinks
    ],
    'dsb': [
        u'Nožki',
    ],
    'en': [              # no explicit policy on where to put the references
        u'Further reading',
        u'External links',
        u'See also',
        u'Notes'
    ],
    'ru': [
        u'Ссылки',
        u'Литература',
    ],
    'eo': [
        u'Eksteraj ligiloj',
        u'Ekstera ligilo',
        u'Eksteraj ligoj',
        u'Ekstera ligo',
        u'Rete'
    ],
    'es': [
        u'Enlaces externos',
        u'Véase también',
        u'Notas',
    ],
    'fa': [
        u'پیوند به بیرون',
        u'پانویس',
        u'جستارهای وابسته'
    ],
    'fi': [
        u'Kirjallisuutta',
        u'Aiheesta muualla',
        u'Ulkoiset linkit',
        u'Linkkejä',
    ],
    'fr': [
        u'Liens externes',
        u'Lien externe',
        u'Voir aussi',
        u'Notes'
    ],
    'he': [
        u'ראו גם',
        u'לקריאה נוספת',
        u'קישורים חיצוניים',
        u'הערות שוליים',
    ],
    'hsb': [
        u'Nóžki',
    ],
    'hu': [
        u'Külső hivatkozások',
        u'Lásd még',
    ],
    'it': [
        u'Bibliografia',
        u'Voci correlate',
        u'Altri progetti',
        u'Collegamenti esterni',
        u'Vedi anche',
    ],
    'ja': [
        u'関連項目',
        u'参考文献',
        u'外部リンク',
    ],
    'ko': [              # no explicit policy on where to put the references
        u'외부 링크',
        u'외부링크',
        u'바깥 고리',
        u'바깥고리',
        u'바깥 링크',
        u'바깥링크'
        u'외부 고리',
        u'외부고리'
    ],
    'lt': [              # no explicit policy on where to put the references
        u'Nuorodos'
    ],
    'nl': [              # no explicit policy on where to put the references
        u'Literatuur',
        u'Zie ook',
        u'Externe verwijzingen',
        u'Externe verwijzing',
    ],
    'pdc': [
        u'Beweisunge',
        u'Quelle unn Literatur',
        u'Gwelle',
        u'Gwuelle',
        u'Auswenniche Gleecher',
        u'Gewebbgleecher',
        u'Guckt mol aa',
        u'Seh aa',
    ],
    'pl': [
        u'Źródła',
        u'Bibliografia',
        u'Zobacz też',
        u'Linki zewnętrzne',
    ],
    'pt': [
        u'Ligações externas',
        u'Veja também',
        u'Ver também',
        u'Notas',
    ],
    'sk': [
        u'Pozri aj',
    ],
    'sr': [
        'Даље читање',
        'Спољашње везе',
        'Види још',
        'Напомене',
    ],
    'szl': [
        u'Przipisy',
        u'Připisy',
    ],
    'th': [
        u'อ่านเพิ่มเติม',
        u'แหล่งข้อมูลอื่น',
        u'ดูเพิ่ม',
        u'หมายเหตุ',
    ],
    'zh': [
        u'外部链接',
        u'外部連结',
        u'外部連結',
        u'外部连接',
    ],
}

# Titles of sections where a reference tag would fit into.
# The first title should be the preferred one: It's the one that
# will be used when a new section has to be created.
# Except for the first, others are tested as regexes.
referencesSections = {
    'ar': [             # not sure about which ones are preferred.
        u'مراجع',
        u'المراجع',
        u'مصادر',
        u'المصادر',
        u'مراجع ومصادر',
        u'مصادر ومراجع',
        u'المراجع والمصادر',
        u'المصادر والمراجع',
    ],
    'ca': [
        u'Referències',
    ],
    'cs': [
        u'Reference',
        u'Poznámky',
    ],
    'da': [
        u'Noter',
    ],
    'de': [             # see [[de:WP:REF]]
        u'Einzelnachweise',
        u'Anmerkungen',
        u'Belege',
        u'Endnoten',
        u'Fußnoten',
        u'Fuß-/Endnoten',
        u'Quellen',
        u'Quellenangaben',
    ],
    'dsb': [
        u'Nožki',
    ],
    'en': [             # not sure about which ones are preferred.
        u'References',
        u'Footnotes',
        u'Notes',
    ],
    'ru': [
        u'Примечания',
        u'Сноски',
        u'Источники',
    ],
    'eo': [
        u'Referencoj',
    ],
    'es': [
        u'Referencias',
        u'Notas',
    ],
    'fa': [
        u'منابع',
        u'منبع'
    ],
    'fi': [
        u'Lähteet',
        u'Viitteet',
    ],
    'fr': [             # [[fr:Aide:Note]]
        'Notes et références',
        'Notes? et r[ée]f[ée]rences?',
        'R[ée]f[ée]rences?',
        'Notes?',
        'Sources?',
    ],
    'he': [
        u'הערות שוליים',
    ],
    'hsb': [
        u'Nóžki',
    ],
    'hu': [
        u'Források és jegyzetek',
        u'Források',
        u'Jegyzetek',
        u'Hivatkozások',
        u'Megjegyzések',
    ],
    'is': [
        u'Heimildir',
        u'Tilvísanir',
    ],
    'it': [
        u'Note',
        u'Riferimenti',
    ],
    'ja': [
        u'脚注',
        u'脚注欄',
        u'脚注・出典',
        u'出典',
        u'注釈',
        u'註',
    ],
    'ko': [
        u'주석',
        u'각주'
        u'주석 및 참고 자료'
        u'주석 및 참고자료',
        u'주석 및 참고 출처'
    ],
    'lt': [             # not sure about which ones are preferred.
        u'Šaltiniai',
        u'Literatūra',
    ],
    'nl': [             # not sure about which ones are preferred.
        u'Voetnoten',
        u'Voetnoot',
        u'Referenties',
        u'Noten',
        u'Bronvermelding',
    ],
    'pdc': [
        u'Aamarrickunge',
    ],
    'pl': [
        u'Przypisy',
        u'Uwagi',
    ],
    'pt': [
        u'Referências',
    ],
    'sk': [
        u'Referencie',
    ],
    'sr': [
        'Референце',
    ],
    'szl': [
        u'Przipisy',
        u'Připisy',
    ],
    'th': [
        u'อ้างอิง',
        u'เชิงอรรถ',
        u'หมายเหตุ',
    ],
    'zh': [
        u'參考資料',
        u'参考资料',
        u'參考文獻',
        u'参考文献',
        u'資料來源',
        u'资料来源',
    ],
}

# Templates which include a <references /> tag. If there is no such template
# on your wiki, you don't have to enter anything here.
referencesTemplates = {
    'wikipedia': {
        'ar': ['Reflist', 'مراجع', 'ثبت المراجع', 'ثبت_المراجع',
               'بداية المراجع', 'نهاية المراجع', 'المراجع'],
        'be': [u'Зноскі', u'Примечания', u'Reflist', u'Спіс заўваг',
               u'Заўвагі'],
        'be-tarask': [u'Зноскі'],
        'ca': [u'Referències', u'Reflist', u'Listaref', u'Referència',
               u'Referencies', u'Referències2',
               u'Amaga', u'Amaga ref', u'Amaga Ref', u'Amaga Ref2', u'Apèndix'],
        'da': [u'Reflist'],
        'dsb': [u'Referency'],
        'en': [u'Reflist', u'Refs', u'FootnotesSmall', u'Reference',
               u'Ref-list', u'Reference list', u'References-small', u'Reflink',
               u'Footnotes', u'FootnotesSmall'],
        'eo': [u'Referencoj'],
        'es': ['Listaref', 'Reflist', 'muchasref'],
        'fa': [u'Reflist', u'Refs', u'FootnotesSmall', u'Reference',
               u'پانویس', u'پانویس‌ها ', u'پانویس ۲', u'پانویس۲',
               u'فهرست منابع'],
        'fi': [u'Viitteet', u'Reflist'],
        'fr': [u'Références', u'Notes', u'References', u'Reflist'],
        'he': [u'הערות שוליים', u'הערה'],
        'hsb': [u'Referency'],
        'hu': [u'reflist', u'források', u'references', u'megjegyzések'],
        'is': [u'reflist'],
        'it': [u'References'],
        'ja': [u'Reflist', u'脚注リスト'],
        'ko': [u'주석', u'Reflist'],
        'lt': [u'Reflist', u'Ref', u'Litref'],
        'nl': [u'Reflist', u'Refs', u'FootnotesSmall', u'Reference',
               u'Ref-list', u'Reference list', u'References-small', u'Reflink',
               u'Referenties', u'Bron', u'Bronnen/noten/referenties', u'Bron2',
               u'Bron3', u'ref', u'references', u'appendix',
               u'Noot', u'FootnotesSmall'],
        'pl': [u'Przypisy', u'Przypisy-lista', u'Uwagi'],
        'pt': [u'Notas', u'ref-section', u'Referências', u'Reflist'],
        'ru': [u'Reflist', u'Ref-list', u'Refs', u'Sources',
               u'Примечания', u'Список примечаний',
               u'Сноска', u'Сноски'],
        'sr': ['Reflist', 'Референце', 'Извори'],
        'szl': [u'Przipisy', u'Připisy'],
        'th': [u'รายการอ้างอิง'],
        'zh': [u'Reflist', u'RefFoot', u'NoteFoot'],
    },
}

# Text to be added instead of the <references /> tag.
# Define this only if required by your wiki.
referencesSubstitute = {
    'wikipedia': {
        'ar': u'{{مراجع}}',
        'be': u'{{зноскі}}',
        'da': u'{{reflist}}',
        'dsb': u'{{referency}}',
        'fa': u'{{پانویس}}',
        'fi': u'{{viitteet}}',
        'fr': u'{{références}}',
        'he': u'{{הערות שוליים}}',
        'hsb': u'{{referency}}',
{'fraction_non_alphanumeric': 0.1503700478885503, 'fraction_numerical': 0.0014801915542011319, 'mean_word_length': 1.8254612546125462, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7284804', 'n_tokens_mistral': 8905, 'n_tokens_neox': 8906, 'n_words': 1995}