<commit_before>#!/usr/bin/env python
# coding: utf-8

import json
import os
import re

import pymysql
os.environ['PYWIKIBOT_DIR'] = os.path.dirname(os.path.realpath(__file__))
import pywikibot

from config import config_page_name, host, password, user  # pylint: disable=E0611,W0614


site = pywikibot.Site('zh', 'wikipedia')
site.login()

# config_page = pywikibot.Page(site, config_page_name)
# cfg = config_page.text
# cfg = json.loads(cfg)
# print(json.dumps(cfg, indent=4, ensure_ascii=False))

# if not cfg['enable']:
#     exit('disabled\n')

conn = pymysql.connect(
    host=host,
    user=user,
    password=password,
    charset="utf8"
)

with conn.cursor() as cur:
    cur.execute('use zhwiki_p')
    cur.execute("""
        SELECT tl_namespace, tl_title, page_id, pr_level
        FROM
        (
            SELECT DISTINCT tl_namespace, tl_title
            FROM templatelinks
            WHERE tl_from_namespace = 8 AND tl_namespace != 8 AND tl_namespace % 2 = 0
            AND tl_from NOT IN (6709868, 6743456, 5545686, 5733059, 6486575, 3073030)
        ) t1
        LEFT JOIN page ON tl_namespace = page_namespace AND tl_title = page_title
        LEFT JOIN page_restrictions ON pr_page = page_id AND pr_type = 'edit'
        WHERE pr_level != 'sysop'
    """)
    res = cur.fetchall()

whitelist = [
    68860,  # TOWitem
    79357,  # TOWpercent
    71082,  # Bulletin
    1224493,  # Bulletin/maintenance
    1612163,  # Recent_changes_article_requests/list
    84599,  # 新条目推荐/候选
]

protectargs = {
    'reason': '[[Wikipedia:保護方針#永久保护|嵌入在MediaWiki命名空間的頁面]]',
    'prompt': False,
    'protections': {
        'edit': 'sysop',
        'move': 'sysop',
    }
}

for row in res:
    ns = int(row[0])
    title = row[1].decode()
    pageid = int(row[2])

    if '历史上的今天' in title:
        continue
    if pageid in whitelist:
        continue

    page = pywikibot.Page(site, title, ns)
    print(ns, title, pageid, page.title())
    page.protect(**protectargs)<commit_msg>Output list to a page<commit_after>#!/usr/bin/env python
# coding: utf-8

import json
import os
import re

import pymysql
os.environ['PYWIKIBOT_DIR'] = os.path.dirname(os.path.realpath(__file__))
import pywikibot

from config import config_page_name, host, password, user  # pylint: disable=E0611,W0614


site = pywikibot.Site('zh', 'wikipedia')
site.login()

config_page = pywikibot.Page(site, config_page_name)
cfg = config_page.text
cfg = json.loads(cfg)
print(json.dumps(cfg, indent=4, ensure_ascii=False))

if not cfg['enable']:
    exit('disabled\n')

conn = pymysql.connect(
    host=host,
    user=user,
    password=password,
    charset="utf8"
)

mediawiki_whitelist = []
for title in cfg['mediawiki_whitelist']:
    page = pywikibot.Page(site, title)
    mediawiki_whitelist.append(str(page.pageid))

mediawiki_whitelist = ', '.join(mediawiki_whitelist)

with conn.cursor() as cur:
    cur.execute('use zhwiki_p')
    cur.execute("""
        SELECT tl_namespace, tl_title, tl_from
        FROM templatelinks
        WHERE tl_from_namespace = 8 AND tl_namespace != 8
        AND tl_from NOT IN ({})
    """.format(mediawiki_whitelist))
    res = cur.fetchall()

templates = dict()
for row in res:
    ns = int(row[0])
    title = row[1].decode()
    tl_from = int(row[2])
    if (ns, title) not in templates:
        templates[(ns, title)] = set()
    templates[(ns, title)].add(tl_from)

titles = []
for (ns, title) in templates:
    page = pywikibot.Page(site, title, ns)
    fulltitle = page.title()
    if re.search(cfg['template_whiteregex'], fulltitle):
        continue

    titles.append(fulltitle)

titles.sort()

text = '\n'
for title in titles:
    # print(title)
    text += '| ' + title + '\n'

page = pywikibot.Page(site, cfg['page'])
try:
    idx1 = page.text.index('<!--listbegins-->') + len('<!--listbegins-->')
    idx2 = page.text.index('<!--listends-->', idx1)
except ValueError:
    print('Cannot locate insertion position')
    exit()

newtext = page.text[:idx1] + text + page.text[idx2:]
pywikibot.showDiff(page.text, newtext)

page.text = newtext
page.save(cfg['summary'])
{'fraction_non_alphanumeric': 0.13616398243045388, 'fraction_numerical': 0.028306490971205467, 'mean_word_length': 3.3560042507970245, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4659770', 'n_tokens_mistral': 1591, 'n_tokens_neox': 1510, 'n_words': 330}