<commit_before>#!/usr/bin/python
# -*- coding: utf-8  -*-
from __future__ import division
import pywikibot
import re
from pywikibot import textlib
import urllib2
import StringIO
import struct
import imghdr
import hashlib
import ksp_util

# Files in these categories are small but no thumbnails
VALID_CATEGORIES = { 'Technology tree node images' }

site = pywikibot.getSite()

def find_new_line(string, index, check):
    if index >= 0:
        new_line_found = False
        while check(index, len(string)):
            new_line = string[index] in ["\n", "\r"]
            if new_line_found and not new_line:
                break;
            elif not new_line_found:
                new_line_found = new_line
            start += 1

# read User:BobBot/Thumbnails
thumbs_table = ksp_util.EditTable(site, "User:BobBot/Thumbnails")
thumb_line = re.compile(r"^{{User:BobBot/thumb\|(File:[^|]*)\|([yn?])\|([ac])\|([0-9a-fA-F]{32})}}$", re.M)
hash_regex = re.compile(r"^[0-9a-fA-F]{32}$")
is_thumb_rep = re.compile("{{([Nn]oThumbnailsPlease|[Nn]o thumbnails please)(\|[0-9a-fA-F]{32})?}}")
is_no_thumb_rep = re.compile("{{[Ii]s not thumbnail(\|[0-9a-fA-F]{32})?}}")
entries = thumb_line.findall(thumbs_table.selected)
remainings = []
images_changed = 0
images_is_template = 0
images_not_template = 0
images_updated = 0
images_already_applied = 0
images_conflict = 0
for entry in entries:
    image_page = pywikibot.page.ImagePage(site, entry[0])
    if image_page.exists():
        if entry[1] not in ["y", "n"]:
            remainings += [ entry ]
        else:
            is_thumbnail = entry[1] == "y"
            image_data = urllib2.urlopen(image_page.fileUrl()).read()
            new_hash = hashlib.md5(image_data).hexdigest().lower()
            is_applicable = new_hash.lower() == entry[3].lower() #hash hashn't changed
            content = image_page.text
            templates = textlib.extract_templates_and_params(content)
            is_thumb_hashes = set()
            is_no_thumb_hashes = set()
            for template in templates:
                hashes = None
                if template[0].lower() in ["nothumbnailsplease", "no thumbnails please"]:
                    hashes = is_thumb_hashes
                elif template[0].lower() == "is not thumbnail":
                    hashes = is_no_thumb_hashes
                if "1" in template[1] and hash_regex.match(template[1]["1"]):
                    hashes.add(template[1]["1"])
            if len(is_no_thumb_hashes) == 0 and len(is_thumb_hashes) == 0:
                # automatic
                if not is_applicable:
                    # hash has changed
                    remainings += [ (entry[0], "?", "a", new_hash) ]
                    images_changed += 1
                elif is_thumbnail:
                    image.text = "{{No thumbnails please|" + new_hash + "}}\n" + content
                    #image.save(comment="+add 'no thumbnails please' template;")
                    print("'{}' would've saved:".format(image_page.title))
                    print(image.text)
                    images_is_template += 1
                else:
                    image.text = content + "\n{{Is not thumbnail|" + new_hash + "}}"
                    #image.save(comment="+add 'is not thumbnails' template;")
                    print("'{}' would've saved:".format(image_page.title))
                    print(image.text)
                    images_not_template += 1
            elif len(is_no_thumb_hashes if is_thumbnail else is_thumb_hashes) == 0 and len(is_thumb_hashes if is_thumbnail else is_no_thumb_hashes) == 1:
                # there is already a correct entry, but the hash might need to be updated
                if next(iter(is_thumb_hashes if is_thumbnail else is_no_thumb_hashes)).lower() != new_hash.lower():
                    if is_applicable:
                        # hash has changed, but the answer was for the current hash
                        image.text = (is_thumb_rep if is_thumbnail else is_no_thumb_rep).sub(r"{{" + ("No thumbnails please" if is_thumbnail else "Is not thumbnail") + "|" + new_hash.lower() + "}}", content)
                        #image.save(comment="*update thumbnail template hash;")
                        print("'{}' would've saved:".format(image_page.title))
                        print(image.text)
                        images_updated += 1
                    else:
                        remainings += [ (entry[0], "?", "a", new_hash) ]
                        images_changed += 1
                else:
                    print("Image '{}' is already marked accordingly. No change necessary.".format(image_page.title()))
                    images_already_applied += 1
            else:
                # conflict, please resolve manually
                remainings += [ (entry[0], "?", "c", new_hash) ]
                images_conflict += 1
    else:
        print("Image '{}' does not exists anymore. Skipped.".format(image_page.title()))
        images_already_applied += 1

# Files in these categories are small but no thumbnails
VALID_CATEGORIES = { 'Technology tree node images' }

names = re.compile("^.*\.(?:png|gif|jpg|jpeg)$", re.I)

print("Searching for new thumbnails.")
images_total = 0
images_unmarked = 0
images_detected = 0
images_outdated = 0
images_multiple = 0
#Maximum size is 20 KiB (20×2²⁰ bytes)
for image in site.allimages(maxsize=20*2<<10):
    for remaining in remainings:
        if remaining[0] == image.title():
            in_remaining = True
            break
    else:
        in_remaining = False
    if not in_remaining and names.match(image.title()):
        images_total += 1
        content = image.text
        valid = []
        for c in image.categories():
            if c.title(withNamespace=False) in VALID_CATEGORIES:
                valid += [c.title(withNamespace=False)]
        if valid:
            print("Image '{}' is in valid category/categories '{}'. Skipped.".format(image.title(), "', '".join(valid)))
        else:
            image_data = urllib2.urlopen(image.fileUrl()).read()
            new_hash = hashlib.md5(image_data).hexdigest().lower()
            templates = textlib.extract_templates_and_params(content)
            hashes = set()
            for template in templates:
                if (template[0].lower() in ["nothumbnailsplease", "no thumbnails please"] or template[0].lower() == "is not thumbnail") and "1" in template[1] and template[1]["1"] and hash_regex.match(template[1]["1"]):
                    hashes.add(template[1]["1"])
            if len(hashes) == 0:
                images_unmarked += 1
                size = ksp_util.get_image_size(image_data)
                if size:
                    # the values are switched for the second statement
                    if size[0]/size[1] > 0.4 and size[1]/size[0] > 0.4 and (size[0] < 100 or size[1] < 100):
                        images_detected += 1
                        remainings += [ (image.title(), "?", "a", hashlib.md5(image_data).hexdigest()) ]
                        print("Image '{}' fit's size constraints.".format(image.title()))
                else:
                    print("Unable to determine size of '{}'! Skipped.".format(image.title()))
            elif len(hashes) == 1 and hashes.pop() != new_hash:
                images_outdated += 1
                remainings += [ (image.title(), "?", "a", new_hash) ]
                print("Image '{}' has the thumbnail info set, but with another hash.".format(image.title()))
            elif len(hashes) > 1:
                # conflict, please resolve manually
                images_multiple += 1
                remainings += [ (entry[0], "?", "c", new_hash) ]
                print("Image '{}' has multiple thumbnail info entries.".format(image.title()))
if images_total == 0:
    print("No images found.")
else:
    print("Statistics: {} images, {} ({:.0%}) already unmarked, {} ({:.0%}) detected, {} ({:.0%}) outdated, {} ({:.0%}) multiple.".format(images_total, images_unmarked, images_unmarked / images_total, images_detected, images_detected / images_total, images_outdated, images_outdated / images_total, images_multiple, images_multiple / images_total))

# generate new list
line = "\n".join([ "{{{{User:BobBot/thumb|{}|{}|{}|{}}}}}".format(*remaining) for remaining in remainings ]) + "\n"

thumbs_table.splice(line)
thumbs_table.page.save(comment="+{} outdated; +{} multiple; +{} new; *{} changed; *{} conflicts; -{} already applied; -{} 'is thumbnail'; -{} 'is not thumbnail'; -{} hash updated;".format(images_outdated, images_multiple, images_detected, images_changed, images_conflict, images_already_applied, images_is_template, images_not_template, images_updated))
<commit_msg>+new handler for thumbnails (more automatic);<commit_after>#!/usr/bin/python
# -*- coding: utf-8  -*-
from __future__ import division
import pywikibot
import re
from pywikibot import textlib
from pywikibot import page
import urllib2
import StringIO
import struct
import imghdr
import hashlib
import ksp_util

import mwparserfromhell

# Files in these categories are small but no thumbnails
VALID_CATEGORIES = { 'Technology tree node images' }

site = pywikibot.getSite()

def set_hash(template, image_hash):
    if template.has("1"):
        for param in template.params:
            if param.name == "1":
                param.value = image_hash
                break
    else:
        template.add("1", image_hash)

def no_template_or_param_inequal(template, param, value, accept_empty=True):
    return not template or (accept_empty and not template.has(param, ignore_empty=True) ) or (template.has(param, ignore_empty=True) and template.get(param).lower() != value)

log_entries = []

names = re.compile("^.*\.(?:png|gif|jpg|jpeg)$", re.I)
handled = set()

def check_image(image, handled):
    if image.title() not in handled and names.match(image.title()):
        handled.add(image.title())
        image_data = urllib2.urlopen(image.fileUrl()).read()
        new_hash = hashlib.md5(image_data).hexdigest().lower()
        valid = []
        for category in image.categories():
            if category.title(withNamespace=False) in VALID_CATEGORIES:
                valid += [category.title(withNamespace=False)]
        if valid:
            print("Image '{}' is in valid category/categories '{}'. Skipped.".format(image.title(), "', '".join(valid)))
        else:
            parsed = mwparserfromhell.parse(image.text)
            try:
                templates = parsed.ifilter_templates(recursive=mwparserfromhell.wikicode.RECURSE_OTHERS)
            except AttributeError: #mwparserfromhell v0.4 required, "dummy" mode might work too:
                templates = parsed.ifilter_templates(recursive=True)
            concerned_templates = [template for template in templates if template.name.lower() in ["nothumbnailsplease", "no thumbnails please", "is not thumbnail"]]
            if len(concerned_templates) > 1:
                print("CONFLICT! Multiple thumbnail templates for image '{}'.".format(image.title()))
                log_entries.append((image.title(), "c", new_hash))
                pass
            else:
                concerned_template = concerned_templates[0] if concerned_templates else None
                # Das "nicht leer" ist optional deaktivierbar
                if no_template_or_param_inequal(concerned_template, "1", new_hash):
                    if no_template_or_param_inequal(concerned_template, "2", "skip"):
                        size = ksp_util.get_image_size(image_data)
                        if size:
                            # the values are switched for the second statement
                            is_thumbnail = size[0]/size[1] > 0.4 and size[1]/size[0] > 0.4 and (size[0] < 100 or size[1] < 100)
                            if is_thumbnail:
                                print("Image '{}' fit's size constraints.".format(image.title()))
                        else:
                            is_thumbnail = None #wtf?
                            print("Unable to determine size of '{}'! Skipped.".format(image.title()))
                    else:
                        is_thumbnail = concerned_template.name.lower() != "is not thumbnail"
                    if is_thumbnail is not None:
                        if concerned_template:
                            # it was a thumbnail, but the new one isn't anymore
                            if concerned_template.name.lower() != "is not thumbnail" and not is_thumbnail:
                                comment = "-is no thumbnail anymore;"
                                #TODO: Remove template: Does ↓ work?
                                parsed.remove(concerned_template) # noinclude remove too?
                                log_entries.append((image.title(), "-", new_hash))
                            else:
                                set_hash(concerned_template, new_hash)
                                comment = "*update thumbnail template hash;"
                                if is_thumbnail and concerned_template.name.lower() == "is not thumbnail":
                                    concerned_template.name = "No thumbnails please"
                                    comment += " *changed to marked as thumbnail;"
                                    log_entries.append((image.title(), "++", new_hash))
                                else:
                                    log_entries.append((image.title(), "*", new_hash))
                            image.text = unicode(parsed)
                            image.save(comment=comment)
                        elif is_thumbnail:
                            image.text = "{{No thumbnails please|" + new_hash + "}}" + image.text
                            image.save(comment="+no thumbnails please template;")
                            log_entries.append((image.title(), "+", new_hash))

for image in page.Category(site, "Category:Image thumbnails").articles(namespaces=6):
    check_image(page.ImagePage(image), handled)

print("Finished checking already marked images. Search for new images.")

for image in site.allimages(maxsize=20*2<<10):
    check_image(image, handled)

if log_entries:
    table = ksp_util.EditTable(site, "User:BobBot/The Thumbnail Job")

    log_entries.sort(key=lambda entry: entry[0])
    line = "\n".join([ "{{{{User:BobBot/The Thumbnail Job/entry|{}|{}|{}}}}}".format(*log_entry) for log_entry in log_entries ]) + "\n"
    table.splice(line, False)
    table.page.save(comment="+added changes;")

{'fraction_non_alphanumeric': 0.12377358490566037, 'fraction_numerical': 0.010291595197255575, 'mean_word_length': 2.1427339370418284, 'pattern_counts': {'":': 3, '<': 11, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4373157', 'n_tokens_mistral': 4341, 'n_tokens_neox': 4128, 'n_words': 998}