<commit_before>#!/usr/bin/env python3
# -*- coding: utf8 -*-
################################################################################
##
## Copyright (C) 2012 Typhos
##
## This Source Code Form is subject to the terms of the Mozilla Public
## License, v. 2.0. If a copy of the MPL was not distributed with this
## file, You can obtain one at http://mozilla.org/MPL/2.0/.
##
################################################################################

import bplib
import bplib.json

class Context(object):
    def __init__(self):
        self.config = None
        self.tag_config = None
        self.sources = {}
        self.drops = {}
        self.next_source_id = 0

    def load_config(self):
        self.config = bplib.load_yaml_file(open("data/rules.yaml"))
        self.tag_config = bplib.load_yaml_file(open("data/tags.yaml"))

    def load_sources(self):
        for sr_key in self.config["Subreddits"]:
            self.load_subreddit(sr_key)

    def load_subreddit(self, sr_key):
        emotes_filename = "emotes/" + sr_key + ".json"
        tag_filename = "tags/" + sr_key + ".json"

        try:
            emote_data = bplib.json.load(open(emotes_filename))
        except IOError:
            return None # Doesn't exist or has no emotes

        try:
            tag_data = bplib.json.load(open(tag_filename))
        except IOError:
            tag_data = {} # No tags is fine

        source_name = "r/" + sr_key
        sr = Source.load(source_name, emote_data, tag_data, context=self)
        sr.source_id = self.next_source_id
        self.next_source_id += 1
        self.sources[source_name] = sr
        return sr

class Source(object):
    def __init__(self, name, emotes, context=None):
        self.name = name
        self.emotes = emotes
        self.context = context

        self.variant_matches = None
        self.emote_groups = None
        self._tag_data = {} # For checktags

    def merge(self, emote_data, tag_data):
        for (name, data) in emote_data.items():
            tags = set(tag_data.get(name, []))
            self.emotes[name] = Emote.load(name, data, tags, source=self, context=self.context)

            if self.context:
                if "+drop" in tags:
                    if name in self.context.drops:
                        print("Source.load_from_data(): ERROR: %s in %s is marked as +drop, but %s has already claimed the name" %
                                (name, self.name, self.context.drops[name].name))
                    self.context.drops[name] = self
        self._tag_data.update(tag_data)

    @classmethod
    def load(cls, name, emote_data, tag_data, context=None):
        source = cls(name, {}, context=context)
        source.merge(emote_data, tag_data)
        return source

    def dump(self):
        return {name: emote.dump() for (name, emote) in self.emotes.items()}

    def dump_tags(self):
        data = {}
        for (name, emote) in self.emotes.items():
            if emote.tags:
                data[name] = sorted(emote.tags)
        return data

    def dropped_emotes(self):
        # Returns emotes with +drop
        for emote in self.emotes.values():
            if emote.ignore:
                continue
            owner = self.context.drops.get(emote.name, None)
            if owner is not None and owner is not self:
                yield emote

    def undropped_emotes(self):
        # Returns emotes that don't have a +drop applied to them
        for emote in self.emotes.values():
            if emote.ignore:
                continue
            owner = self.context.drops.get(emote.name, None)
            if owner is None or owner is self:
                yield emote

    def is_ignored(self, emote):
        if emote.ignore:
            return True
        if "+remove" in emote.all_tags(self.context):
            return True
        owner = self.context.drops.get(emote.name, None)
        if owner is not None and owner is not self:
            return True
        return False

    def unignored_emotes(self):
        # Returns emotes that actually matter (not removed and not dropped)
        for emote in self.emotes.values():
            if self.is_ignored(emote):
                continue
            yield emote

    def group_emotes(self):
        groups = {}
        for emote in self.unignored_emotes():
            base = emote.base_variant()
            if emote.name[:2].lower() == "/r":
                # Try to follow reversed emotes back to the unreversed name
                r_emote = self.emotes.get("/" + emote.name[2:])
                if r_emote is not None and not self.is_ignored(r_emote):
                    base = r_emote.base_variant()
            if hasattr(base, "info_set"):
                info = base.info_set()<commit_msg>Fix group_emotes() to include +remove'ed emotes, which matters to tagapp<commit_after>#!/usr/bin/env python3
# -*- coding: utf8 -*-
################################################################################
##
## Copyright (C) 2012 Typhos
##
## This Source Code Form is subject to the terms of the Mozilla Public
## License, v. 2.0. If a copy of the MPL was not distributed with this
## file, You can obtain one at http://mozilla.org/MPL/2.0/.
##
################################################################################

import bplib
import bplib.json

class Context(object):
    def __init__(self):
        self.config = None
        self.tag_config = None
        self.sources = {}
        self.drops = {}
        self.next_source_id = 0

    def load_config(self):
        self.config = bplib.load_yaml_file(open("data/rules.yaml"))
        self.tag_config = bplib.load_yaml_file(open("data/tags.yaml"))

    def load_sources(self):
        for sr_key in self.config["Subreddits"]:
            self.load_subreddit(sr_key)

    def load_subreddit(self, sr_key):
        emotes_filename = "emotes/" + sr_key + ".json"
        tag_filename = "tags/" + sr_key + ".json"

        try:
            emote_data = bplib.json.load(open(emotes_filename))
        except IOError:
            return None # Doesn't exist or has no emotes

        try:
            tag_data = bplib.json.load(open(tag_filename))
        except IOError:
            tag_data = {} # No tags is fine

        source_name = "r/" + sr_key
        sr = Source.load(source_name, emote_data, tag_data, context=self)
        sr.source_id = self.next_source_id
        self.next_source_id += 1
        self.sources[source_name] = sr
        return sr

class Source(object):
    def __init__(self, name, emotes, context=None):
        self.name = name
        self.emotes = emotes
        self.context = context

        self.variant_matches = None
        self.emote_groups = None
        self._tag_data = {} # For checktags

    def merge(self, emote_data, tag_data):
        for (name, data) in emote_data.items():
            tags = set(tag_data.get(name, []))
            self.emotes[name] = Emote.load(name, data, tags, source=self, context=self.context)

            if self.context:
                if "+drop" in tags:
                    if name in self.context.drops:
                        print("Source.load_from_data(): ERROR: %s in %s is marked as +drop, but %s has already claimed the name" %
                                (name, self.name, self.context.drops[name].name))
                    self.context.drops[name] = self
        self._tag_data.update(tag_data)

    @classmethod
    def load(cls, name, emote_data, tag_data, context=None):
        source = cls(name, {}, context=context)
        source.merge(emote_data, tag_data)
        return source

    def dump(self):
        return {name: emote.dump() for (name, emote) in self.emotes.items()}

    def dump_tags(self):
        data = {}
        for (name, emote) in self.emotes.items():
            if emote.tags:
                data[name] = sorted(emote.tags)
        return data

    def dropped_emotes(self):
        # Returns emotes with +drop
        for emote in self.emotes.values():
            if emote.ignore:
                continue
            owner = self.context.drops.get(emote.name, None)
            if owner is not None and owner is not self:
                yield emote

    def undropped_emotes(self):
        # Returns emotes that don't have a +drop applied to them
        for emote in self.emotes.values():
            if emote.ignore:
                continue
            owner = self.context.drops.get(emote.name, None)
            if owner is None or owner is self:
                yield emote

    def is_ignored(self, emote):
        if emote.ignore:
            return True
        if "+remove" in emote.all_tags(self.context):
            return True
        owner = self.context.drops.get(emote.name, None)
        if owner is not None and owner is not self:
            return True
        return False

    def unignored_emotes(self):
        # Returns emotes that actually matter (not removed and not dropped)
        for emote in self.emotes.values():
            if self.is_ignored(emote):
                continue
            yield emote

    def group_emotes(self):
        groups = {}
        for emote in self.undropped_emotes():
            base = emote.base_variant()
            if emote.name[:2].lower() == "/r":
                # Try to follow reversed emotes back to the unreversed name
                r_emote = self.emotes.get("/" + emote.name[2:])
                if r_emote is not None and not self.is_ignored(r_emote):
                    base = r_emote.base_variant()
            if hasattr(base, "info_set"):
                info = base.info_set()
{'fraction_non_alphanumeric': 0.1320559295701709, 'fraction_numerical': 0.002900051786639047, 'mean_word_length': 2.013732833957553, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7413255', 'n_tokens_mistral': 2914, 'n_tokens_neox': 2801, 'n_words': 766}