<commit_before>import ipfs_pubsub
from base64 import b64decode
import os
import logging

pepenet_channels = {"upload": "dev_pepenet_pepes_upload",
                    "update_control": "dev_pepenet_pepes_update_control",
                    "update": "dev_pepenet_pepes_update"
                    }
control_codes = {"updating_finished": "0"

                 }


def load_hash_set(path):
    """Loads a local copy of pepe hashes, useful for banning locally
    malicious files or protecting the dht"""
    hashes = set()
    try:
        with open(path, "r") as hash_list:
            for i in hash_list:
                hashes.update((i.rstrip(),))
    except FileNotFoundError:
        open(path, "w").close()
    logging.debug("Loaded {} as {}".format(path, hashes))
    return hashes


def save_hash_set(path, hash_set):
    """Loads a local copy of pepe hashes, useful for banning locally
    malicious files or protecting the dht"""
    with open(path, "w") as hash_list:
        for i in hash_set:
            if i != "":
                hash_list.write(i + "\n")
    logging.debug("saved {} as {}".format(path, hash_set))


class PepeMan:
    def __init__(self,
                 ipfs_conn,
                 database="rare_pepes",
                 banned_pepes_path="rare_pepes/avoid_pepes.db",
                 local_pepes_path="rare_pepes/known_pepes.db"):
        self.pubsub = ipfs_pubsub.PubSub()
        self.ipfs_conn = ipfs_conn
        self.database = database

        # List of pepe hashes that you've banned LOCALLY, this helps in
        # avoiding unwanted content to show up in your browser
        self.banned_pepes = load_hash_set(banned_pepes_path)
        self.banned_pepes_path = banned_pepes_path

        # Local list of pepes
        self.local_pepes = load_hash_set(local_pepes_path)
        self.local_pepes_path = local_pepes_path
        for key in pepenet_channels:
            self.pubsub.topic_sub(pepenet_channels[key])

        logging.debug("Topics subscribed: {}".format(self.pubsub.topic_ls()))

        self.pubsub.topic_set_callback(pepenet_channels["update_control"],
                                       self._on_update_control)

        self.pubsub.topic_set_callback(pepenet_channels["upload"],
                                       self._on_upload)

        self.update_pepe_list(self.local_pepes)

    def update_pepe_list(self, rare_pepes_set):
        """
            basically a client publishes his entire pepes hash list
            then other clients read that list and publish the missing hashes
            now all the clients should have an updated list
            PS: malicious clients could DOS the network by uploading a
            giant list
            The pepe list should be updated by the _on_update_control callback
        """
        for pepe in rare_pepes_set:
            self.pubsub.topic_pub(pepenet_channels["update"], pepe)

        self.pubsub.topic_pub(pepenet_channels["update_control"],
                              control_codes["updating_finished"])

    def _on_update_control(self, data):
        """Callback when pubsub updates
        data has the following keys: ["content", "sender", "timestamp"]"""
        logging.debug("Received. {}".format(data))
        code = b64decode(data["content"]).decode('utf8')
        if code == control_codes["updating_finished"]:
            self._on_update_request(data["sender"])

    def _on_update_request(self, sender_id):
        """
            After the updater has sent his uploading_finished CC
            we upload our hashes
        """
        pepe_hash = []
        encoded_pepe_hash = self.pubsub.\
            topic_pop_messages_from_sender(pepenet_channels["update"],
                                           sender_id)
        logging.debug("update_request: {}".format(encoded_pepe_hash))
        for i in encoded_pepe_hash:
            logging.debug("decoding {} decoded {}".format(i, b64decode(i)))
            pepe_hash.append(b64decode(i).decode("utf-8"))

        logging.debug("updating hashes: {}".format(pepe_hash))
        # We remove the pepes that we don't want
        received_hashes = set(pepe_hash) - self.banned_pepes
        # Then we check if they've sent us some new hashes
        new_hash_delta = received_hashes - self.local_pepes

        if (received_hashes not in self.local_pepes):
            delta = self.local_pepes - received_hashes
            for i in delta:
                self.pubsub.topic_pub(pepenet_channels["update"], i)

        self.local_pepes.update(new_hash_delta)

    def _on_upload(self, data):
        """Callback called when someone on the network uploads a pepe
        self.local_pepes is set, so if the pepe is a duplicate i will not
        be added twice"""

        self._register_pepe(b64decode(data["content"]).decode('utf8'))
        logging.info("Received pepe: {}".format(b64decode(data["content"])
                                                .decode('utf8')))

    def get_peer_number(self):
        "Returns the number of connected peers to the network"
        return self.pubsub.topic_peer_number(pepenet_channels["update"])

    def get_pepe(self, pepe_hash):
        "Downloads pepe from ipfs and saves it in database folder"<commit_msg>* Fixed bug where a client would flood the udpate channel * Fixed bug where a connecting client wouldn't update it's pepe list<commit_after>import ipfs_pubsub
from base64 import b64decode
import os
import logging

pepenet_channels = {"upload": "dev_pepenet_pepes_upload",
                    "update_control": "dev_pepenet_pepes_update_control",
                    "update": "dev_pepenet_pepes_update"
                    }
control_codes = {"updating_finished": "0"

                 }


def load_hash_set(path):
    """Loads a local copy of pepe hashes, useful for banning locally
    malicious files or protecting the dht"""
    hashes = set()
    try:
        with open(path, "r") as hash_list:
            for i in hash_list:
                hashes.update((i.rstrip(),))
    except FileNotFoundError:
        os.makedirs("/".join(path.split("/")[:-1]), exist_ok=True)
        open(path, "w").close()
    logging.debug("Loaded {} as {}".format(path, hashes))
    return hashes


def save_hash_set(path, hash_set):
    """Loads a local copy of pepe hashes, useful for banning locally
    malicious files or protecting the dht"""
    with open(path, "w") as hash_list:
        for i in hash_set:
            if i != "":
                hash_list.write(i + "\n")
    logging.debug("saved {} as {}".format(path, hash_set))


class PepeMan:
    def __init__(self,
                 ipfs_conn,
                 database="rare_pepes",
                 banned_pepes_path="rare_pepes/avoid_pepes.db",
                 local_pepes_path="rare_pepes/known_pepes.db"):
        self.pubsub = ipfs_pubsub.PubSub()
        self.ipfs_conn = ipfs_conn
        self.database = database

        # List of pepe hashes that you've banned LOCALLY, this helps in
        # avoiding unwanted content to show up in your browser
        self.banned_pepes = load_hash_set(banned_pepes_path)
        self.banned_pepes_path = banned_pepes_path

        # Local list of pepes
        self.local_pepes = load_hash_set(local_pepes_path) - self.banned_pepes
        self.local_pepes_path = local_pepes_path
        for key in pepenet_channels:
            self.pubsub.topic_sub(pepenet_channels[key])

        logging.debug("Topics subscribed: {}".format(self.pubsub.topic_ls()))

        self.pubsub.topic_set_callback(pepenet_channels["update_control"],
                                       self._on_update_control)

        self.pubsub.topic_set_callback(pepenet_channels["upload"],
                                       self._on_upload)

        self.update_pepe_list(self.local_pepes)

    def update_pepe_list(self, rare_pepes_set):
        """
            basically a client publishes his entire pepes hash list
            then other clients read that list and publish the missing hashes
            now all the clients should have an updated list
            PS: malicious clients could DOS the network by uploading a
            giant list
            The pepe list should be updated by the _on_update_control callback
        """
        for pepe in rare_pepes_set:
            self.pubsub.topic_pub(pepenet_channels["update"], pepe)

        self.pubsub.topic_pub(pepenet_channels["update_control"],
                              control_codes["updating_finished"])

    def _on_update_control(self, data):
        """Callback when pubsub updates
        data has the following keys: ["content", "sender", "timestamp"]"""
        logging.debug("Received. {}".format(data))
        code = b64decode(data["content"]).decode('utf8')
        if code == control_codes["updating_finished"]:
            self._on_update_request(data["sender"])

    def _on_update_request(self, sender_id):
        """
            After the updater has sent his uploading_finished CC
            we upload our hashes
        """
        pepe_hash = []
        encoded_pepe_hash = self.pubsub.\
            topic_pop_messages_from_sender(pepenet_channels["update"],
                                           sender_id)

        for i in encoded_pepe_hash:
            logging.debug("decoding {} decoded {}".format(i, b64decode(i)))
            pepe_hash.append(b64decode(i).decode("utf-8"))

        logging.debug("updating hashes: {}".format(pepe_hash))
        # We remove the pepes that we don't want
        received_hashes = set(pepe_hash) - self.banned_pepes
        # Then we check if they've sent us some new hashes
        new_hash_delta = received_hashes - self.local_pepes

        delta = self.local_pepes - received_hashes
        logging.debug("Sending hashes: {}".format(delta))

        for i in delta:
            self.pubsub.topic_pub(pepenet_channels["update"], i)
        if delta:
            self.pubsub.topic_pub(pepenet_channels["update_control"],
                                  control_codes["updating_finished"])

        self.local_pepes.update(new_hash_delta)

    def _on_upload(self, data):
        """Callback called when someone on the network uploads a pepe
        self.local_pepes is set, so if the pepe is a duplicate i will not
        be added twice"""

        self._register_pepe(b64decode(data["content"]).decode('utf8'))
        logging.info("Received pepe: {}".format(b64decode(data["content"])
                                                .decode('utf8')))

    def get_peer_number(self):
        "Returns the number of connected peers to the network"
        return self.pubsub.topic_peer_number(pepenet_channels["update"])

    def get_pepe(self, pepe_hash):
        "Downloads pepe from ipfs and saves it in database folder"
{'fraction_non_alphanumeric': 0.09310728182323513, 'fraction_numerical': 0.003613118399110617, 'mean_word_length': 2.3379715522572666, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7596885', 'n_tokens_mistral': 3188, 'n_tokens_neox': 3090, 'n_words': 775}