<commit_before>import idiokit
from abusehelper.core import bot, utils


COLUMNS = ("first seen", "threat", "malware", "host", "url", "status", "registrar", "ip", "asn", "country")


def _value_split(values):
    results = set()

    for value in values:
        results = results | set([x for x in value.split("|") if x])

    return tuple(results)


@idiokit.stream
def _parse():
    while True:
        event = yield idiokit.next()

        for key in event.keys():
            event.pop(key, filter=lambda value: not value.strip())

        for key in ("ip", "asn", "country"):
            event.update(key, _value_split(event.pop(key)))

        yield idiokit.send(event)


class RansomwareTrackerBot(bot.PollingBot):
    feed_url = bot.Param(default="https://ransomwaretracker.abuse.ch/feeds/csv/")

    @idiokit.stream
    def poll(self):
        self.log.info("Downloading %s" % self.feed_url)
        try:
            info, fileobj = yield utils.fetch_url(self.feed_url)
        except utils.FetchUrlFailed, fuf:
            self.log.error("Download failed: %r", fuf)
            idiokit.stop()

        lines = []
        for line in fileobj:
            line = line.strip()

            if line and not line.startswith("#"):
                lines.append(line)

        yield idiokit.pipe(
            utils.csv_to_events(tuple(lines),
                                columns=COLUMNS,
                                charset=info.get_param("charset", None)),
            _parse()
        )

if __name__ == "__main__":
    RansomwareTrackerBot.from_command_line().execute()
<commit_msg>abusehelper.bots.abusecd.ransomwarebot: Better handling for failed fetches. Style fixes.<commit_after>import idiokit
from abusehelper.core import bot, utils


COLUMNS = ("first seen", "threat", "malware", "host", "url", "status", "registrar", "ip", "asn", "cc")


def _value_split(values):
    results = set()
    for value in values:
        results = results | set([x for x in value.split("|") if x])
    return tuple(results)


@idiokit.stream
def _parse():
    while True:
        event = yield idiokit.next()

        for key in event.keys():
            event.pop(key, filter=lambda value: not value.strip())

        for key in ("ip", "asn", "country"):
            event.update(key, _value_split(event.pop(key)))

        yield idiokit.send(event)


class RansomwareTrackerBot(bot.PollingBot):
    feed_url = bot.Param(default="https://ransomwaretracker.abuse.ch/feeds/csv/")

    @idiokit.stream
    def poll(self):
        self.log.info("Downloading %s" % self.feed_url)
        try:
            info, fileobj = yield utils.fetch_url(self.feed_url)
        except utils.FetchUrlFailed as fuf:
            raise bot.PollSkipped("Download failed: {1}".format(fuf))

        lines = []
        for line in fileobj:
            line = line.strip()

            if line and not line.startswith("#"):
                lines.append(line)

        yield idiokit.pipe(
            utils.csv_to_events(tuple(lines),
                                columns=COLUMNS,
                                charset=info.get_param("charset", None)),
            _parse()
        )

if __name__ == "__main__":
    RansomwareTrackerBot.from_command_line().execute()

{'fraction_non_alphanumeric': 0.13000616142945162, 'fraction_numerical': 0.00030807147258163895, 'mean_word_length': 2.27979797979798, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '356248', 'n_tokens_mistral': 1032, 'n_tokens_neox': 999, 'n_words': 231}