<commit_before>import logging
import subprocess
import os
from pprint import pprint

from cif.constants import VERSION, SMRT_CACHE

import magic
import re
RE_SUPPORTED_DECODE = re.compile("zip|lzf|lzma|xz|lzop")


class Fetcher(object):

    def __init__(self, rule, feed, cache=SMRT_CACHE):

        self.logger = logging.getLogger(__name__)
        self.feed = feed
        self.rule = rule
        self.cache = cache
        self.fetcher = rule.fetcher

        if self.rule.remote:
            self.remote = self.rule.remote
        elif self.rule.defaults.get('remote'):
            self.remote = self.rule.defaults.get('remote')
        else:
            self.remote = self.rule.feeds[feed]['remote']

        self.dir = os.path.join(self.cache, self.rule.defaults.get('provider'))

        if not os.path.exists(self.dir):
            try:
                os.makedirs(self.dir)
            except OSError:
                self.logger.critical('failed to create {0}'.format(self.dir))
                raise

        self.cache = os.path.join(self.dir, self.feed)

        self.logger.debug(self.cache)

        # http://www-archive.mozilla.org/build/revised-user-agent-strings.html
        self.ua = "User-Agent: cif-smrt/{0} (csirtgadgets.org)".format(VERSION)

        if not self.fetcher:
            if self.remote.startswith('http'):
                self.fetcher = 'http'
            else:
                self.fetcher = 'file'

    def process(self, split="\n", limit=0):
        if self.fetcher == 'http':
            # using wget until we can find a better way to mirror files in python
            subprocess.check_call([
                'wget', '--header', self.ua,  '-q', self.remote, '-N', '-O', self.cache
            ])
        else:
            if self.fetcher == 'file':
                self.cache = self.remote
            else:
                raise NotImplementedError

        ftype = magic.from_file(self.cache, mime=True).decode('utf-8')
        self.logger.debug(ftype)

        if ftype.startswith('application/x-gzip'):
            import gzip
            with gzip.open(self.cache, 'rb') as f:
                for l in f:
                    yield l.strip()

        elif (ftype.startswith('text') or ftype.startswith('application/xml')):
            with open(self.cache) as f:
                for l in f:
                    yield l.strip()

        # if ftype == "application/zip":
        #     with ZipFile(self.cache) as f:
        #         for m in f.infolist():
        #             while True:
        #                 yield f.read(m.filename).split(split)[0:limit]<commit_msg>adding gzip magic type..<commit_after>import logging
import subprocess
import os
from pprint import pprint

from cif.constants import VERSION, SMRT_CACHE

import magic
import re
RE_SUPPORTED_DECODE = re.compile("zip|lzf|lzma|xz|lzop")


class Fetcher(object):

    def __init__(self, rule, feed, cache=SMRT_CACHE):

        self.logger = logging.getLogger(__name__)
        self.feed = feed
        self.rule = rule
        self.cache = cache
        self.fetcher = rule.fetcher

        if self.rule.remote:
            self.remote = self.rule.remote
        elif self.rule.defaults.get('remote'):
            self.remote = self.rule.defaults.get('remote')
        else:
            self.remote = self.rule.feeds[feed]['remote']

        self.dir = os.path.join(self.cache, self.rule.defaults.get('provider'))

        if not os.path.exists(self.dir):
            try:
                os.makedirs(self.dir)
            except OSError:
                self.logger.critical('failed to create {0}'.format(self.dir))
                raise

        self.cache = os.path.join(self.dir, self.feed)

        self.logger.debug(self.cache)

        # http://www-archive.mozilla.org/build/revised-user-agent-strings.html
        self.ua = "User-Agent: cif-smrt/{0} (csirtgadgets.org)".format(VERSION)

        if not self.fetcher:
            if self.remote.startswith('http'):
                self.fetcher = 'http'
            else:
                self.fetcher = 'file'

    def process(self, split="\n", limit=0):
        if self.fetcher == 'http':
            # using wget until we can find a better way to mirror files in python
            subprocess.check_call([
                'wget', '--header', self.ua,  '-q', self.remote, '-N', '-O', self.cache
            ])
        else:
            if self.fetcher == 'file':
                self.cache = self.remote
            else:
                raise NotImplementedError

        ftype = magic.from_file(self.cache, mime=True).decode('utf-8')
        self.logger.debug(ftype)

        if ftype.startswith('application/x-gzip') or ftype.startswith('application/gzip'):
            import gzip
            with gzip.open(self.cache, 'rb') as f:
                for l in f:
                    yield l.strip()

        elif (ftype.startswith('text') or ftype.startswith('application/xml')):
            with open(self.cache) as f:
                for l in f:
                    yield l.strip()

        # if ftype == "application/zip":
        #     with ZipFile(self.cache) as f:
        #         for m in f.infolist():
        #             while True:
        #                 yield f.read(m.filename).split(split)[0:limit]
{'fraction_non_alphanumeric': 0.12280370300396751, 'fraction_numerical': 0.001889287738522577, 'mean_word_length': 1.7487019730010385, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7348337', 'n_tokens_mistral': 1632, 'n_tokens_neox': 1569, 'n_words': 344}