<commit_before>from os.path import dirname, exists, join
from urllib import urlretrieve
import inspect
import json
import os
import urlparse

import requests
import unicodecsv

from .state import StateBase

class BaseFetcher(StateBase):
    """
    Base class for interacting with source data.
    Primary use is fetching data files from source and standardizing names of files,
    which are then cached on S3 by their standardized name and used downstream to load
    results into data store.

    Intended to be subclassed in state-specific fetch.py modules.

    """

    def fetch(self, url, fname=None, overwrite=False):
        """Fetch and cache web page or data file

        ARGS

            url - link to download
            fname - file name for local storage in cache_dir
            overwrite - if True, overwrite cached copy with fresh donwload

        """
        local_file_name = self._standardized_filename(url, fname)
        if overwrite:
            name, response = urlretrieve(url, local_file_fname)
        else:
            if exists(local_file_name):
                print "File is cached: %s" % local_file_name
            else:
                name, response = urlretrieve(url, local_file_name)
                print "Added to cache: %s" % local_file_name

    def _standardized_filename(self, url, fname):
        """A standardized, fully qualified path name"""
        #TODO:apply filename standardization logic
        # non-result pages/files use default urllib name conventions
        # result files need standardization logic (TBD)
        if fname:
            filename = join(self.cache_dir, fname)
        else:
            filename = self._filename_from_url(url)
        return filename

    def _filename_from_url(self, url):
        #TODO: this is quick and dirty
        # see urlretrieve code for more robust conversion of
        # url to local filepath
        result = urlparse.urlsplit(url)
        bits = [
            self.cache_dir,
            result.netloc + '_' +
            result.path.strip('/'),
        ]
        name = join(*bits)
        return name
<commit_msg>replace cache_dir with cache.abspath on BaseFetcher<commit_after>from os.path import dirname, exists, join
from urllib import urlretrieve
import inspect
import json
import os
import urlparse

import requests
import unicodecsv

from .state import StateBase

class BaseFetcher(StateBase):
    """
    Base class for interacting with source data.
    Primary use is fetching data files from source and standardizing names of files,
    which are then cached on S3 by their standardized name and used downstream to load
    results into data store.

    Intended to be subclassed in state-specific fetch.py modules.

    """

    def fetch(self, url, fname=None, overwrite=False):
        """Fetch and cache web page or data file

        ARGS

            url - link to download
            fname - file name for local storage in cache directory
            overwrite - if True, overwrite cached copy with fresh donwload

        """
        local_file_name = self._standardized_filename(url, fname)
        if overwrite:
            name, response = urlretrieve(url, local_file_fname)
        else:
            if exists(local_file_name):
                print "File is cached: %s" % local_file_name
            else:
                name, response = urlretrieve(url, local_file_name)
                print "Added to cache: %s" % local_file_name

    def _standardized_filename(self, url, fname):
        """A standardized, fully qualified path name"""
        #TODO:apply filename standardization logic
        # non-result pages/files use default urllib name conventions
        # result files need standardization logic (TBD)
        if fname:
            filename = join(self.cache.abspath, fname)
        else:
            filename = self._filename_from_url(url)
        return filename

    def _filename_from_url(self, url):
        #TODO: this is quick and dirty
        # see urlretrieve code for more robust conversion of
        # url to local filepath
        result = urlparse.urlsplit(url)
        bits = [
            self.cache.abspath,
            result.netloc + '_' +
            result.path.strip('/'),
        ]
        name = join(*bits)
        return name

{'fraction_non_alphanumeric': 0.06389405204460967, 'fraction_numerical': 0.00046468401486988845, 'mean_word_length': 2.276255707762557, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5863615', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1113, 'n_words': 420}