<commit_before>import urllib
import math
import os

import pyfits

__version__ = "$Id: mastio.py 1780 2014-08-27 16:36:11Z fmullall $"
__URL__ = "$URL: svn+ssh://flux/home/fmullall/svn/kepler/py/mastio.py $"


def createKeplerArchiveForLios():
    """Create an archive on my work machine, lios, caching data in the correct place"""
    localPath="/disk2/data"
    return KeplerArchive(localPath)

def createKeplerArchiveForUllord():
    localPath="/Users/fergal/data/kepler"
    return KeplerArchive(localPath)


def createKeplerArchiveForDunalmu():
    localPath="/home/fergal/data/fits/kepler"
    return KeplerArchive(localPath)


class MastArchive():
    """Base class for queries of the MAST Kepler and K2 archives.
    Handles submitting the URL, receiving and caching the results

    The child class determines the url to submit
    """
    def __init__(self, localDir, remoteServer, remoteFluxPath, remoteTpfPath):
        self.localDir = localDir
        self.remoteServer = remoteServer
        self.remoteFluxPath = remoteFluxPath
        self.remoteTpfPath = remoteTpfPath

        if not os.path.exists(localDir):
            raise ValueError("Local path %s doesn't exist" %(localDir))


    def getData(self, localUrl, remoteUrl, compressedOnServer, *args, **kwargs):

        forceDownload = kwargs.pop("force", False)<commit_msg>Removing some functions that only have local applicability from this inherited module. The module was inherited from a different project and contains some functions to enable easy use of specific machines. These functions don't belong in DAVE.<commit_after>import urllib
import math
import os

import pyfits

__version__ = "$Id: mastio.py 1780 2014-08-27 16:36:11Z fmullall $"
__URL__ = "$URL: svn+ssh://flux/home/fmullall/svn/kepler/py/mastio.py $"


class MastArchive():
    """Base class for queries of the MAST Kepler and K2 archives.
    Handles submitting the URL, receiving and caching the results

    The child class determines the url to submit
    """
    def __init__(self, localDir, remoteServer, remoteFluxPath, remoteTpfPath):
        self.localDir = localDir
        self.remoteServer = remoteServer
        self.remoteFluxPath = remoteFluxPath
        self.remoteTpfPath = remoteTpfPath

        if not os.path.exists(localDir):
            raise ValueError("Local path %s doesn't exist" %(localDir))


    def getData(self, localUrl, remoteUrl, compressedOnServer, *args, **kwargs):

        forceDownload = kwargs.pop("force", False)
{'fraction_non_alphanumeric': 0.0900720576461169, 'fraction_numerical': 0.015612489991993595, 'mean_word_length': 3.796545105566219, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2079650', 'n_tokens_mistral': 794, 'n_tokens_neox': 723, 'n_words': 228}