<commit_before>import yaml
from textwrap import dedent
from jsonschema import validate, ValidationError
import pyaml
import json
import os
import glob
import subprocess
from colorama import init, Fore, Back, Style
from trackhub import Track, default_hub, CompositeTrack, ViewTrack
import pkg_resources
from hubward import utils
from hubward.log import log

HERE = os.path.abspath(os.path.dirname(__file__))
SCHEMA = json.loads(utils.get_resource('schema.json'))


class Data(object):
    def __init__(self, obj, reldir):
        """
        Represents a single track destined for upload to UCSC as a track hub.

        In the metadata.yaml file, this is a single entry in the `data` list.

        Parameters
        ----------
        obj : dict
            One entry from the `data` list in a metadata file.

        reldir : str
            The dirname of the metadata file.  All paths (specifically,
            "original", "processed", and "script", are assumed to be relative
            to to `reldir`.
        """
        self.obj = obj
        self.reldir = reldir
        self.original = os.path.join(reldir, obj['original'])
        self.processed = os.path.join(reldir, obj['processed'])
        self.description = obj['description']
        self.label = obj['label']
        self.type_ = obj['type']
        self.genome = obj['genome']
        self.script = os.path.join(reldir, obj['script'])
        try:
            self.trackinfo = obj['trackinfo']
        except KeyError:
            self.trackinfo = {}

    def __str__(self):
        return pyaml.dumps(self.obj)

    def needs_update(self):
        if not os.path.exists(self.processed):
            return True
        if (

            # if processed is a link, then check the LINK time
            utils.link_is_newer(self.script, self.processed) or

            # but for the original data, we want to FOLLOW the link
            utils.is_newer(self.original, self.processed)
        ):
            return True

    def process(self):
        """
        Runs the processing script, which must accept 2 args, the absolute path
        of the original file and absolute path of the processed file.

        It's up to the processing script to handle everything else.
        """
        cmds = [
            self.script,
            self.original,
            self.processed]
        if not os.path.exists(self.original):
            raise ValueError(
                "Original data file %s does not exist" % self.original)
        if not os.path.exists(self.script):
            raise ValueError(
                "Script %s does not exist" % self.script)
        p = subprocess.Popen(
            cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        try:
            stdout, stderr = p.communicate()
        except OSError:
            print "Commands were", ' '.join(cmds)
            raise

        if p.returncode:
            msg = 'CMDS: %s' % (' '.join(cmds))
            msg += '\n'
            msg += 'STDERR: %s' % stderr
            raise ValueError(msg)

        # Check again to make sure we updated the file we thought we did...
        if self.needs_update():
            print(
                Fore.RED + 'The following command did not update '
                '{1}:\n\n{0}\n'.format(' \\\n'.join(cmds), self.processed) +<commit_msg>download and unpack raw data<commit_after>import yaml
from textwrap import dedent
from jsonschema import validate, ValidationError
import pyaml
import json
import os
import glob
import subprocess
from colorama import init, Fore, Back, Style
from trackhub import Track, default_hub, CompositeTrack, ViewTrack
import pkg_resources
from hubward import utils
from hubward.log import log
import conda.fetch

HERE = os.path.abspath(os.path.dirname(__file__))
SCHEMA = json.loads(utils.get_resource('schema.json'))


class Data(object):
    def __init__(self, obj, reldir):
        """
        Represents a single track destined for upload to UCSC as a track hub.

        In the metadata.yaml file, this is a single entry in the `data` list.

        Parameters
        ----------
        obj : dict
            One entry from the `data` list in a metadata file.

        reldir : str
            The dirname of the metadata file.  All paths (specifically,
            "original", "processed", and "script", are assumed to be relative
            to to `reldir`.
        """
        self.obj = obj
        self.reldir = reldir
        self.original = os.path.join(reldir, obj['original'])
        self.source = obj['source']
        self.processed = os.path.join(reldir, obj['processed'])
        self.description = obj['description']
        self.label = obj['label']
        self.type_ = obj['type']
        self.genome = obj['genome']
        self.script = os.path.join(reldir, obj['script'])
        try:
            self.trackinfo = obj['trackinfo']
        except KeyError:
            self.trackinfo = {}

    def __str__(self):
        return pyaml.dumps(self.obj)

    def needs_download(self):
        if not os.path.exists(self.original):
            return True

    def needs_update(self):
        if not os.path.exists(self.processed):
            return True
        if (

            # if processed is a link, then check the LINK time
            utils.link_is_newer(self.script, self.processed) or

            (os.path.exists(self.original) and
                # but for the original data, we want to FOLLOW the link
                utils.is_newer(self.original, self.processed))
        ):
            return True

    def process(self):
        """
        Runs the processing script, which must accept 2 args, the absolute path
        of the original file and absolute path of the processed file.

        It's up to the processing script to handle everything else.
        """
        cmds = [
            self.script,
            self.original,
            self.processed]
        if not os.path.exists(self.original):
            url = self.source['url']
            log(
                'Downloading "%s" -> "%s"' %
                (url, self.reldir), indent=4)
            fn = os.path.join(self.reldir, 'raw-data', self.source['fn'])
            utils.download(url, fn)
            utils.unpack(fn, os.path.join(self.reldir, 'raw-data'))

        if not os.path.exists(self.original):
            raise ValueError(
                "Original data file %s does not exist" % self.original)
        if not os.path.exists(self.script):
            raise ValueError(
                "Script %s does not exist" % self.script)
        p = subprocess.Popen(
            cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        try:
            stdout, stderr = p.communicate()
        except OSError:
            print "Commands were", ' '.join(cmds)
            raise

        if p.returncode:
            msg = 'CMDS: %s' % (' '.join(cmds))
            msg += '\n'
            msg += 'STDERR: %s' % stderr
            raise ValueError(msg)

        # Check again to make sure we updated the file we thought we did...
        if self.needs_update():
            print(
                Fore.RED + 'The following command did not update '
                '{1}:\n\n{0}\n'.format(' \\\n'.join(cmds), self.processed) +
{'fraction_non_alphanumeric': 0.10199587061252581, 'fraction_numerical': 0.0009635237439779766, 'mean_word_length': 1.9169008430349257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '461713', 'n_tokens_mistral': 2073, 'n_tokens_neox': 2004, 'n_words': 644}