<commit_before>#!/usr/bin/env python

import os
import warnings

import argparse

import shutil
import subprocess
import pandas as pd

from panoptes.utils import images

gsutil = shutil.which('gsutil')


def list_remote_dir(prefix=None, verbose=False):

    if prefix is not None:
        rp = '{}/{}'.format(REMOTE_PATH, prefix)
    else:
        rp = REMOTE_PATH

    cmd = [gsutil, 'ls', rp]

    if verbose:
        print(cmd)

    output = None

    try:
        output = subprocess.run(cmd, stdout=subprocess.PIPE, check=True, universal_newlines=True)
    except Exception as e:
        warnings.warn("Can't run command: {}".format(e))

    return output.stdout.strip()


def get_remote_dir(remote_dir, verbose=False):

    full_dir = '/var/panoptes/images/fields/{}/'.format(remote_dir.rstrip('/').split('/')[-2])

    gsutil = shutil.which('gsutil')

    cmd = [gsutil, '-m', 'cp', '-r', remote_dir, full_dir]

    if verbose:
        print(cmd)

    try:
        os.mkdir(full_dir)
    except OSError as e:
        warnings.warn("Can't create dir: {}".format(e))

    try:
        subprocess.call(cmd)
    except Exception as e:
        warnings.warn("Can't run command: {}".format(e))

    return full_dir


def make_pec_data(image_dir, observer=None, verbose=False):

    name, obs_time = image_dir.split('/')

    data_table = images.get_pec_data(image_dir, observer=observer)

    if verbose:
        print(data_table.meta)

    hdf5_fn = '/var/panoptes/images/pec.hdf5'

    hdf5_path = 'observing/{}/{}'.format(data_table.meta['name'], data_table.meta['obs_date_start'])

    data_table.write(hdf5_fn, path=hdf5_path, append=True, serialize_meta=True, overwrite=True)


def main(remote=None, project=None, unit=None, verbose=False, **kwargs):

    # See if the remote path exists in the HDF5 data store
    store = pd.HDFStore(kwargs.get('hdf5_file'))

    hdf_path = 'observing/{}'.format(remote)

    if hdf_path not in store.keys():
        remote_path = 'gs://{}/{}/{}'.format(project, unit, remote)

        # Get the data
        local_dir = get_remote_dir(remote_path)<commit_msg>Split dir correctly<commit_after>#!/usr/bin/env python

import os
import warnings

import argparse

import shutil
import subprocess
import pandas as pd

from panoptes.utils import images

gsutil = shutil.which('gsutil')


def list_remote_dir(prefix=None, verbose=False):

    if prefix is not None:
        rp = '{}/{}'.format(REMOTE_PATH, prefix)
    else:
        rp = REMOTE_PATH

    cmd = [gsutil, 'ls', rp]

    if verbose:
        print(cmd)

    output = None

    try:
        output = subprocess.run(cmd, stdout=subprocess.PIPE, check=True, universal_newlines=True)
    except Exception as e:
        warnings.warn("Can't run command: {}".format(e))

    return output.stdout.strip()


def get_remote_dir(remote_dir, verbose=False):

    full_dir = '/var/panoptes/images/fields/{}/'.format(remote_dir.rstrip('/').split('/')[-2])

    gsutil = shutil.which('gsutil')

    cmd = [gsutil, '-m', 'cp', '-r', remote_dir, full_dir]

    if verbose:
        print(cmd)

    try:
        os.mkdir(full_dir)
    except OSError as e:
        warnings.warn("Can't create dir: {}".format(e))

    try:
        subprocess.call(cmd)
    except Exception as e:
        warnings.warn("Can't run command: {}".format(e))

    return full_dir


def make_pec_data(image_dir, observer=None, verbose=False):

    name, obs_time = image_dir.rsplit('/').split('/')

    data_table = images.get_pec_data(image_dir, observer=observer)

    if verbose:
        print(data_table.meta)

    hdf5_fn = '/var/panoptes/images/pec.hdf5'

    hdf5_path = 'observing/{}/{}'.format(data_table.meta['name'], data_table.meta['obs_date_start'])

    data_table.write(hdf5_fn, path=hdf5_path, append=True, serialize_meta=True, overwrite=True)


def main(remote=None, project=None, unit=None, verbose=False, **kwargs):

    # See if the remote path exists in the HDF5 data store
    store = pd.HDFStore(kwargs.get('hdf5_file'))

    hdf_path = 'observing/{}'.format(remote)

    if hdf_path not in store.keys():
        remote_path = 'gs://{}/{}/{}'.format(project, unit, remote)

        # Get the data
        local_dir = get_remote_dir(remote_path)
{'fraction_non_alphanumeric': 0.14116532449076266, 'fraction_numerical': 0.0037896731406916154, 'mean_word_length': 3.2786220871327254, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6582312', 'n_tokens_mistral': 1519, 'n_tokens_neox': 1423, 'n_words': 317}