<commit_before>from __future__ import division, print_function

import datetime as dt
import logging
import os
import shutil
import sys

import matplotlib.image as mplimg
import pandas as pd
from pathlib import Path

from . import helper_functions as hf
from .exceptions import NoFilesFoundError

try:
    from urllib import urlretrieve
except ImportError:
    from urllib.request import urlretrieve

HOME = Path(os.environ["HOME"])
if os.environ['USER'] == 'gapo7695':
    data_root = Path('/Users/gapo7695/Dropbox/myPy/others/P4_sandbox/databaseP4')
else:
    data_root = HOME / 'Dropbox' / 'data' / 'planet4'

done_path = data_root / 'done.h5'

location_target_codes = {'giza': [850],
                         'spider_evolution': [950],
                         'ithaca': [945, 850, 950]}


def dropbox():
    home = Path.home()
    return home / 'Dropbox' / 'data' / 'planet4'


def get_image_id_data(image_id, feedback=False):
    return pd.read_hdf(get_current_database_fname(), 'df',
                       where='image_id=' + image_id)


def get_image_name_data(image_name, feedback=False):
    if feedback:
        print("Getting current data for image_name {}".format(image_name))
    return pd.read_hdf(get_current_database_fname(), 'df',
                       where='image_name=' + image_name)


def get_list_of_image_names_data(image_names):
    dfs = []
    for image_name in image_names:
        dfs.append(get_image_name_data(image_name))
    return pd.concat(dfs)


class ResultManager:

    def __init__(self, inputpath):
        self.inpath = Path(inputpath)
        self.obsid = '_'.join(self.inpath.name.split('_')[:3])
        self.inkind = self.inpath.stem.split('_')[-1]

    @property
    def fanfile(self):
        return self.inpath.with_name(self.obsid + '_fans' + self.inpath.suffix)

    @property
    def fandf(self):
        return pd.read_hdf(str(self.fanfile))

    @property
    def blotchfile(self):
        return self.inpath.with_name(self.obsid + '_blotches' + self.inpath.suffix)

    @property
    def blotchdf(self):
        return pd.read_hdf(str(self.blotchfile))

    @property
    def fnotchfile(self):
        return self.inpath.with_name(self.obsid + '_fnotches' + self.inpath.suffix)

    @property
    def fnotchdf(self):
        return pd.read_hdf(str(self.fnotchfile))



def is_catalog_production_good():
    from pandas.core.index import InvalidIndexError
    db = DBManager(get_current_database_fname())
    not_there = []
    invalid_index = []
    value_error = []
    for image_name in db.image_names:
        try:
            ResultManager(image_name)
        except InvalidIndexError:
            invalid_index.append(image_name)
        except ValueError:
            value_error.append(image_name)
        except:
            not_there.append(image_name)
    if len(value_error) == 0 and len(not_there) == 0 and\
            len(invalid_index) == 0:
        return True
    else:
        return False


def get_subframe(url):
    """Download image if not there yet and return numpy array.

    Takes a data record (called 'line'), picks out the image_url.
    First checks if the name of that image is already stored in
    the image path. If not, it grabs it from the server.
    Then uses matplotlib.image to read the image into a numpy-array
    and finally returns it.
    """
    targetpath = data_root / 'images' / os.path.basename(url)
    targetpath.parent.mkdir(exist_ok=True)
    if not targetpath.exists():
        logging.info("Did not find image in cache. Downloading ...")
        sys.stdout.flush()
        path = urlretrieve(url)[0]
        logging.debug("Done.")
        shutil.move(path, str(targetpath))
    else:
        logging.debug("Found image in cache.")
    im = mplimg.imread(targetpath)
    return im


class P4DBName(object):

    def __init__(self, fname):
        self.p = Path(fname)
        self.name = self.p.name
        self.parent = self.p.parent
        date = str(self.name)[:10]
        self.date = dt.datetime(*[int(i) for i in date.split('-')])


<commit_msg>Add capability to deal with image_id runs.<commit_after>from __future__ import division, print_function

import datetime as dt
import logging
import os
import shutil
import sys

import matplotlib.image as mplimg
import pandas as pd
from pathlib import Path

from . import helper_functions as hf
from .exceptions import NoFilesFoundError

try:
    from urllib import urlretrieve
except ImportError:
    from urllib.request import urlretrieve

HOME = Path(os.environ["HOME"])
if os.environ['USER'] == 'gapo7695':
    data_root = Path('/Users/gapo7695/Dropbox/myPy/others/P4_sandbox/databaseP4')
else:
    data_root = HOME / 'Dropbox' / 'data' / 'planet4'

done_path = data_root / 'done.h5'

location_target_codes = {'giza': [850],
                         'spider_evolution': [950],
                         'ithaca': [945, 850, 950]}


def dropbox():
    home = Path.home()
    return home / 'Dropbox' / 'data' / 'planet4'


def get_image_id_data(image_id, feedback=False):
    return pd.read_hdf(get_current_database_fname(), 'df',
                       where='image_id=' + image_id)


def get_image_name_data(image_name, feedback=False):
    if feedback:
        print("Getting current data for image_name {}".format(image_name))
    return pd.read_hdf(get_current_database_fname(), 'df',
                       where='image_name=' + image_name)


def get_list_of_image_names_data(image_names):
    dfs = []
    for image_name in image_names:
        dfs.append(get_image_name_data(image_name))
    return pd.concat(dfs)


class ResultManager:

    def __init__(self, inputpath):
        self.inpath = Path(inputpath)
        if self.inpath.stem.startswith('APF'):
            self.obsid = self.inpath.stem.split('_')[0]
        else:
            self.obsid = '_'.join(self.inpath.name.split('_')[:3])

    @property
    def fanfile(self):
        return self.inpath.with_name(self.obsid + '_fans' + self.inpath.suffix)

    @property
    def fandf(self):
        return pd.read_hdf(str(self.fanfile))

    @property
    def blotchfile(self):
        return self.inpath.with_name(self.obsid + '_blotches' + self.inpath.suffix)

    @property
    def blotchdf(self):
        return pd.read_hdf(str(self.blotchfile))

    @property
    def fnotchfile(self):
        return self.inpath.with_name(self.obsid + '_fnotches' + self.inpath.suffix)

    @property
    def fnotchdf(self):
        return pd.read_hdf(str(self.fnotchfile))


def is_catalog_production_good():
    from pandas.core.index import InvalidIndexError
    db = DBManager(get_current_database_fname())
    not_there = []
    invalid_index = []
    value_error = []
    for image_name in db.image_names:
        try:
            ResultManager(image_name)
        except InvalidIndexError:
            invalid_index.append(image_name)
        except ValueError:
            value_error.append(image_name)
        except:
            not_there.append(image_name)
    if len(value_error) == 0 and len(not_there) == 0 and\
            len(invalid_index) == 0:
        return True
    else:
        return False


def get_subframe(url):
    """Download image if not there yet and return numpy array.

    Takes a data record (called 'line'), picks out the image_url.
    First checks if the name of that image is already stored in
    the image path. If not, it grabs it from the server.
    Then uses matplotlib.image to read the image into a numpy-array
    and finally returns it.
    """
    targetpath = data_root / 'images' / os.path.basename(url)
    targetpath.parent.mkdir(exist_ok=True)
    if not targetpath.exists():
        logging.info("Did not find image in cache. Downloading ...")
        sys.stdout.flush()
        path = urlretrieve(url)[0]
        logging.debug("Done.")
        shutil.move(path, str(targetpath))
    else:
        logging.debug("Found image in cache.")
    im = mplimg.imread(targetpath)
    return im


class P4DBName(object):

    def __init__(self, fname):
        self.p = Path(fname)
        self.name = self.p.name
        self.parent = self.p.parent
        date = str(self.name)[:10]
        self.date = dt.datetime(*[int(i) for i in date.split('-')])


{'fraction_non_alphanumeric': 0.10909754317684262, 'fraction_numerical': 0.009000243249817562, 'mean_word_length': 2.7841693511274737, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '58365', 'n_tokens_mistral': 2739, 'n_tokens_neox': 2628, 'n_words': 631}