<commit_before># -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the HERA Team.
# Licensed under the BSD License.

"""Utilities for interacting with the librarian.

These are various, mainly I/O-related, functions that could be shared between
a variety of scripts. A lot of what we do is related to the fact that some
Librarian "files" are MIRIAD data sets that are actually directories.

"""

__all__ = str('''
get_start_jd_from_path
get_type_from_path
get_pol_from_path
get_obsid_from_path
get_md5_from_path
get_size_from_path
''').split()

import hashlib, locale, os.path, re

import numpy as np
import aipy as a


def get_start_jd_from_path(path):
    """Get the JD from a path, assuming it follows HERA naming conventions.

    This is super fragile!!!! There should be a better way.

    """
    return float(re.findall(r'\d+\.\d+', path)[0])


def get_type_from_path(path):
    """Get the file type from a path, assuming it follows HERA naming conventions.

    This is super fragile!!!! There should be a better way.

    """
    return path.split ('.')[-1]


def get_pol_from_path(path):
    """Get the data polarization from a path, assuming it follows HERA naming
    conventions.

    This is super fragile!!!! There should be a better way. Also we hardcode
    the XY basis.

    """
    return re.findall (r'\.([xy][xy])\.', path)[0]


def get_obsid_from_path(path):
    """Get the obsid from a path.

    We assume that it is a MIRIAD UV dataset.

    """
    if os.path.isdir(path):
        try:
            uv = a.miriad.UV(path)
            try:
                return uv['obsid']
            except KeyError:
                pass
        except RuntimeError:
            pass # this happens if this file is not actually a UV dataset<commit_msg>hera_librarian/utils.py: make importable without requiring aipy<commit_after># -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the HERA Team.
# Licensed under the BSD License.

"""Utilities for interacting with the librarian.

These are various, mainly I/O-related, functions that could be shared between
a variety of scripts. A lot of what we do is related to the fact that some
Librarian "files" are MIRIAD data sets that are actually directories.

"""

__all__ = str('''
get_start_jd_from_path
get_type_from_path
get_pol_from_path
get_obsid_from_path
get_md5_from_path
get_size_from_path
''').split()

import hashlib, locale, os.path, re

import numpy as np


def get_start_jd_from_path(path):
    """Get the JD from a path, assuming it follows HERA naming conventions.

    This is super fragile!!!! There should be a better way.

    """
    return float(re.findall(r'\d+\.\d+', path)[0])


def get_type_from_path(path):
    """Get the file type from a path, assuming it follows HERA naming conventions.

    This is super fragile!!!! There should be a better way.

    """
    return path.split ('.')[-1]


def get_pol_from_path(path):
    """Get the data polarization from a path, assuming it follows HERA naming
    conventions.

    This is super fragile!!!! There should be a better way. Also we hardcode
    the XY basis.

    """
    return re.findall (r'\.([xy][xy])\.', path)[0]


def get_obsid_from_path(path):
    """Get the obsid from a path.

    We assume that it is a MIRIAD UV dataset.

    """
    if os.path.isdir(path):
        try:
            import aipy
            uv = aipy.miriad.UV(path)
            try:
                return uv['obsid']
            except KeyError:
                pass
        except RuntimeError:
            pass # this happens if this file is not actually a UV dataset
{'fraction_non_alphanumeric': 0.09593993325917687, 'fraction_numerical': 0.005005561735261402, 'mean_word_length': 2.884449244060475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3913683', 'n_tokens_mistral': 1160, 'n_tokens_neox': 1085, 'n_words': 428}