<commit_before># -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the HERA Collaboration
# Licensed under the BSD License.

"Files."

from __future__ import absolute_import, division, print_function, unicode_literals

__all__ = str('''
File
FileInstance
FileEvent
''').split()

import datetime
import json
import os.path
import re
from flask import flash, redirect, render_template, url_for

from . import app, db, logger
from .dbutil import NotNull
from .webutil import ServerError, json_api, login_required, optional_arg, required_arg
from .observation import Observation
from .store import Store


def infer_file_obsid(parent_dirs, name, info):
    """Infer the obsid associated with a file based on the limited information we
    have about it. Raises an exception if this cannot be done *with
    certainty*.

    The "hera" mode does this by looking for existing files whose names start
    with the same "zen.JD" prefix.

    The "none" mode refuses to do this.

    """
    mode = app.config.get('obsid_inference_mode', 'none')

    if mode == 'none':
        raise ServerError('refusing to try to infer the obsid of candidate new file \"%s\"', name)

    if mode == 'hera':
        bits = name.split('.')
        if len(bits) < 4:
            raise ServerError(
                'need to infer obsid of HERA file \"%s\", but its name looks weird', name)

        prefix = '.'.join(bits[:3])
        obsids = list(db.session.query(File.obsid)
                      .filter_by(File.name.like(prefix + '.%'))
                      .group_by(File.obsid))

        if len(obsids) != 1:
            raise ServerError('need to infer obsid of HERA file \"%s\", but got %d candidate '
                              'obsids from similarly-named files', name, len(obsids))

        return obsids[0]

    raise ServerError('configuration problem: unknown "obsid_inference_mode" setting %r', mode)


class File (db.Model):
    """A File describes a data product generated by HERA.

    The information described in a File structure never changes, and is
    universal between Librarians. Actual "instances" of files come and go, but
    a File record should never be deleted. The only exception to this is the
    "source" column, which is Librarian-dependent.

    A File may represent an actual flat file or a directory tree. The latter
    use case is important for MIRIAD data files, which are directories, and
    which we want to store in their native form for rapid analysis.
<commit_msg>server/librarian_server/file.py: bugfix the untested code<commit_after># -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the HERA Collaboration
# Licensed under the BSD License.

"Files."

from __future__ import absolute_import, division, print_function, unicode_literals

__all__ = str('''
File
FileInstance
FileEvent
''').split()

import datetime
import json
import os.path
import re
from flask import flash, redirect, render_template, url_for

from . import app, db, logger
from .dbutil import NotNull
from .webutil import ServerError, json_api, login_required, optional_arg, required_arg
from .observation import Observation
from .store import Store


def infer_file_obsid(parent_dirs, name, info):
    """Infer the obsid associated with a file based on the limited information we
    have about it. Raises an exception if this cannot be done *with
    certainty*.

    The "hera" mode does this by looking for existing files whose names start
    with the same "zen.JD" prefix.

    The "none" mode refuses to do this.

    """
    mode = app.config.get('obsid_inference_mode', 'none')

    if mode == 'none':
        raise ServerError('refusing to try to infer the obsid of candidate new file \"%s\"', name)

    if mode == 'hera':
        bits = name.split('.')
        if len(bits) < 4:
            raise ServerError(
                'need to infer obsid of HERA file \"%s\", but its name looks weird', name)

        prefix = '.'.join(bits[:3])
        obsids = list(db.session.query(File.obsid)
                      .filter(File.name.like(prefix + '.%'))
                      .group_by(File.obsid))

        if len(obsids) != 1:
            raise ServerError('need to infer obsid of HERA file \"%s\", but got %d candidate '
                              'obsids from similarly-named files', name, len(obsids))

        return obsids[0]

    raise ServerError('configuration problem: unknown "obsid_inference_mode" setting %r', mode)


class File (db.Model):
    """A File describes a data product generated by HERA.

    The information described in a File structure never changes, and is
    universal between Librarians. Actual "instances" of files come and go, but
    a File record should never be deleted. The only exception to this is the
    "source" column, which is Librarian-dependent.

    A File may represent an actual flat file or a directory tree. The latter
    use case is important for MIRIAD data files, which are directories, and
    which we want to store in their native form for rapid analysis.

{'fraction_non_alphanumeric': 0.08709484987074965, 'fraction_numerical': 0.003579240405647246, 'mean_word_length': 2.966876971608833, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3722464', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1377, 'n_words': 600}