<commit_before>"""Module containing class `NfcImporter`."""


from __future__ import print_function

from collections import defaultdict, namedtuple
import datetime
import itertools
import logging
import os
import re

import pytz

from vesper.archive.recording import Recording
import vesper.archive.recording_utils as recording_utils
import vesper.util.sound_utils as sound_utils
import vesper.util.text_utils as text_utils
import vesper.util.time_utils as time_utils
import vesper.vcl.vcl_utils as vcl_utils


# TODO: Implement an indenting `logging` message formatter, so that
# logging code does not have to format messages with an `_indent`
# method.


# We assume that the time zone is US/Mountain for all times that occur
# in input file names.
_TIME_ZONE = pytz.timezone('US/Mountain')


_identity = lambda s: s
_capitalize = lambda s: s.capitalize()
_parse_date = time_utils.parse_date
_parse_date6 = lambda mm, dd, yy: _parse_date(yy, mm, dd)
_parse_time = time_utils.parse_time
_parse_dur = time_utils.parse_time_delta


_STATION_NAMES = {
    
    'baldy': 'Baldy NFC',
    
    'flam': 'Flam Point 569 SMX-II',
    
    'flam-east': 'Flam Point 457 SMX-II',
    
    'flood': 'Floodplain NFC',
    'floodnfc': 'Floodplain NFC',
    'flood21c': 'Floodplain 21c',
    
    'ridge': 'Ridge NFC',
    'ridgenfc': 'Ridge NFC',
    'ridge21c': 'Ridge 21c',
    
    'sheep': 'Sheep Camp NFC',
    'sheepcamp': 'Sheep Camp NFC',
    'sheepnfc': 'Sheep Camp NFC',
    'sheepsmxii': 'Sheep Camp SMX-II',
    
    'florence': 'Florence 21c',
    
    'medwheel': 'Medicine Wheel NFC'
    
}


def _parse_station_name(s):
    try:
        return _STATION_NAMES[s.lower()]
    except KeyError:
        return s.capitalize()
    
    
_FieldFormat = namedtuple(
    '_FieldFormat', ('name', 'field_name', 're', 'num_subfields', 'parser'))
    
_FIELD_FORMATS = dict((t[0], _FieldFormat(*t)) for t in (
    ('class', 'clip_class_name', r'([a-zA-Z_]+)', 1, _identity),
    ('station', 'station_name', r'([a-zA-Z0-9\-]+)', 1,
     _parse_station_name),
    ('detector', 'detector_name', r'(Tseep|Thrush|Manual)', 1, _capitalize),
    ('date6', 'monitoring_start_date', r'(\d{2})(\d{2})(\d{2})', 3,
     _parse_date6),
    ('time6', 'monitoring_start_time', r'(\d{2})(\d{2})(\d{2})', 3,
     _parse_time),
    ('time4', 'monitoring_start_time', r'(\d{2})(\d{2})', 2, _parse_time),
    ('dur6', 'monitoring_duration', r'(\d{2})(\d{2})(\d{2})', 3, _parse_dur),
    ('dur4', 'monitoring_duration', r'(\d{2})(\d{2})', 2, _parse_dur),
    ('interior_comment', 'interior_comment', r'(.+)', 1, _identity),
    ('second_dur', 'second_dur', r'(\d{2})(\d{2})(\d{2})', 3, _parse_dur),
    ('elapsed7', 'clip_start_time', r'(\d{3})\.(\d{2})\.(\d{2})', 3,
     _parse_dur),
    ('elapsed6', 'clip_start_time', r'(\d{2})(\d{2})(\d{2})', 3, _parse_dur),
    ('num2', 'clip_num', r'(\d{2})', 1, int),
    ('comment', 'comment', r'(.+)', 1, _identity)
))


_FIELD_NAMES = (
    'station_name', 'detector_name', 'monitoring_start_date',
    'monitoring_start_time', 'monitoring_duration', 'interior_comment',
    'second_dur', 'clip_start_time', 'clip_num', 'clip_class_name',<commit_msg>Fixed bug in MPG Ranch NFC importer.<commit_after>"""Module containing class `NfcImporter`."""


from __future__ import print_function

from collections import defaultdict, namedtuple
import datetime
import itertools
import logging
import os
import re

import pytz

from vesper.archive.recording import Recording
import vesper.archive.recording_utils as recording_utils
import vesper.util.sound_utils as sound_utils
import vesper.util.text_utils as text_utils
import vesper.util.time_utils as time_utils
import vesper.vcl.vcl_utils as vcl_utils


# TODO: Implement an indenting `logging` message formatter, so that
# logging code does not have to format messages with an `_indent`
# method.


# We assume that the time zone is US/Mountain for all times that occur
# in input file names.
_TIME_ZONE = pytz.timezone('US/Mountain')


_identity = lambda s: s
_capitalize = lambda s: s.capitalize()
_parse_date = time_utils.parse_date
_parse_date6 = lambda mm, dd, yy: _parse_date(yy, mm, dd)
_parse_time = time_utils.parse_time
_parse_dur = time_utils.parse_time_delta


_STATION_NAMES = {
    
    'baldy': 'Baldy NFC',
    
    'flam': 'Flam Point 569 SMX-II',
    
    'flam-east': 'Flam Point 457 SMX-II',
    
    'flood': 'Floodplain NFC',
    'floodnfc': 'Floodplain NFC',
    'flood21c': 'Floodplain 21c',
    
    'ridge': 'Ridge NFC',
    'ridgenfc': 'Ridge NFC',
    'ridge21c': 'Ridge 21c',
    
    'sheep': 'Sheep Camp NFC',
    'sheepcamp': 'Sheep Camp NFC',
    'sheepnfc': 'Sheep Camp NFC',
    'sheepsmxii': 'Sheep Camp SMX-II',
    
    'florence': 'Florence 21c',
    
    'medwheel': 'Medicine Wheel NFC'
    
}


def _parse_station_name(s):
    try:
        return _STATION_NAMES[s.lower()]
    except KeyError:
        return s.capitalize()
    
    
_FieldFormat = namedtuple(
    '_FieldFormat', ('name', 'field_name', 're', 'num_subfields', 'parser'))
    
_FIELD_FORMATS = dict((t[0], _FieldFormat(*t)) for t in (
    ('class', 'clip_class_name', r'([a-zA-Z_]+)', 1, _identity),
    ('station', 'station_name', r'([a-zA-Z][a-zA-Z0-9\-]+)', 1,
     _parse_station_name),
    ('detector', 'detector_name', r'(Tseep|Thrush|Manual)', 1, _capitalize),
    ('date6', 'monitoring_start_date', r'(\d{2})(\d{2})(\d{2})', 3,
     _parse_date6),
    ('time6', 'monitoring_start_time', r'(\d{2})(\d{2})(\d{2})', 3,
     _parse_time),
    ('time4', 'monitoring_start_time', r'(\d{2})(\d{2})', 2, _parse_time),
    ('dur6', 'monitoring_duration', r'(\d{2})(\d{2})(\d{2})', 3, _parse_dur),
    ('dur4', 'monitoring_duration', r'(\d{2})(\d{2})', 2, _parse_dur),
    ('interior_comment', 'interior_comment', r'(.+)', 1, _identity),
    ('second_dur', 'second_dur', r'(\d{2})(\d{2})(\d{2})', 3, _parse_dur),
    ('elapsed7', 'clip_start_time', r'(\d{3})\.(\d{2})\.(\d{2})', 3,
     _parse_dur),
    ('elapsed6', 'clip_start_time', r'(\d{2})(\d{2})(\d{2})', 3, _parse_dur),
    ('num2', 'clip_num', r'(\d{2})', 1, int),
    ('comment', 'comment', r'(.+)', 1, _identity)
))


_FIELD_NAMES = (
    'station_name', 'detector_name', 'monitoring_start_date',
    'monitoring_start_time', 'monitoring_duration', 'interior_comment',
    'second_dur', 'clip_start_time', 'clip_num', 'clip_class_name',
{'fraction_non_alphanumeric': 0.18057520037718058, 'fraction_numerical': 0.020744931636020744, 'mean_word_length': 4.199346405228758, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6620242', 'n_tokens_mistral': 2582, 'n_tokens_neox': 2436, 'n_words': 545}