<commit_before>import json
from housepy import util, log, config, strings
from ingest import ingest_json_file, ingest_data

def parse(request):
    log.info("databoat.parse")
    datas = ingest_json_file(request)
    try:
        for key, data in datas.items():
            t_local = strings.as_numeric(data['t_local'])
            data = data['data']        
            log.debug(json.dumps(data, indent=4, default=lambda x: str(x)))
            if 'gps_long' in data and 'gps_lat' in data:
                data['Longitude'] = data['gps_long']
                del data['gps_long']
                data['Latitude'] = data['gps_lat']
                del data['gps_lat']
            data['FeatureType'] = "sensor"
            data['FeatureSubType'] = "hybrid"
            data['SensorName'] = "databoat"
            data['t_utc'] = util.delocalize_timestamp(t_local, tz=config['local_tz'])
    except Exception as e:
        log.error("--> failed: %s" % log.exc(e))
        return None, "Unexpected format"   
    return data


"""
{ t_local: '1429805254',
 data: 
  { dissolved_oxygen: '9.79',
    orp: '202.2',
    salinity: '0.00',
    ph: '3.747',
    conductivity: '0',
    'water temp': '0' },
 t_utc: 1429805257 }

"""<commit_msg>changing databoat to bulk<commit_after>import json
from housepy import util, log, config, strings
from ingest import ingest_json_file, ingest_data

def parse(request):
    log.info("databoat.parse")
    data = ingest_json_file(request)
    try:
        for key, reading in data.items():
            t_local = strings.as_numeric(reading['t_local'])
            reading = reading['data']        
            log.debug(json.dumps(reading, indent=4, default=lambda x: str(x)))
            if 'gps_long' in reading and 'gps_lat' in reading:
                reading['Longitude'] = reading['gps_long']
                del reading['gps_long']
                reading['Latitude'] = reading['gps_lat']
                del reading['gps_lat']
            reading['FeatureType'] = "sensor"
            reading['FeatureSubType'] = "hybrid"
            reading['SensorName'] = "databoat"
            reading['t_utc'] = util.delocalize_timestamp(t_local, tz=config['local_tz'])
            success, value = ingest_data("sensor", reading)
            if not success:
                log.error("--> failed: %s" % value)            
    except Exception as e:
        log.error("--> failed: %s" % log.exc(e))
        return None, "Unexpected format"   
    return data


"""
{ t_local: '1429805254',
 data: 
  { dissolved_oxygen: '9.79',
    orp: '202.2',
    salinity: '0.00',
    ph: '3.747',
    conductivity: '0',
    'water temp': '0' },
 t_utc: 1429805257 }

"""
{'fraction_non_alphanumeric': 0.1418994413407821, 'fraction_numerical': 0.027560521415270017, 'mean_word_length': 2.349127182044888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6392951', 'n_tokens_mistral': 923, 'n_tokens_neox': 852, 'n_words': 196}