<commit_before>#!/usr/bin/env python
import os
import shutil
from glob import glob
from tempfile import mkdtemp
from collections import OrderedDict

import numpy as np
import pandas as pd
from gsw import z_from_p, p_from_z

from gutils import (
    generate_stream,
    get_decimal_degrees,
    interpolate_gps,
    masked_epoch,
    safe_makedirs
)
from gutils.ctd import calculate_practical_salinity, calculate_density

import logging
L = logging.getLogger(__name__)


MODE_MAPPING = {
    "rt": ["sbd", "tbd", "mbd", "nbd"],
    "delayed": ["dbd", "ebd"]
}
ALL_EXTENSIONS = [".sbd", ".tbd", ".mbd", ".nbd", ".dbd", ".ebd"]


class SlocumReader(object):

    TIMESTAMP_SENSORS = ['m_present_time', 'sci_m_present_time']
    PRESSURE_SENSORS = ['sci_water_pressure', 'm_water_pressure', 'm_pressure']
    DEPTH_SENSORS = ['m_depth', 'm_water_depth']
    TEMPERATURE_SENSORS = ['sci_water_temp']
    CONDUCTIVITY_SENSORS = ['sci_water_cond']

    def __init__(self, ascii_file):
        self.ascii_file = ascii_file
        self.metadata, self.data = self.read()

        # Set the mode to 'rt' or 'delayed'
        self.mode = None
        if 'filename_extension' in self.metadata:
            filemode = self.metadata['filename_extension']
            for m, extensions in MODE_MAPPING.items():
                if filemode in extensions:
                    self.mode = m
                    break

    def read(self):
        metadata = OrderedDict()
        headers = None
        with open(self.ascii_file, 'rt') as af:
            for li, al in enumerate(af):
                if 'm_present_time' in al:
                    headers = al.strip().split(' ')
                elif headers is not None:
                    data_start = li + 2  # Skip units line and the interger row after that
                    break
                else:
                    title, value = al.split(':', 1)
                    metadata[title.strip()] = value.strip()

        # Pull out the number of bytes for each column
        #   The last numerical field is the number of bytes transmitted for each sensor:
        #     1    A 1 byte integer value [-128 .. 127].
        #     2    A 2 byte integer value [-32768 .. 32767].
        #     4    A 4 byte float value (floating point, 6-7 significant digits,
        #                                approximately 10^-38 to 10^38 dynamic range).
        #     8    An 8 byte double value (floating point, 15-16 significant digits,
        #                                  approximately 10^-308 to 10^308 dyn. range).
        dtypedf = pd.read_csv(
            self.ascii_file,
            index_col=False,
            skiprows=data_start - 1,
            nrows=1,
            header=None,
            names=headers,
            sep=' ',
            skip_blank_lines=True,
        )

        def intflag_to_dtype(intvalue):
            if intvalue == 1:
                return np.object  # ints can't have NaN so use object for now
            elif intvalue == 2:
                return np.object  # ints can't have NaN so use object for now
            elif intvalue == 4:
                return np.float32
            elif intvalue == 8:
                return np.float64
            else:
                return np.object

        inttypes = [ intflag_to_dtype(x) for x in dtypedf.iloc[0].astype(int).values ]
        dtypes = dict(zip(dtypedf.columns, inttypes))

        df = pd.read_csv(
            self.ascii_file,
            index_col=False,
            skiprows=data_start,
            header=None,
            names=headers,
            dtype=dtypes,
            sep=' ',
            skip_blank_lines=True,
        )
        return metadata, df

    def standardize(self, gps_prefix=None):

        df = self.data.copy()

        # Convert NMEA coordinates to decimal degrees
        for col in df.columns:
            # Ignore if the m_gps_lat and/or m_gps_lon value is the default masterdata value
            if '_lat' in col:
                df[col] = df[col].map(lambda x: get_decimal_degrees(x) if x <= 9000 else np.nan)
            elif '_lon' in col:
                df[col] = df[col].map(lambda x: get_decimal_degrees(x) if x < 18000 else np.nan)

        # Standardize 'time' to the 't' column
        for t in self.TIMESTAMP_SENSORS:
            if t in df.columns:
                df['t'] = pd.to_datetime(df[t], unit='s')
                break

        # Interpolate GPS coordinates
        if 'm_gps_lat' in df.columns and 'm_gps_lon' in df.columns:

            df['drv_m_gps_lat'] = df.m_gps_lat.copy()
            df['drv_m_gps_lon'] = df.m_gps_lon.copy()

            # Fill in data will nulls where value is the default masterdata value
            masterdatas = (df.drv_m_gps_lon >= 18000) | (df.drv_m_gps_lat > 9000)
            df.loc[masterdatas, 'drv_m_gps_lat'] = np.nan
            df.loc[masterdatas, 'drv_m_gps_lon'] = np.nan
<commit_msg>Better matching for Slocum lat/lon columns<commit_after>#!/usr/bin/env python
import os
import shutil
from glob import glob
from tempfile import mkdtemp
from collections import OrderedDict

import numpy as np
import pandas as pd
from gsw import z_from_p, p_from_z

from gutils import (
    generate_stream,
    get_decimal_degrees,
    interpolate_gps,
    masked_epoch,
    safe_makedirs
)
from gutils.ctd import calculate_practical_salinity, calculate_density

import logging
L = logging.getLogger(__name__)


MODE_MAPPING = {
    "rt": ["sbd", "tbd", "mbd", "nbd"],
    "delayed": ["dbd", "ebd"]
}
ALL_EXTENSIONS = [".sbd", ".tbd", ".mbd", ".nbd", ".dbd", ".ebd"]


class SlocumReader(object):

    TIMESTAMP_SENSORS = ['m_present_time', 'sci_m_present_time']
    PRESSURE_SENSORS = ['sci_water_pressure', 'm_water_pressure', 'm_pressure']
    DEPTH_SENSORS = ['m_depth', 'm_water_depth']
    TEMPERATURE_SENSORS = ['sci_water_temp']
    CONDUCTIVITY_SENSORS = ['sci_water_cond']

    def __init__(self, ascii_file):
        self.ascii_file = ascii_file
        self.metadata, self.data = self.read()

        # Set the mode to 'rt' or 'delayed'
        self.mode = None
        if 'filename_extension' in self.metadata:
            filemode = self.metadata['filename_extension']
            for m, extensions in MODE_MAPPING.items():
                if filemode in extensions:
                    self.mode = m
                    break

    def read(self):
        metadata = OrderedDict()
        headers = None
        with open(self.ascii_file, 'rt') as af:
            for li, al in enumerate(af):
                if 'm_present_time' in al:
                    headers = al.strip().split(' ')
                elif headers is not None:
                    data_start = li + 2  # Skip units line and the interger row after that
                    break
                else:
                    title, value = al.split(':', 1)
                    metadata[title.strip()] = value.strip()

        # Pull out the number of bytes for each column
        #   The last numerical field is the number of bytes transmitted for each sensor:
        #     1    A 1 byte integer value [-128 .. 127].
        #     2    A 2 byte integer value [-32768 .. 32767].
        #     4    A 4 byte float value (floating point, 6-7 significant digits,
        #                                approximately 10^-38 to 10^38 dynamic range).
        #     8    An 8 byte double value (floating point, 15-16 significant digits,
        #                                  approximately 10^-308 to 10^308 dyn. range).
        dtypedf = pd.read_csv(
            self.ascii_file,
            index_col=False,
            skiprows=data_start - 1,
            nrows=1,
            header=None,
            names=headers,
            sep=' ',
            skip_blank_lines=True,
        )

        def intflag_to_dtype(intvalue):
            if intvalue == 1:
                return np.object  # ints can't have NaN so use object for now
            elif intvalue == 2:
                return np.object  # ints can't have NaN so use object for now
            elif intvalue == 4:
                return np.float32
            elif intvalue == 8:
                return np.float64
            else:
                return np.object

        inttypes = [ intflag_to_dtype(x) for x in dtypedf.iloc[0].astype(int).values ]
        dtypes = dict(zip(dtypedf.columns, inttypes))

        df = pd.read_csv(
            self.ascii_file,
            index_col=False,
            skiprows=data_start,
            header=None,
            names=headers,
            dtype=dtypes,
            sep=' ',
            skip_blank_lines=True,
        )
        return metadata, df

    def standardize(self, gps_prefix=None):

        df = self.data.copy()

        # Convert NMEA coordinates to decimal degrees
        for col in df.columns:
            # Ignore if the m_gps_lat and/or m_gps_lon value is the default masterdata value
            if col.endswith('_lat'):
                df[col] = df[col].map(lambda x: get_decimal_degrees(x) if x <= 9000 else np.nan)
            elif col.endswith('_lon'):
                df[col] = df[col].map(lambda x: get_decimal_degrees(x) if x < 18000 else np.nan)

        # Standardize 'time' to the 't' column
        for t in self.TIMESTAMP_SENSORS:
            if t in df.columns:
                df['t'] = pd.to_datetime(df[t], unit='s')
                break

        # Interpolate GPS coordinates
        if 'm_gps_lat' in df.columns and 'm_gps_lon' in df.columns:

            df['drv_m_gps_lat'] = df.m_gps_lat.copy()
            df['drv_m_gps_lon'] = df.m_gps_lon.copy()

            # Fill in data will nulls where value is the default masterdata value
            masterdatas = (df.drv_m_gps_lon >= 18000) | (df.drv_m_gps_lat > 9000)
            df.loc[masterdatas, 'drv_m_gps_lat'] = np.nan
            df.loc[masterdatas, 'drv_m_gps_lon'] = np.nan

{'fraction_non_alphanumeric': 0.09444275736410568, 'fraction_numerical': 0.01599352161149914, 'mean_word_length': 1.8140131016804328, 'pattern_counts': {'":': 4, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2455153', 'n_tokens_mistral': 3225, 'n_tokens_neox': 2971, 'n_words': 839}