<commit_before>"""
    IGOR 1D data reader
"""
#####################################################################
# This software was developed by the University of Tennessee as part of the
# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
# project funded by the US National Science Foundation.
# See the license text in license.txt
# copyright 2008, University of Tennessee
######################################################################

import logging
import numpy as np
from sas.sascalc.dataloader.file_reader_base_class import FileReader
from sas.sascalc.dataloader.data_info import DataInfo, plottable_1D, Data1D,\
    Detector
from sas.sascalc.dataloader.loader_exceptions import FileContentsException,\
    DefaultReaderException

logger = logging.getLogger(__name__)


class Reader(FileReader):
    """
    Class to load IGOR reduced .ABS files
    """
    # File type
    type_name = "IGOR 1D"
    # Wildcards
    type = ["IGOR 1D files (*.abs)|*.abs"]
    # List of allowed extensions
    ext = ['.abs']
    
    def get_file_contents(self):
        """ 
        Get the contents of the file
        
        :raise RuntimeError: when the file can't be opened
        :raise ValueError: when the length of the data vectors are inconsistent
        """
        buff = self.f_open.read()
        filepath = self.f_open.name
        lines = buff.splitlines()
        self.has_converter = True
        try:
            from sas.sascalc.data_util.nxsunit import Converter
        except:
            self.has_converter = False
        self.output = []
        self.current_datainfo = DataInfo()
        self.current_datainfo.filename = filepath
        self.reset_data_list(len(lines))
        detector = Detector()
        data_line = 0
        self.reset_data_list(len(lines))
        self.current_datainfo.detector.append(detector)
        self.current_datainfo.filename = filepath

        is_info = False
        is_center = False
        is_data_started = False

        base_q_unit = '1/A'
        base_i_unit = '1/cm'
        data_conv_q = Converter(base_q_unit)
        data_conv_i = Converter(base_i_unit)

        for line in lines:
            # Information line 1
            if is_info:
                is_info = False
                line_toks = line.split()

                # Wavelength in Angstrom
                try:
                    value = float(line_toks[1])
                    if self.has_converter and \
                            self.current_datainfo.source.wavelength_unit != 'A':
                        conv = Converter('A')
                        self.current_datainfo.source.wavelength = conv(value,
                            units=self.current_datainfo.source.wavelength_unit)
                    else:
                        self.current_datainfo.source.wavelength = value
                except KeyError:
                    msg = "ABSReader cannot read wavelength from %s" % filepath
                    self.current_datainfo.errors.append(msg)

                # Detector distance in meters
                try:
                    value = float(line_toks[3])
                    if self.has_converter and detector.distance_unit != 'm':
                        conv = Converter('m')
                        detector.distance = conv(value,
                                        units=detector.distance_unit)
                    else:
                        detector.distance = value
                except:
                    msg = "ABSReader cannot read SDD from %s" % filepath
                    self.current_datainfo.errors.append(msg)

                # Transmission
                try:
                    self.current_datainfo.sample.transmission = \
                        float(line_toks[4])
                except ValueError:
                    # Transmission isn't always in the header
                    pass

                # Sample thickness in mm
                try:
                    value = float(line_toks[5])
                    if self.has_converter and \
                            self.current_datainfo.sample.thickness_unit != 'cm':
                        conv = Converter('cm')
                        self.current_datainfo.sample.thickness = conv(value,
                            units=self.current_datainfo.sample.thickness_unit)
                    else:
                        self.current_datainfo.sample.thickness = value
                except ValueError:
                    # Thickness is not a mandatory entry
                    pass

            # MON CNT  LAMBDA  DET ANG  DET DIST  TRANS  THICK  AVE   STEP
            if line.count("LAMBDA") > 0:
                is_info = True

            # Find center info line
            if is_center:
                is_center = False
                line_toks = line.split()
                # Center in bin number
                center_x = float(line_toks[0])
                center_y = float(line_toks[1])

                # Bin size
                if self.has_converter and detector.pixel_size_unit != 'mm':
                    conv = Converter('mm')
                    detector.pixel_size.x = conv(5.08,
                                        units=detector.pixel_size_unit)
                    detector.pixel_size.y = conv(5.08,<commit_msg>Trim last character from sample thickness in ABS reader to remove the averaging parameter.<commit_after>"""
    IGOR 1D data reader
"""
#####################################################################
# This software was developed by the University of Tennessee as part of the
# Distributed Data Analysis of Neutron Scattering Experiments (DANSE)
# project funded by the US National Science Foundation.
# See the license text in license.txt
# copyright 2008, University of Tennessee
######################################################################

import logging
import numpy as np
from sas.sascalc.dataloader.file_reader_base_class import FileReader
from sas.sascalc.dataloader.data_info import DataInfo, plottable_1D, Data1D,\
    Detector
from sas.sascalc.dataloader.loader_exceptions import FileContentsException,\
    DefaultReaderException

logger = logging.getLogger(__name__)


class Reader(FileReader):
    """
    Class to load IGOR reduced .ABS files
    """
    # File type
    type_name = "IGOR 1D"
    # Wildcards
    type = ["IGOR 1D files (*.abs)|*.abs"]
    # List of allowed extensions
    ext = ['.abs']
    
    def get_file_contents(self):
        """ 
        Get the contents of the file
        
        :raise RuntimeError: when the file can't be opened
        :raise ValueError: when the length of the data vectors are inconsistent
        """
        buff = self.f_open.read()
        filepath = self.f_open.name
        lines = buff.splitlines()
        self.has_converter = True
        try:
            from sas.sascalc.data_util.nxsunit import Converter
        except:
            self.has_converter = False
        self.output = []
        self.current_datainfo = DataInfo()
        self.current_datainfo.filename = filepath
        self.reset_data_list(len(lines))
        detector = Detector()
        data_line = 0
        self.reset_data_list(len(lines))
        self.current_datainfo.detector.append(detector)
        self.current_datainfo.filename = filepath

        is_info = False
        is_center = False
        is_data_started = False

        base_q_unit = '1/A'
        base_i_unit = '1/cm'
        data_conv_q = Converter(base_q_unit)
        data_conv_i = Converter(base_i_unit)

        for line in lines:
            # Information line 1
            if is_info:
                is_info = False
                line_toks = line.split()

                # Wavelength in Angstrom
                try:
                    value = float(line_toks[1])
                    if self.has_converter and \
                            self.current_datainfo.source.wavelength_unit != 'A':
                        conv = Converter('A')
                        self.current_datainfo.source.wavelength = conv(value,
                            units=self.current_datainfo.source.wavelength_unit)
                    else:
                        self.current_datainfo.source.wavelength = value
                except KeyError:
                    msg = "ABSReader cannot read wavelength from %s" % filepath
                    self.current_datainfo.errors.append(msg)

                # Detector distance in meters
                try:
                    value = float(line_toks[3])
                    if self.has_converter and detector.distance_unit != 'm':
                        conv = Converter('m')
                        detector.distance = conv(value,
                                        units=detector.distance_unit)
                    else:
                        detector.distance = value
                except:
                    msg = "ABSReader cannot read SDD from %s" % filepath
                    self.current_datainfo.errors.append(msg)

                # Transmission
                try:
                    self.current_datainfo.sample.transmission = \
                        float(line_toks[4])
                except ValueError:
                    # Transmission isn't always in the header
                    pass

                # Sample thickness in mm
                try:
                    value = float(line_toks[5][:-1])
                    if self.has_converter and \
                            self.current_datainfo.sample.thickness_unit != 'cm':
                        conv = Converter('cm')
                        self.current_datainfo.sample.thickness = conv(value,
                            units=self.current_datainfo.sample.thickness_unit)
                    else:
                        self.current_datainfo.sample.thickness = value
                except ValueError:
                    # Thickness is not a mandatory entry
                    pass

            # MON CNT  LAMBDA  DET ANG  DET DIST  TRANS  THICK  AVE   STEP
            if line.count("LAMBDA") > 0:
                is_info = True

            # Find center info line
            if is_center:
                is_center = False
                line_toks = line.split()
                # Center in bin number
                center_x = float(line_toks[0])
                center_y = float(line_toks[1])

                # Bin size
                if self.has_converter and detector.pixel_size_unit != 'mm':
                    conv = Converter('mm')
                    detector.pixel_size.x = conv(5.08,
                                        units=detector.pixel_size_unit)
                    detector.pixel_size.y = conv(5.08,
{'fraction_non_alphanumeric': 0.0939828614008942, 'fraction_numerical': 0.004936661698956781, 'mean_word_length': 1.5216063879755755, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1363407', 'n_tokens_mistral': 2884, 'n_tokens_neox': 2680, 'n_words': 674}