<commit_before>package parsers;

import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Map;

import com.google.common.collect.ImmutableMap;

public class CharsetDetector {

    private static final int MIN_BOM_WIDTH = 2;
    private static final int MAX_BOM_WIDTH = 5;

    private static final Map<Charset, byte[]> charsetBoms = ImmutableMap.<Charset, byte[]> builder()
            .put(StandardCharsets.UTF_8, new byte[] { (byte) 239, (byte) 187, (byte) 191 })
            .put(StandardCharsets.UTF_16, new byte[] { (byte) 255, (byte) 254 })
            .put(StandardCharsets.UTF_16BE, new byte[] { (byte) 254, (byte) 255 })
            .build();

    public static Charset detect(Path path) {
        Charset charset = StandardCharsets.UTF_8;

        try (InputStream input = new FileInputStream(path.toFile())) {
            final byte[] bom = new byte[MAX_BOM_WIDTH];
            if (input.read(bom) > MIN_BOM_WIDTH) {
                charset = charsetBoms.entrySet().stream()
                        .filter(pair -> matchesBom(bom, pair.getValue()))
                        .map(pair -> pair.getKey())
                        .findFirst()
                        .orElse(StandardCharsets.UTF_8);
            }
        }
        catch (Exception e) {
            throw new RuntimeException(e);
        }

        return charset;
    }<commit_msg>Fix detector when file contains no data except BOM<commit_after>package parsers;

import java.io.FileInputStream;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Map;

import com.google.common.collect.ImmutableMap;

public class CharsetDetector {

    private static final int MIN_BOM_WIDTH = 2;
    private static final int MAX_BOM_WIDTH = 5;

    private static final Map<Charset, byte[]> charsetBoms = ImmutableMap.<Charset, byte[]> builder()
            .put(StandardCharsets.UTF_8, new byte[] { (byte) 239, (byte) 187, (byte) 191 })
            .put(StandardCharsets.UTF_16, new byte[] { (byte) 255, (byte) 254 })
            .put(StandardCharsets.UTF_16BE, new byte[] { (byte) 254, (byte) 255 })
            .build();

    public static Charset detect(Path path) {
        Charset charset = StandardCharsets.UTF_8;

        try (InputStream input = new FileInputStream(path.toFile())) {
            final byte[] bom = new byte[MAX_BOM_WIDTH];
            if (input.read(bom) >= MIN_BOM_WIDTH) {
                charset = charsetBoms.entrySet().stream()
                        .filter(pair -> matchesBom(bom, pair.getValue()))
                        .map(pair -> pair.getKey())
                        .findFirst()
                        .orElse(StandardCharsets.UTF_8);
            }
        }
        catch (Exception e) {
            throw new RuntimeException(e);
        }

        return charset;
    }
{'fraction_non_alphanumeric': 0.11463903743315508, 'fraction_numerical': 0.020053475935828877, 'mean_word_length': 2.2854006586169047, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1073024', 'n_tokens_mistral': 942, 'n_tokens_neox': 858, 'n_words': 212}