<commit_before>package ai.elimu.util.csv;

import ai.elimu.dao.SoundDao;
import ai.elimu.model.content.Letter;
import ai.elimu.web.content.letter.LetterCsvExportController;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

public class CsvLetterExtractionHelper {

    private static final Logger logger = LogManager.getLogger();

    /**
     * For information on how the CSV files were generated, see {@link LetterCsvExportController#handleRequest}.
     */
    public static List<Letter> getLettersFromCsvBackup(File csvFile, SoundDao soundDao) {
        logger.info("getLettersFromCsvBackup");

        List<Letter> letters = new ArrayList<>();

        Path csvFilePath = Paths.get(csvFile.toURI());
        logger.info("csvFilePath: {}", csvFilePath);
        try {
            Reader reader = Files.newBufferedReader(csvFilePath);
            CSVFormat csvFormat = CSVFormat.DEFAULT
                    .withHeader(
                            "id",
                            "text",
                            "diacritic",
                            "usage_count"
                    )
                    .withSkipHeaderRecord();
            CSVParser csvParser = new CSVParser(reader, csvFormat);
            for (CSVRecord csvRecord : csvParser) {
                logger.info("csvRecord: {}", csvRecord);

                Letter letter = new Letter();

                String text = csvRecord.get("text");
                letter.setText(text);

                boolean diacritic = Boolean.parseBoolean(csvRecord.get("diacritic"));
                letter.setDiacritic(diacritic);
<commit_msg>#1396 Create CSVFormat without usage of deprecated API<commit_after>package ai.elimu.util.csv;

import ai.elimu.dao.SoundDao;
import ai.elimu.model.content.Letter;
import ai.elimu.web.content.letter.LetterCsvExportController;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;

public class CsvLetterExtractionHelper {

    private static final Logger logger = LogManager.getLogger();

    /**
     * For information on how the CSV files were generated, see {@link LetterCsvExportController#handleRequest}.
     */
    public static List<Letter> getLettersFromCsvBackup(File csvFile, SoundDao soundDao) {
        logger.info("getLettersFromCsvBackup");

        List<Letter> letters = new ArrayList<>();

        Path csvFilePath = Paths.get(csvFile.toURI());
        logger.info("csvFilePath: {}", csvFilePath);

        CSVFormat csvFormat = CSVFormat.Builder.create()
            .setHeader(
                "id",
                "text",
                "diacritic",
                "usage_count"
            )
            .setSkipHeaderRecord(true)
            .build();

        try {
            Reader reader = Files.newBufferedReader(csvFilePath);
            CSVParser csvParser = new CSVParser(reader, csvFormat);
            for (CSVRecord csvRecord : csvParser) {
                logger.info("csvRecord: {}", csvRecord);

                Letter letter = new Letter();

                String text = csvRecord.get("text");
                letter.setText(text);

                boolean diacritic = Boolean.parseBoolean(csvRecord.get("diacritic"));
                letter.setDiacritic(diacritic);

{'fraction_non_alphanumeric': 0.10571923743500866, 'fraction_numerical': 0.0019806882891804903, 'mean_word_length': 2.546971027216857, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1371464', 'n_tokens_mistral': 1194, 'n_tokens_neox': 1122, 'n_words': 228}