<commit_before>import {Resource} from 'idai-components-2';
import {reduce, flatReduce} from 'tsfun';

/**
 * @author <NAME>
 */
export module CsvRowsConversion {

    const PATH_SEP = '.';


    /**
     * @param sep
     */
    export function parse(sep: string) { return (rows: string[]): Array<Resource> => {

        if (rows.length < 1) return [];
        const headings = rows[0].split(sep);
        rows.shift();

        return flatReduce((row: string) => makeResource(headings)(row.split(sep)))(rows);
    }}


    function implodePaths(currentSegmentObject: any, pathSegments: any[], val: any) {

        let index = parseInt(pathSegments[0]);
        if (isNaN(index)) index = pathSegments[0];

        if (pathSegments.length < 2) {

            currentSegmentObject[index] = val;
            return;
        }

        const nextIndex = parseInt(pathSegments[1]);
        const newItem = isNaN(nextIndex) ? {} : Array(nextIndex + 1);

        if (!currentSegmentObject[index]) currentSegmentObject[index] = newItem;<commit_msg>Extract function<commit_after>import {Resource} from 'idai-components-2';
import {reduce, flatReduce} from 'tsfun';

/**
 * @author <NAME>
 */
export module CsvRowsConversion {

    const PATH_SEP = '.';


    /**
     * @param sep
     */
    export function parse(sep: string) { return (rows: string[]): Array<Resource> => {

        if (rows.length < 1) return [];
        const headings = rows[0].split(sep);
        rows.shift();

        return flatReduce(splitRows(headings, sep))(rows);
    }}


    const splitRows = (headings: string[], sep: string) => (row: string) => makeResource(headings)(row.split(sep));


    function implodePaths(currentSegmentObject: any, pathSegments: any[], val: any) {

        let index = parseInt(pathSegments[0]);
        if (isNaN(index)) index = pathSegments[0];

        if (pathSegments.length < 2) {

            currentSegmentObject[index] = val;
            return;
        }

        const nextIndex = parseInt(pathSegments[1]);
        const newItem = isNaN(nextIndex) ? {} : Array(nextIndex + 1);

        if (!currentSegmentObject[index]) currentSegmentObject[index] = newItem;
{'fraction_non_alphanumeric': 0.1456221198156682, 'fraction_numerical': 0.007373271889400922, 'mean_word_length': 2.9046762589928057, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4730478', 'n_tokens_mistral': 672, 'n_tokens_neox': 646, 'n_words': 148}