<commit_before>const striptags = require('striptags');

const { convertFormatToSeconds, convertSecondsToFormat, normalizeString } = require('../utils');

class SubtitlePiece {
    static fromSubtitlesParserItem(item) {
        return new SubtitlePiece(item);
    }

    constructor({ id, startTime, endTime, text, data }) {
        if (!id) throw new Error();
        if (!startTime) throw new Error();
        if (!endTime) throw new Error();
        if (!text) throw new Error();

        this.id = id;
        this.startTime = typeof startTime === 'number' ? convertSecondsToFormat(startTime) : startTime;
        this.endTime = typeof endTime === 'number' ? convertSecondsToFormat(endTime) : endTime;
        this.text = text;
        this.data = data;
    }

    get words() {
        return striptags(this.text).split(' ');
    }

    get normalizedWords() {
        return this.words.map(normalizeString).filter(word => word.length > 0);
    }

    get startTimeS() {
        return convertFormatToSeconds(this._startTime);
    }

    get startTimeF() {
        return this._startTime;
    }

    get textLevenshtein() {
        return this.normalizedWords.join("");
    }
}

module.exports = SubtitlePiece;<commit_msg>Fix wrong text splitting to words<commit_after>const striptags = require('striptags');

const { convertFormatToSeconds, convertSecondsToFormat, normalizeString } = require('../utils');

class SubtitlePiece {
    static fromSubtitlesParserItem(item) {
        return new SubtitlePiece(item);
    }

    constructor({ id, startTime, endTime, text, data }) {
        if (!id) throw new Error();
        if (!startTime) throw new Error();
        if (!endTime) throw new Error();
        if (!text) throw new Error();

        this.id = id;
        this.startTime = typeof startTime === 'number' ? convertSecondsToFormat(startTime) : startTime;
        this.endTime = typeof endTime === 'number' ? convertSecondsToFormat(endTime) : endTime;
        this.text = text;
        this.data = data;
    }

    get words() {
        return striptags(this.text).split(/\s+/);
    }

    get normalizedWords() {
        return this.words.map(normalizeString).filter(word => word.length > 0);
    }

    get startTimeS() {
        return convertFormatToSeconds(this._startTime);
    }

    get startTimeF() {
        return this._startTime;
    }

    get textLevenshtein() {
        return this.normalizedWords.join("");
    }
}

module.exports = SubtitlePiece;
{'fraction_non_alphanumeric': 0.12611156022635409, 'fraction_numerical': 0.0008084074373484236, 'mean_word_length': 2.7844036697247705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6479839', 'n_tokens_mistral': 743, 'n_tokens_neox': 716, 'n_words': 170}