<commit_before>'use strict';

function Text2Speech(adapter, libs, options, sayIt) {
    const that         = this;
    const engines      = require('../admin/engines.js');
    const sayitEngines = engines.sayitEngines;
    const MP3FILE      = __dirname + '/../' + adapter.namespace + '.say.' + options.outFileExt;
    let appkey         = null;
    let cloudUrl       = null;
    let polly          = null;
    let yandexCloudIAM = null;

    function splitText(text, max) {
        if (!max) max = 70;
        if (text.length > max) {
            const parts = text.split(/,|.|;|:/);
            const result = [];
            for (let p = 0; p < parts.length; p++) {
                if (parts[p].length < max) {
                    result.push(parts[p]);
                    continue;
                }

                const _parts = parts[p].split(' ');
                let i = 0;
                for (let w = 0; w < _parts.length; w++) {
                    if (_parts[i] && ((result[i] || '') + ' ' + _parts[w]).length > max) {
                        i++;
                    }
                    if (!result[i]) {
                        result.push(_parts[w]);
                    } else {
                        result[i] += ' ' + _parts[w];
                    }
                }
            }
            return result;
        } else {
            return [text];
        }
    }

    function copyFile(text, language, volume, source, dest, callback) {
        libs.fs = libs.fs || require('fs');

        const input  = libs.fs.createReadStream(source);               // Input stream
        const output = libs.fs.createWriteStream(dest);                // Output stream

        input.on('data',  d =>output.write(d));
        // Copy in to out
        input.on('error', err => {
            if (callback) {
                callback('Cannot copy file "' + source + '" to "' + dest + '": ' + e, '', '', volume);
                callback = null;
            }              // And notify callback
        });
        output.on('error', err => {
            if (callback) {
                callback('Cannot copy file "' + source + '" to "' + dest + '": ' + e, '', '', volume);
                callback = null;
            }              // And notify callback
        });

        // Raise errors
        input.on('end', () => {                               // When input ends
            output.end();                                            // close output
            if (callback) {
                adapter.log.info('Copied file "' + source + '" to "' + dest + '"');
                callback(null, text, language, volume);          // And notify callback
                callback = null;
            }
        });
    }

    function cacheFile(error, text, language, volume, seconds, callback) {
        libs.fs   = libs.fs   || require('fs');
        libs.path = libs.path || require('path');

        if (!error) {
            if (adapter.config.cache) {
                const md5filename = libs.path.join(options.cacheDir, libs.crypto.createHash('md5').update(language + ';' + text).digest('hex') + '.' + options.outFileExt);

                const stat = libs.fs.statSync(MP3FILE);
                if (stat.size < 100) {
                    adapter.log.warn('Received file is too short: ' + libs.fs.readFileSync(MP3FILE).toString());
                } else {
                    adapter.log.debug('Caching File ' + md5filename + ' for "' + language + ';' + text + '" now');
                    copyFile(text, language, volume, MP3FILE, md5filename, (error, text, language, volume) => {
                        if (error) adapter.log.error(error);
                    });
                }
            }
        }

        callback(error, text, language, volume, seconds);
    }

    function sayItGetSpeechGoogle(text, language, volume, callback) {
        if (typeof volume === 'function') {
            callback = volume;
            volume = undefined;
        }

        if (text.length === 0) {
            if (callback) callback(null, '', language, volume, 0);
            return;
        }

        if (text.length > 70) {
            const parts = splitText(text);
            for (let t = 1; t < parts.length; t++) {
                sayIt(parts[t], language, volume);
            }
            text = parts[0];
        }

        language = language || adapter.config.engine;

        // https://translate.google.com/translate_tts?ie=UTF-8&client=tw-ob&q=%D0%BE%D1%82%D0%B2%D0%B5%D1%82%D0%B8%D1%82%D1%8C%207&tl=ru&total=1&idx=0&textlen=10
        const options = {
            host: 'translate.google.com',
            //port: 443,
            path: '/translate_tts?ie=UTF-8&client=tw-ob&q=' + encodeURI(text) + '&tl=' + encodeURI(language) + '&total=1&idx=0&textlen=' + text.length //
        };

        if (language === 'ru') {
            options.headers = {
                'Accept-Encoding':  'identity;q=1, *;q=0',
                Range:              'bytes=0-',
                Referer:            'https://www.google.de/',
                'User-Agent':       'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36'
                //"User-Agent":      "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) Gecko/20100101 Firefox/25.0",
                //"Accept":          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                //"Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
                //"Accept-Encoding": "gzip, deflate"//
            };
        }

        if (!libs.https) libs.https = require('https');
        if (!libs.fs)    libs.fs    = require('fs');

        let sounddata = '';
        libs.https.get(options, res => {
            res.setEncoding('binary');

            res.on('data', chunk => sounddata += chunk);

            res.on('end', () => {
                if (sounddata.length < 100) {
                    if (callback) callback('Cannot get file: received file is too short', text, language, volume, 0);
                    return;
                }

                if (sounddata.toString().indexOf('302 Moved') !== -1) {
                    if (callback) callback('http://' + options.host + options.path + '\nCannot get file: ' + sounddata, text, language, volume, 0);
                    return;
                }

                try {
                    libs.fs.writeFileSync(MP3FILE, sounddata, 'binary');
                } catch (err) {
                    if (callback) callback('File error: ' + err, text, language, volume, 0);
                    return;
                }
                if (callback) callback(null, text, language, volume, seconds);
            });
        }).on('error', err => {
            sounddata = '';
            if (callback) callback('Cannot get file:' + err, text, language, volume, 0);
        });
    }

    function sayItGetSpeechAcapela(text, language, volume, callback) {
        const options = {
            host: 'vaassl3.acapela-group.com',
            path: '/Services/Synthesizer?prot_vers=2&req_voice=' + language + '22k&cl_env=FLASH_AS_3.0&req_text=%5Cvct%3D100%5C+%5Cspd%3D180%5C+' +
            encodeURI(text) + '&req_asw_type=STREAM&cl_vers=1-30&req_err_as_id3=yes&cl_login=ACAPELA_BOX&cl_app=PROD&cl_pwd=<PASSWORD>'
        };

        if (!libs.https) libs.https = require('https');

        let sounddata = '';
        libs.https.get(options, res => {
            res.setEncoding('binary');

            res.on('data', chunk => sounddata += chunk);

            res.on('end', () => {
                if (sounddata.length < 100) {
                    if (callback) callback('Cannot get file: received file is too short', text, language, volume);<commit_msg>* fix<commit_after>'use strict';

function Text2Speech(adapter, libs, options, sayIt) {
    const that         = this;
    const engines      = require('../admin/engines.js');
    const sayitEngines = engines.sayitEngines;
    const MP3FILE      = __dirname + '/../' + adapter.namespace + '.say.' + options.outFileExt;
    let appkey         = null;
    let cloudUrl       = null;
    let polly          = null;
    let yandexCloudIAM = null;

    function splitText(text, max) {
        if (!max) max = 70;
        if (text.length > max) {
            const parts = text.split(/,|.|;|:/);
            const result = [];
            for (let p = 0; p < parts.length; p++) {
                if (parts[p].length < max) {
                    result.push(parts[p]);
                    continue;
                }

                const _parts = parts[p].split(' ');
                let i = 0;
                for (let w = 0; w < _parts.length; w++) {
                    if (_parts[i] && ((result[i] || '') + ' ' + _parts[w]).length > max) {
                        i++;
                    }
                    if (!result[i]) {
                        result.push(_parts[w]);
                    } else {
                        result[i] += ' ' + _parts[w];
                    }
                }
            }
            return result;
        } else {
            return [text];
        }
    }

    function copyFile(text, language, volume, source, dest, callback) {
        libs.fs = libs.fs || require('fs');

        const input  = libs.fs.createReadStream(source);               // Input stream
        const output = libs.fs.createWriteStream(dest);                // Output stream

        input.on('data',  d =>output.write(d));
        // Copy in to out
        input.on('error', err => {
            if (callback) {
                callback('Cannot copy file "' + source + '" to "' + dest + '": ' + e, '', '', volume);
                callback = null;
            }              // And notify callback
        });
        output.on('error', err => {
            if (callback) {
                callback('Cannot copy file "' + source + '" to "' + dest + '": ' + e, '', '', volume);
                callback = null;
            }              // And notify callback
        });

        // Raise errors
        input.on('end', () => {                               // When input ends
            output.end();                                            // close output
            if (callback) {
                adapter.log.info('Copied file "' + source + '" to "' + dest + '"');
                callback(null, text, language, volume);          // And notify callback
                callback = null;
            }
        });
    }

    function cacheFile(error, text, language, volume, seconds, callback) {
        libs.fs   = libs.fs   || require('fs');
        libs.path = libs.path || require('path');

        if (!error) {
            if (adapter.config.cache) {
                const md5filename = libs.path.join(options.cacheDir, libs.crypto.createHash('md5').update(language + ';' + text).digest('hex') + '.' + options.outFileExt);

                const stat = libs.fs.statSync(MP3FILE);
                if (stat.size < 100) {
                    adapter.log.warn('Received file is too short: ' + libs.fs.readFileSync(MP3FILE).toString());
                } else {
                    adapter.log.debug('Caching File ' + md5filename + ' for "' + language + ';' + text + '" now');
                    copyFile(text, language, volume, MP3FILE, md5filename, (error, text, language, volume) => {
                        if (error) adapter.log.error(error);
                    });
                }
            }
        }

        callback(error, text, language, volume, seconds);
    }

    function sayItGetSpeechGoogle(text, language, volume, callback) {
        if (typeof volume === 'function') {
            callback = volume;
            volume = undefined;
        }

        if (text.length === 0) {
            if (callback) callback(null, '', language, volume, 0);
            return;
        }

        if (text.length > 70) {
            const parts = splitText(text);
            for (let t = 1; t < parts.length; t++) {
                sayIt(parts[t], language, volume);
            }
            text = parts[0];
        }

        language = language || adapter.config.engine;

        // https://translate.google.com/translate_tts?ie=UTF-8&client=tw-ob&q=%D0%BE%D1%82%D0%B2%D0%B5%D1%82%D0%B8%D1%82%D1%8C%207&tl=ru&total=1&idx=0&textlen=10
        const options = {
            host: 'translate.google.com',
            //port: 443,
            path: '/translate_tts?ie=UTF-8&client=tw-ob&q=' + encodeURI(text) + '&tl=' + encodeURI(language) + '&total=1&idx=0&textlen=' + text.length //
        };

        if (language === 'ru') {
            options.headers = {
                'Accept-Encoding':  'identity;q=1, *;q=0',
                Range:              'bytes=0-',
                Referer:            'https://www.google.de/',
                'User-Agent':       'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36'
                //"User-Agent":      "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:25.0) Gecko/20100101 Firefox/25.0",
                //"Accept":          "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                //"Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
                //"Accept-Encoding": "gzip, deflate"//
            };
        }

        if (!libs.https) libs.https = require('https');
        if (!libs.fs)    libs.fs    = require('fs');

        let sounddata = '';
        libs.https.get(options, res => {
            res.setEncoding('binary');

            res.on('data', chunk => sounddata += chunk);

            res.on('end', () => {
                if (sounddata.length < 100) {
                    if (callback) callback('Cannot get file: received file is too short', text, language, volume, 0);
                    return;
                }

                if (sounddata.toString().indexOf('302 Moved') !== -1) {
                    if (callback) callback('http://' + options.host + options.path + '\nCannot get file: ' + sounddata, text, language, volume, 0);
                    return;
                }

                try {
                    libs.fs.writeFileSync(MP3FILE, sounddata, 'binary');
                } catch (err) {
                    if (callback) callback('File error: ' + err, text, language, volume, 0);
                    return;
                }
                that.getLength(MP3FILE, (error, seconds) => {
                    if (callback) callback(error, text, language, volume, seconds);
                });
            });
        }).on('error', err => {
            sounddata = '';
            if (callback) callback('Cannot get file:' + err, text, language, volume, 0);
        });
    }

    function sayItGetSpeechAcapela(text, language, volume, callback) {
        const options = {
            host: 'vaassl3.acapela-group.com',
            path: '/Services/Synthesizer?prot_vers=2&req_voice=' + language + '22k&cl_env=FLASH_AS_3.0&req_text=%5Cvct%3D100%5C+%5Cspd%3D180%5C+' +
            encodeURI(text) + '&req_asw_type=STREAM&cl_vers=1-30&req_err_as_id3=yes&cl_login=ACAPELA_BOX&cl_app=PROD&cl_pwd=<PASSWORD>'
        };

        if (!libs.https) libs.https = require('https');

        let sounddata = '';
        libs.https.get(options, res => {
            res.setEncoding('binary');

            res.on('data', chunk => sounddata += chunk);

            res.on('end', () => {
                if (sounddata.length < 100) {
                    if (callback) callback('Cannot get file: received file is too short', text, language, volume);
{'fraction_non_alphanumeric': 0.15190278221937958, 'fraction_numerical': 0.019379597057882954, 'mean_word_length': 1.47679391731348, 'pattern_counts': {'":': 12, '<': 19, '<?xml version=': 0, '>': 36, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6094057', 'n_tokens_mistral': 4805, 'n_tokens_neox': 4462, 'n_words': 941}