<commit_before>'use strict';

const _ = require('lodash');

class PWPropParser {
    constructor({smwDataArbitrator}) {
        this._smwDataArbitrator = smwDataArbitrator;

        this._rgx = {
            /* durations */
            range_dur: /(.*?)_(.*?)_(.*?)_time$/i,

            /* doses */
            range_dose: /(.*?)_(.*?)_(.*?)_dose$/i,
            def_dose: /(.*?)_(.*?)_dose$/i,

            /* bioavailability */
            def_bioavailability: /(.*?)_(.*?)_bioavailability$/i,

            /* units */
            dose_units: /(.*?)_dose_units$/i,
            roa_time_units: /(.*?)_(.*?)_time_units$/i,

            /* meta: tolerance */
            meta_tolerance_time: /Time_to_(.*?)_tolerance$/i,

            /* misc */
            wt_prop_glob: /\[\[(.*?)\]\]/g,
            wt_prop: /\[\[(.*?)\]\]/,

            /* misc sanitization */
            wt_link: /(\[\[.*?\]\])/ig,
            wt_named_link: /(\[\[.*?\|.*?\]\])/ig,
            wt_sub_sup: /<su[bp]>(.*?)<\/su[bp]>/ig,
        };

        this._flatMetaProps = new Map([
            ['addiction_potential', 'addictionPotential'],
            ['uncertaininteraction', 'uncertainInteractions'],
            ['unsafeinteraction', 'unsafeInteractions'],
            ['dangerousinteraction', 'dangerousInteractions'],
            ['effect', 'effects']
        ]);

        this._mappedMetaProps = new Map([
            ['cross-tolerance', prop => {
                const propTarget = 'crossTolerances';

                if (!this._rgx.wt_prop_glob.test(prop)) {
                    return [propTarget, []];
                }

                const mappedCrossTolerance = Array.prototype.slice
                    .call(prop.match(this._rgx.wt_prop_glob))
                    .map(item =>
                        this._rgx.wt_prop.exec(item)[1]
                    );

                return [propTarget, mappedCrossTolerance];
            }],
            ['featured', prop => ['featured', prop === 't']],
            ['psychoactive_class', prop => (['class.psychoactive', [].concat(prop)])],
            ['chemical_class', prop => (['class.chemical', [].concat(prop)])],
            ['toxicity', prop => (['toxicity', [].concat(prop)])]
        ]);

        this._sanitizers = new Map([
            [
                'addictionPotential',
                val => this._sanitizeText(val)
            ],
            [
                'toxicity',
                val => val.map(item => this._sanitizeText(item))
            ],
        ]);<commit_msg>Remove trailing hash from sem class, closes #17<commit_after>'use strict';

const _ = require('lodash');

class PWPropParser {
    constructor({smwDataArbitrator}) {
        this._smwDataArbitrator = smwDataArbitrator;

        this._rgx = {
            /* durations */
            range_dur: /(.*?)_(.*?)_(.*?)_time$/i,

            /* doses */
            range_dose: /(.*?)_(.*?)_(.*?)_dose$/i,
            def_dose: /(.*?)_(.*?)_dose$/i,

            /* bioavailability */
            def_bioavailability: /(.*?)_(.*?)_bioavailability$/i,

            /* units */
            dose_units: /(.*?)_dose_units$/i,
            roa_time_units: /(.*?)_(.*?)_time_units$/i,

            /* meta: tolerance */
            meta_tolerance_time: /Time_to_(.*?)_tolerance$/i,

            /* misc */
            wt_prop_glob: /\[\[(.*?)]]/g,
            wt_prop: /\[\[(.*?)]]/,

            /* misc sanitization */
            wt_link: /(\[\[.*?]])/ig,
            wt_named_link: /(\[\[.*?\|.*?]])/ig,
            wt_sub_sup: /<su[bp]>(.*?)<\/su[bp]>/ig,
        };

        this._flatMetaProps = new Map([
            ['addiction_potential', 'addictionPotential'],
            ['uncertaininteraction', 'uncertainInteractions'],
            ['unsafeinteraction', 'unsafeInteractions'],
            ['dangerousinteraction', 'dangerousInteractions'],
            ['effect', 'effects']
        ]);

        this._mappedMetaProps = new Map([
            ['cross-tolerance', prop => {
                const propTarget = 'crossTolerances';

                if (!this._rgx.wt_prop_glob.test(prop)) {
                    return [propTarget, []];
                }

                const mappedCrossTolerance = Array.prototype.slice
                    .call(prop.match(this._rgx.wt_prop_glob))
                    .map(item =>
                        this._rgx.wt_prop.exec(item)[1]
                    );

                return [propTarget, mappedCrossTolerance];
            }],
            ['featured', prop => ['featured', prop === 't']],
            ['toxicity', prop => (['toxicity', [].concat(prop)])],
            [
                'psychoactive_class',
                prop => ([
                    'class.psychoactive',
                    [].concat(prop && String(prop).replace(/#$/, '')),
                ]),
            ],
            [
                'chemical_class',
                prop => ([
                    'class.chemical',
                    [].concat(prop && String(prop).replace(/#$/, '')),
                ]),
            ],
        ]);

        this._sanitizers = new Map([
            [
                'addictionPotential',
                val => this._sanitizeText(val)
            ],
            [
                'toxicity',
                val => val.map(item => this._sanitizeText(item))
            ],
        ]);
{'fraction_non_alphanumeric': 0.18278367803242035, 'fraction_numerical': 0.0007452953232718465, 'mean_word_length': 1.4093357271095153, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2747632', 'n_tokens_mistral': 1682, 'n_tokens_neox': 1530, 'n_words': 215}