<commit_before>korpApp = angular.module("korpApp")
korpApp.factory "utils", ($location) ->
    valfilter : (attrobj) ->
        return if attrobj.isStructAttr then "_." + attrobj.value else attrobj.value

    setupHash : (scope, config) ->

        onWatch = () ->
            for obj in config
                val = $location.search()[obj.key]
                unless val
                    if obj.default? then val = obj.default else continue


                val = (obj.val_in or _.identity)(val)

                if "scope_name" of obj
                    scope[obj.scope_name] = val
                else if "scope_func" of obj
                    scope[obj.scope_func](val)
                else
                    scope[obj.key] = val

        onWatch()
        scope.$watch ( () -> $location.search() ), ->
            onWatch()

        for obj in config
            watch = obj.expr or obj.scope_name or obj.key
            scope.$watch watch, do (obj, watch) ->
                (val) ->
                    val = (obj.val_out or _.identity)(val)
                    if val == obj.default then val = null
                    $location.search obj.key, val or null
                    if obj.key == "page" then c.log "post change", watch, val
                    obj.post_change?(val)


korpApp.factory 'backend', ($http, $q, utils, lexicons) ->
    requestCompare : (cmpObj1, cmpObj2, reduce) ->
        reduce = _.map reduce, (item) -> item.replace(/^_\./, "")

        # remove all corpora which do not include all the "reduce"-attributes
        filterFun = (item) -> settings.corpusListing.corpusHasAttrs item, reduce
        corpora1 = _.filter cmpObj1.corpora, filterFun
        corpora2 = _.filter cmpObj2.corpora, filterFun

        corpusListing = settings.corpusListing.subsetFactory cmpObj1.corpora

        split = _.filter(reduce, (r) ->
            settings.corpusListing.getCurrentAttributes()[r]?.type == "set").join(',')

        rankedReduce = _.filter reduce, (item) ->
            settings.corpusListing.getCurrentAttributes(settings.corpusListing.getReduceLang())[item]?.ranked
        top = _.map(rankedReduce, (item) ->
            return item + ":1").join ','

        def = $q.defer()
        params =
            group_by : reduce.join ','
            set1_corpus : corpora1.join(",").toUpperCase()
            set1_cqp : cmpObj1.cqp
            set2_corpus : corpora2.join(",").toUpperCase()
            set2_cqp : cmpObj2.cqp
            max : 50
            split : split
            top : top

        conf =
            url : settings.korpBackendURL + "/loglike"
            params : params
            method : "GET"
            headers : {}


        _.extend conf.headers, model.getAuthorizationHeader()


        xhr = $http(conf)

        xhr.then (response) ->
            data = response.data

            if data.ERROR
                def.reject()
                return

            loglikeValues = data.loglike

            objs = _.map loglikeValues, (value, key) ->
                return {
                    value: key
                    loglike: value
                }

            tables = _.groupBy objs, (obj) ->
                if obj.loglike > 0
                    obj.abs = data.set2[obj.value]
                    return "positive"
                else
                    obj.abs = data.set1[obj.value]
                    return "negative"

            groupAndSum = (table, currentMax) ->
                groups = _.groupBy table, (obj) ->
                    obj.value.replace(/(:.+?)(\/|$| )/g, "$2")

                res = _.map groups, (value, key) ->
                    tokenLists = _.map key.split("/"), (tokens) ->
                        return tokens.split(" ")

                    loglike = 0
                    abs = 0
                    cqp = []
                    elems = []

                    _.map value, (val) ->
                        abs += val.abs
                        loglike += val.loglike
                        elems.push val.value
                    if Math.abs(loglike) > currentMax
                        currentMax = Math.abs loglike
                    { key: key, loglike : loglike, abs : abs, elems : elems, tokenLists: tokenLists }

                return [res, currentMax]

            [tables.positive, max] = groupAndSum tables.positive, 0
            [tables.negative, max] = groupAndSum tables.negative, max

            def.resolve [tables, max, cmpObj1, cmpObj2, reduce], xhr

        return def.promise

    relatedWordSearch : (lemgram) ->
        return lexicons.relatedWordSearch(lemgram)

    requestMapData: (cqp, cqpExprs, within, attribute) ->
        cqpSubExprs = {}
        _.map _.keys(cqpExprs), (subCqp, idx) ->
            cqpSubExprs["subcqp" + idx] = subCqp

        def = $q.defer()
        params =
            group_by_struct: attribute.label
            cqp: cqp
            corpus: attribute.corpora.join(",")
            incremental: true
            split: attribute.label
        _.extend params, settings.corpusListing.getWithinParameters()

        _.extend params, cqpSubExprs

        conf =
            url : settings.korpBackendURL + "/count"
            params : params
            method : "GET"
            headers : {}

        _.extend conf.headers, model.getAuthorizationHeader()


        xhr = $http(conf)

        xhr.then (response) ->
            data = response.data
            createResult = (subResult, cqp, label) ->
                mergeSubResults = (absolute, relative) ->
                    res_list = []
                    for {value: value1, freq: abs_freq} in absolute
                        remove_idxs = []
                        for {value: value2, freq: rel_freq}, idx in relative
                            val1 = _.values(value1)[0][0]
                            val2 = _.values(value2)[0][0]
                            if val1 == val2
                                res_list.push {value: val1, abs_freq: abs_freq, rel_freq: rel_freq}
                                remove_idxs.push idx
                        removed_elems = _.pullAt relative, remove_idxs
                    return res_list

                points = []
                _.map mergeSubResults(subResult.absolute, subResult.relative), (actual_hit) ->
                    hit = actual_hit.value
                    if (hit is "") or (hit.startsWith " ")
                        return
                    [name, countryCode, lat, lng] = hit.split ";"

                    points.push (
                        abs: actual_hit.abs_freq
                        rel: actual_hit.rel_freq
                        name: name
                        countryCode: countryCode
                        lat : parseFloat lat
                        lng : parseFloat lng)

                return (
                    label: label
                    cqp: cqp
                    points: points
                )

            if _.isEmpty cqpExprs
                result = [createResult data.total, cqp, "Σ"]
            else
                result = []
                for subResult in data.total.slice(1, data.total.length)
                    result.push createResult subResult, subResult.cqp, cqpExprs[subResult.cqp]

            if data.ERROR
                def.reject()
                return

            def.resolve [{corpora: attribute.corpora, cqp: cqp, within: within, data: result, attribute: attribute}], xhr

        return def.promise

korpApp.factory 'nameEntitySearch', ($rootScope, $q) ->

    class NameEntities
        constructor: () ->

        request: (cqp) ->
            @def = $q.defer()
            @promise = @def.promise<commit_msg>minor syntax fix before converting to es6<commit_after>korpApp = angular.module("korpApp")
korpApp.factory "utils", ($location) ->
    valfilter : (attrobj) ->
        return if attrobj.isStructAttr then "_." + attrobj.value else attrobj.value

    setupHash : (scope, config) ->

        onWatch = () ->
            for obj in config
                val = $location.search()[obj.key]
                unless val
                    if obj.default? then val = obj.default else continue


                val = (obj.val_in or _.identity)(val)

                if "scope_name" of obj
                    scope[obj.scope_name] = val
                else if "scope_func" of obj
                    scope[obj.scope_func](val)
                else
                    scope[obj.key] = val

        onWatch()
        scope.$watch ( () -> $location.search() ), ->
            onWatch()

        for obj in config
            watch = obj.expr or obj.scope_name or obj.key
            scope.$watch watch, do (obj, watch) ->
                (val) ->
                    val = (obj.val_out or _.identity)(val)
                    if val == obj.default then val = null
                    $location.search obj.key, val or null
                    if obj.key == "page" then c.log "post change", watch, val
                    obj.post_change?(val)


korpApp.factory 'backend', ($http, $q, utils, lexicons) ->
    requestCompare : (cmpObj1, cmpObj2, reduce) ->
        reduce = _.map reduce, (item) -> item.replace(/^_\./, "")

        # remove all corpora which do not include all the "reduce"-attributes
        filterFun = (item) -> settings.corpusListing.corpusHasAttrs item, reduce
        corpora1 = _.filter cmpObj1.corpora, filterFun
        corpora2 = _.filter cmpObj2.corpora, filterFun

        corpusListing = settings.corpusListing.subsetFactory cmpObj1.corpora

        split = _.filter(reduce, (r) ->
            settings.corpusListing.getCurrentAttributes()[r]?.type == "set").join(',')

        rankedReduce = _.filter reduce, (item) ->
            settings.corpusListing.getCurrentAttributes(settings.corpusListing.getReduceLang())[item]?.ranked
        top = _.map(rankedReduce, (item) ->
            return item + ":1").join ','

        def = $q.defer()
        params =
            group_by : reduce.join ','
            set1_corpus : corpora1.join(",").toUpperCase()
            set1_cqp : cmpObj1.cqp
            set2_corpus : corpora2.join(",").toUpperCase()
            set2_cqp : cmpObj2.cqp
            max : 50
            split : split
            top : top

        conf =
            url : settings.korpBackendURL + "/loglike"
            params : params
            method : "GET"
            headers : {}


        _.extend conf.headers, model.getAuthorizationHeader()


        xhr = $http(conf)

        xhr.then (response) ->
            data = response.data

            if data.ERROR
                def.reject()
                return

            loglikeValues = data.loglike

            objs = _.map loglikeValues, (value, key) ->
                return {
                    value: key
                    loglike: value
                }

            tables = _.groupBy objs, (obj) ->
                if obj.loglike > 0
                    obj.abs = data.set2[obj.value]
                    return "positive"
                else
                    obj.abs = data.set1[obj.value]
                    return "negative"

            groupAndSum = (table, currentMax) ->
                groups = _.groupBy table, (obj) ->
                    obj.value.replace(/(:.+?)(\/|$| )/g, "$2")

                res = _.map groups, (value, key) ->
                    tokenLists = _.map key.split("/"), (tokens) ->
                        return tokens.split(" ")

                    loglike = 0
                    abs = 0
                    cqp = []
                    elems = []

                    _.map value, (val) ->
                        abs += val.abs
                        loglike += val.loglike
                        elems.push val.value
                    if Math.abs(loglike) > currentMax
                        currentMax = Math.abs loglike
                    { key: key, loglike : loglike, abs : abs, elems : elems, tokenLists: tokenLists }

                return [res, currentMax]

            [tables.positive, max] = groupAndSum tables.positive, 0
            [tables.negative, max] = groupAndSum tables.negative, max

            def.resolve [tables, max, cmpObj1, cmpObj2, reduce], xhr

        return def.promise

    relatedWordSearch : (lemgram) ->
        return lexicons.relatedWordSearch(lemgram)

    requestMapData: (cqp, cqpExprs, within, attribute) ->
        cqpSubExprs = {}
        _.map _.keys(cqpExprs), (subCqp, idx) ->
            cqpSubExprs["subcqp" + idx] = subCqp

        def = $q.defer()
        params =
            group_by_struct: attribute.label
            cqp: cqp
            corpus: attribute.corpora.join(",")
            incremental: true
            split: attribute.label
        _.extend params, settings.corpusListing.getWithinParameters()

        _.extend params, cqpSubExprs

        conf =
            url : settings.korpBackendURL + "/count"
            params : params
            method : "GET"
            headers : {}

        _.extend conf.headers, model.getAuthorizationHeader()


        xhr = $http(conf)

        xhr.then (response) ->
            data = response.data
            createResult = (subResult, cqp, label) ->
                mergeSubResults = (absolute, relative) ->
                    res_list = []
                    for {value: value1, freq: abs_freq} in absolute
                        remove_idxs = []
                        for {value: value2, freq: rel_freq}, idx in relative
                            val1 = _.values(value1)[0][0]
                            val2 = _.values(value2)[0][0]
                            if val1 == val2
                                res_list.push {value: val1, abs_freq: abs_freq, rel_freq: rel_freq}
                                remove_idxs.push idx
                        removed_elems = _.pullAt relative, remove_idxs
                    return res_list

                points = []
                _.map mergeSubResults(subResult.absolute, subResult.relative), (actual_hit) ->
                    hit = actual_hit.value
                    if (hit is "") or (hit.startsWith " ")
                        return
                    [name, countryCode, lat, lng] = hit.split ";"

                    points.push {
                        abs: actual_hit.abs_freq
                        rel: actual_hit.rel_freq
                        name: name
                        countryCode: countryCode
                        lat : parseFloat lat
                        lng : parseFloat lng}

                return (
                    label: label
                    cqp: cqp
                    points: points
                )

            if _.isEmpty cqpExprs
                result = [createResult data.total, cqp, "Σ"]
            else
                result = []
                for subResult in data.total.slice(1, data.total.length)
                    result.push createResult subResult, subResult.cqp, cqpExprs[subResult.cqp]

            if data.ERROR
                def.reject()
                return

            def.resolve [{corpora: attribute.corpora, cqp: cqp, within: within, data: result, attribute: attribute}], xhr

        return def.promise

korpApp.factory 'nameEntitySearch', ($rootScope, $q) ->

    class NameEntities
        constructor: () ->

        request: (cqp) ->
            @def = $q.defer()
            @promise = @def.promise
{'fraction_non_alphanumeric': 0.10208576240445653, 'fraction_numerical': 0.005376344086021506, 'mean_word_length': 1.284889743969217, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4496947', 'n_tokens_mistral': 4278, 'n_tokens_neox': 4007, 'n_words': 981}