<commit_before>import {Subject} from 'rxjs'
import {finalize, first, takeUntil, tap} from 'rxjs/operators'
import {getLogger} from 'log'
import {requestTag, tileProviderTag} from 'tag'
import _ from 'lodash'

const log = getLogger('tileManager/executor')

export const getRequestExecutor = concurrency => {
    const finished$ = new Subject()

    const activeRequests = {} // by requestId
    const activeRequestCount = {} // by tileProviderId
    const hiddenTileProviders = {} // by tileProviderId

    const getCount = tileProviderId =>
        tileProviderId
            ? activeRequestCount[tileProviderId] || 0
            : Object.keys(activeRequests).length

    const increaseCount = tileProviderId => {
        const count = getCount(tileProviderId) + 1
        activeRequestCount[tileProviderId] = count
        return count
    }
        
    const decreaseCount = tileProviderId => {
        const count = getCount(tileProviderId) - 1
        activeRequestCount[tileProviderId] = count
        return count
    }
    
    const isAvailable = () =>
        getCount() < concurrency

    const getPriorityTileProviderIds = tileProviderId => {
        const visibleTileProviderIds = _(activeRequestCount)
            .toPairs()
            .filter(([tileProviderId, _count]) => isHidden(tileProviderId) === false)
            .sortBy(([_tileProviderId, count]) => count)
            .map(([tileProviderId, _count]) => tileProviderId)
            .value()
        const hiddenTileProviderIds = _(activeRequestCount)
            .toPairs()
            .filter(([tileProviderId, _count]) => isHidden(tileProviderId) === true)
            .sortBy(([_tileProviderId, count]) => count)
            .map(([tileProviderId, _count]) => tileProviderId)
            .value()
        const tileProviderIds = _.uniq([tileProviderId, ...visibleTileProviderIds, ...hiddenTileProviderIds])
        log.debug(() => 'Priority tile providers: ', tileProviderIds.map(tileProviderId => tileProviderTag(tileProviderId)))
        return tileProviderIds
    }
    
    const start = ({tileProviderId, requestId, request, response$, cancel$}) => {
        activeRequests[requestId] = {tileProviderId, requestId, request, response$, cancel$, timestamp: Date.now()}
        const activeRequestCountByTileProviderId = increaseCount(tileProviderId)
        log.debug(() => `Started ${requestTag({tileProviderId, requestId})}, active: ${activeRequestCountByTileProviderId}/${getCount()}`)
    }<commit_msg>Simplified calculation of priority tile providers.<commit_after>import {Subject} from 'rxjs'
import {finalize, first, takeUntil, tap} from 'rxjs/operators'
import {getLogger} from 'log'
import {requestTag, tileProviderTag} from 'tag'
import _ from 'lodash'

const log = getLogger('tileManager/executor')

export const getRequestExecutor = concurrency => {
    const finished$ = new Subject()

    const activeRequests = {} // by requestId
    const activeRequestCount = {} // by tileProviderId
    const hiddenTileProviders = {} // by tileProviderId

    const getCount = tileProviderId =>
        tileProviderId
            ? activeRequestCount[tileProviderId] || 0
            : Object.keys(activeRequests).length

    const increaseCount = tileProviderId => {
        const count = getCount(tileProviderId) + 1
        activeRequestCount[tileProviderId] = count
        return count
    }
        
    const decreaseCount = tileProviderId => {
        const count = getCount(tileProviderId) - 1
        activeRequestCount[tileProviderId] = count
        return count
    }
    
    const isAvailable = () =>
        getCount() < concurrency

    const getPriorityTileProviderIds = tileProviderId => {
        const priorityTileProviderIds = _(activeRequestCount)
            .toPairs()
            .sortBy([
                ([tileProviderId, _count]) => isHidden(tileProviderId), // sort by visibility (visible first)
                ([_tileProviderId, count]) => count                     // sort by count (lowest first)
            ])
            .map(([tileProviderId, _count]) => tileProviderId)
            .value()
        const tileProviderIds = _.uniq([tileProviderId, ...priorityTileProviderIds])
        log.debug(() => 'Priority tile providers: ', tileProviderIds.map(tileProviderId => tileProviderTag(tileProviderId)))
        return tileProviderIds
    }
    
    const start = ({tileProviderId, requestId, request, response$, cancel$}) => {
        activeRequests[requestId] = {tileProviderId, requestId, request, response$, cancel$, timestamp: Date.now()}
        const activeRequestCountByTileProviderId = increaseCount(tileProviderId)
        log.debug(() => `Started ${requestTag({tileProviderId, requestId})}, active: ${activeRequestCountByTileProviderId}/${getCount()}`)
    }
{'fraction_non_alphanumeric': 0.11115754283326369, 'fraction_numerical': 0.0012536564981195152, 'mean_word_length': 3.0879590093936806, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6988186', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1242, 'n_words': 297}