<commit_before>//
//  PipelineProcessing.swift
//  Siesta
//
//  Created by Paul on 2016/8/7.
//  Copyright © 2016 Bust Out Solutions. All rights reserved.
//

import Foundation

internal extension Pipeline
    {
    private var stagesInOrder: [PipelineStage]
        { return order.flatMap { self[$0] } }

    private typealias StageAndEntry = (PipelineStage, CacheEntryProtocol?)

    private func stagesAndEntries(for resource: Resource) -> [StageAndEntry]
        {
        return stagesInOrder.map
            { stage in (stage, stage.cacheBox?.buildEntry(resource)) }
        }

    internal func makeProcessor(_ rawResponse: Response, resource: Resource) -> (Void) -> Response
        {
        // Generate cache keys on main thread (because this touches Resource)
        let stagesAndEntries = self.stagesAndEntries(for: resource)

        // Return deferred processor to run on background queue
        return
            {
            let result = self.processAndCache(rawResponse, using: stagesAndEntries)

            debugLog(.pipeline,       ["  └╴Response after pipeline:", result.summary()])
            debugLog(.networkDetails, ["    Details:", result.dump("      ")])

            return result
            }
        }

    // Runs on a background queue
    private func processAndCache<StagesAndEntries: Collection>(
            _ rawResponse: Response,
            using stagesAndEntries: StagesAndEntries)
        -> Response
        where StagesAndEntries.Iterator.Element == StageAndEntry
        {
        return stagesAndEntries.reduce(rawResponse)
            {
            let input = $0, (stage, cacheEntry) = $1

            let output = stage.process(input)

            if case .success(let entity) = output,
               let cacheEntry = cacheEntry
                {
                debugLog(.cache, ["  ├╴Caching entity with", type(of: entity.content), "content in", cacheEntry])
                cacheEntry.write(entity)
                }

            return output
            }
        }

    internal func cachedEntity(for resource: Resource, onHit: @escaping (Entity<Any>) -> ())
        {
        // Extract cache keys on main thread
        let stagesAndEntries = self.stagesAndEntries(for: resource)

        defaultEntityCacheWorkQueue.async
            {
            if let entity = self.cacheLookup(using: stagesAndEntries)
                {
                DispatchQueue.main.async
                    { onHit(entity) }
                }
            }
        }

    // Runs on a background queue
    private func cacheLookup(using stagesAndEntries: [StageAndEntry]) -> Entity<Any>?
        {
        for (index, (_, cacheEntry)) in stagesAndEntries.enumerated().reversed()
            {
            if let result = cacheEntry?.read()
                {
                debugLog(.cache, ["Cache hit for", cacheEntry])

                let processed = processAndCache(
                    .success(result),
                    using: stagesAndEntries.suffix(from: index + 1))

                switch processed
                    {
                    case .failure:
                        debugLog(.cache, ["Error processing cached entity; will ignore cached value. Error:", processed])

                    case .success(let entity):
                        return entity
                    }
                }
            }
        return nil
        }

    internal func updateCacheEntryTimestamps(_ timestamp: TimeInterval, for resource: Resource)
        {
        for (_, cacheEntry) in stagesAndEntries(for: resource)
            { cacheEntry?.updateTimestamp(timestamp) }
        }

    internal func removeCacheEntries(for resource: Resource)
        {
        for (_, cacheEntry) in stagesAndEntries(for: resource)
            { cacheEntry?.remove() }<commit_msg>Internal Pipeline cache methods now static<commit_after>//
//  PipelineProcessing.swift
//  Siesta
//
//  Created by Paul on 2016/8/7.
//  Copyright © 2016 Bust Out Solutions. All rights reserved.
//

import Foundation

internal extension Pipeline
    {
    private var stagesInOrder: [PipelineStage]
        { return order.flatMap { self[$0] } }

    private typealias StageAndEntry = (PipelineStage, CacheEntryProtocol?)

    private func stagesAndEntries(for resource: Resource) -> [StageAndEntry]
        {
        return stagesInOrder.map
            { stage in (stage, stage.cacheBox?.buildEntry(resource)) }
        }

    internal func makeProcessor(_ rawResponse: Response, resource: Resource) -> (Void) -> Response
        {
        // Generate cache keys on main thread (because this touches Resource)
        let stagesAndEntries = self.stagesAndEntries(for: resource)

        // Return deferred processor to run on background queue
        return
            {
            let result = Pipeline.processAndCache(rawResponse, using: stagesAndEntries)

            debugLog(.pipeline,       ["  └╴Response after pipeline:", result.summary()])
            debugLog(.networkDetails, ["    Details:", result.dump("      ")])

            return result
            }
        }

    // Runs on a background queue
    private static func processAndCache<StagesAndEntries: Collection>(
            _ rawResponse: Response,
            using stagesAndEntries: StagesAndEntries)
        -> Response
        where StagesAndEntries.Iterator.Element == StageAndEntry
        {
        return stagesAndEntries.reduce(rawResponse)
            {
            let input = $0, (stage, cacheEntry) = $1

            let output = stage.process(input)

            if case .success(let entity) = output,
               let cacheEntry = cacheEntry
                {
                debugLog(.cache, ["  ├╴Caching entity with", type(of: entity.content), "content in", cacheEntry])
                cacheEntry.write(entity)
                }

            return output
            }
        }

    internal func cachedEntity(for resource: Resource, onHit: @escaping (Entity<Any>) -> ())
        {
        // Extract cache keys on main thread
        let stagesAndEntries = self.stagesAndEntries(for: resource)

        defaultEntityCacheWorkQueue.async
            {
            if let entity = Pipeline.cacheLookup(using: stagesAndEntries)
                {
                DispatchQueue.main.async
                    { onHit(entity) }
                }
            }
        }

    // Runs on a background queue
    private static func cacheLookup(using stagesAndEntries: [StageAndEntry]) -> Entity<Any>?
        {
        for (index, (_, cacheEntry)) in stagesAndEntries.enumerated().reversed()
            {
            if let result = cacheEntry?.read()
                {
                debugLog(.cache, ["Cache hit for", cacheEntry])

                let processed = Pipeline.processAndCache(
                    .success(result),
                    using: stagesAndEntries.suffix(from: index + 1))

                switch processed
                    {
                    case .failure:
                        debugLog(.cache, ["Error processing cached entity; will ignore cached value. Error:", processed])

                    case .success(let entity):
                        return entity
                    }
                }
            }
        return nil
        }

    internal func updateCacheEntryTimestamps(_ timestamp: TimeInterval, for resource: Resource)
        {
        for (_, cacheEntry) in stagesAndEntries(for: resource)
            { cacheEntry?.updateTimestamp(timestamp) }
        }

    internal func removeCacheEntries(for resource: Resource)
        {
        for (_, cacheEntry) in stagesAndEntries(for: resource)
            { cacheEntry?.remove() }
{'fraction_non_alphanumeric': 0.08813603322949118, 'fraction_numerical': 0.0036344755970924196, 'mean_word_length': 1.7606592619132928, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7142267', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1809, 'n_words': 503}