<commit_before>//
//  CFFCharStrings.swift
//
//  The MIT License
//  Copyright (c) 2015 - 2017 <NAME>. All rights reserved.
//
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
//
//  The above copyright notice and this permission notice shall be included in
//  all copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
//  THE SOFTWARE.
//

import Foundation

extension CFFFontFace {
    
    func fontDICT(glyph: UInt16) -> CFFFontDICT {
        guard let fdSelect = self.fdSelect, let fontDICTArray = self.fontDICTArray else { return self.DICT }
        guard let index = fdSelect.fdIndex(glyph: glyph), index < fontDICTArray.count else { return self.DICT }
        return fontDICTArray[Int(index)]
    }
    
    func subroutineBias(_ subroutine: CFFINDEX) -> Int {
        if self.charstringType == 1 {
            return 0
        } else if subroutine.count < 1240 {
            return 107
        } else if subroutine.count < 33900 {
            return 1131
        } else {
            return 32768
        }
    }
    
    func shape(glyph: Int) -> [Shape.Component] {
        
        guard glyph < charStrings.count else { return [] }
        
        let fontDICT = self.fontDICT(glyph: UInt16(glyph))
        let subroutine = self.subroutine
        
        if let pDICT = fontDICT.pDICT {
            
            let pSubroutine = fontDICT.pSubroutine
            let subroutineBias = self.subroutineBias(subroutine)
            let pSubroutineBias = pSubroutine.map { self.subroutineBias($0) }
            
            var components: [Shape.Component] = []
            var component = Shape.Component(start: Point(), closed: true, segments: [])
            
            var stack: ArraySlice<Double> = []
            var _stems_count = 0
            var point = Point()
            var flag = false
            
            struct ParserError : Error {
                
            }
            
            func _stems() {
                
                if stack.count & 1 != 0 && !flag {
                    stack.removeFirst()
                }
                
                _stems_count += stack.count >> 1
                stack.removeAll(keepingCapacity: true)
                flag = true
            }
            
            func _parser(_ data: Data, tracing: (Set<Int>, Set<Int>)) throws {
                
                var data = data
                
                while let code = try? data.decode(UInt8.self) {
                    
                    switch code {
                        
                    case 11, 14:
                        
                        return
                        
                    case 1, 3, 18, 23:
                        
                        _stems()
                        
                    case 19, 20:
                        
                        _stems()
                        data.removeFirst((_stems_count + 7) >> 3)
                        
                    case 10:
                        
                        guard let code = stack.popLast() else { throw ParserError() }
                        
                        let codeIndex = Int(code) + subroutineBias
                        
                        guard !tracing.0.contains(codeIndex) else { throw ParserError() }
                        
                        var tracing = tracing
                        tracing.0.insert(codeIndex)
                        
                        if codeIndex < subroutine.count {
                            try _parser(subroutine[codeIndex], tracing: tracing)
                        }
                        
                    case 29:
                        
                        guard let code = stack.popLast() else { throw ParserError() }
                        
                        if let pSubroutine = pSubroutine, let pSubroutineBias = pSubroutineBias {
                            
                            let codeIndex = Int(code) + pSubroutineBias
                            
                            guard !tracing.1.contains(codeIndex) else { throw ParserError() }
                            
                            var tracing = tracing
                            tracing.1.insert(codeIndex)
                            
                            if codeIndex < pSubroutine.count {
                                try _parser(pSubroutine[codeIndex], tracing: tracing)
                            }
                        }
                        
                    case 22:
                        
                        if stack.count > 1 && !flag {
                            stack.removeFirst()
                            flag = true<commit_msg>bug fix<commit_after>//
//  CFFCharStrings.swift
//
//  The MIT License
//  Copyright (c) 2015 - 2017 <NAME>. All rights reserved.
//
//  Permission is hereby granted, free of charge, to any person obtaining a copy
//  of this software and associated documentation files (the "Software"), to deal
//  in the Software without restriction, including without limitation the rights
//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
//  copies of the Software, and to permit persons to whom the Software is
//  furnished to do so, subject to the following conditions:
//
//  The above copyright notice and this permission notice shall be included in
//  all copies or substantial portions of the Software.
//
//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
//  THE SOFTWARE.
//

import Foundation

extension CFFFontFace {
    
    func fontDICT(glyph: UInt16) -> CFFFontDICT {
        guard let fdSelect = self.fdSelect, let fontDICTArray = self.fontDICTArray else { return self.DICT }
        guard let index = fdSelect.fdIndex(glyph: glyph), index < fontDICTArray.count else { return self.DICT }
        return fontDICTArray[Int(index)]
    }
    
    func subroutineBias(_ subroutine: CFFINDEX) -> Int {
        if self.charstringType == 1 {
            return 0
        } else if subroutine.count < 1240 {
            return 107
        } else if subroutine.count < 33900 {
            return 1131
        } else {
            return 32768
        }
    }
    
    func shape(glyph: Int) -> [Shape.Component] {
        
        guard glyph < charStrings.count else { return [] }
        
        let fontDICT = self.fontDICT(glyph: UInt16(glyph))
        let subroutine = self.subroutine
        
        if let pDICT = fontDICT.pDICT {
            
            let pSubroutine = fontDICT.pSubroutine
            let subroutineBias = self.subroutineBias(subroutine)
            let pSubroutineBias = pSubroutine.map { self.subroutineBias($0) }
            
            var components: [Shape.Component] = []
            var component = Shape.Component(start: Point(), closed: true, segments: [])
            
            var stack: ArraySlice<Double> = []
            var _stems_count = 0
            var point = Point()
            var flag = false
            
            struct ParserError : Error {
                
            }
            
            func _stems() {
                
                if stack.count & 1 != 0 && !flag {
                    stack.removeFirst()
                }
                
                _stems_count += stack.count >> 1
                stack.removeAll(keepingCapacity: true)
                flag = true
            }
            
            func _parser(_ data: Data, tracing: (Set<Int>, Set<Int>)) throws {
                
                var data = data
                
                while let code = try? data.decode(UInt8.self) {
                    
                    switch code {
                        
                    case 11, 14:
                        
                        return
                        
                    case 1, 3, 18, 23:
                        
                        _stems()
                        
                    case 19, 20:
                        
                        _stems()
                        data.removeFirst((_stems_count + 7) >> 3)
                        
                    case 29:
                        
                        guard let code = stack.popLast() else { throw ParserError() }
                        
                        let codeIndex = Int(code) + subroutineBias
                        
                        guard !tracing.0.contains(codeIndex) else { throw ParserError() }
                        
                        var tracing = tracing
                        tracing.0.insert(codeIndex)
                        
                        if codeIndex < subroutine.count {
                            try _parser(subroutine[codeIndex], tracing: tracing)
                        }
                        
                    case 10:
                        
                        guard let code = stack.popLast() else { throw ParserError() }
                        
                        if let pSubroutine = pSubroutine, let pSubroutineBias = pSubroutineBias {
                            
                            let codeIndex = Int(code) + pSubroutineBias
                            
                            guard !tracing.1.contains(codeIndex) else { throw ParserError() }
                            
                            var tracing = tracing
                            tracing.1.insert(codeIndex)
                            
                            if codeIndex < pSubroutine.count {
                                try _parser(pSubroutine[codeIndex], tracing: tracing)
                            }
                        }
                        
                    case 22:
                        
                        if stack.count > 1 && !flag {
                            stack.removeFirst()
                            flag = true
{'fraction_non_alphanumeric': 0.0675, 'fraction_numerical': 0.012142857142857143, 'mean_word_length': 1.0983514424878231, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3399956', 'n_tokens_mistral': 2812, 'n_tokens_neox': 2569, 'n_words': 818}