<commit_before>var _module_nsstring = true

#if os(Linux)
    import Foundation
    import Glibc
    
    private let O = "0".ord, A = "A".ord, percent = "%".ord
    
    private func unhex( char: Int8 ) -> Int8 {
        return char < A ? char - O : char - A + 10
    }
    
    extension String {
        
        var ord: Int8 {
            return Int8(utf8.first!)
        }
        
        var stringByRemovingPercentEncoding: String? {
            var arr = [Int8]( count: 100000, repeatedValue: 0 )
            var out = UnsafeMutablePointer<Int8>( arr )
            
            withCString { (bytes) in
                var bytes = UnsafeMutablePointer<Int8>(bytes)
                
                while out < &arr + arr.count {
                    let start = strchr( bytes, Int32(percent) ) - UnsafeMutablePointer<Int8>( bytes )
                    
                    let extralen = start < 0 ? Int(strlen( bytes )) : start + 1
                    let required = out - UnsafeMutablePointer<Int8>(arr) + extralen
                    if required > arr.count {
                        var newarr = [Int8]( count: Int(Double(required) * 1.5), repeatedValue: 0 )
                        strcpy( &newarr, arr )
                        arr = newarr
                        out = &arr + Int(strlen( arr ))
                    }
                    
                    if start < 0 {
                        strcat( out, bytes )
                        break
                    }
                    
                    bytes[start] = 0
                    strcat( out, bytes )
                    bytes += start + 3
                    out += start + 1
                    out[-1] = (unhex( bytes[-2] ) << 4) + unhex( bytes[-1] )
                }
            }
            
            return String.fromCString( arr )
        }
        
        func stringByAddingPercentEscapesUsingEncoding( encoding: UInt ) -> String? {
            return self
        }
        
        func stringByTrimmingCharactersInSet( cset: NSCharacterSet ) -> String {
            return self
        }
        
        func componentsSeparatedByString( sep: String ) -> [String] {
            var out = [String]()
            
            withCString { (bytes) in
                sep.withCString { (sbytes) in
                    var bytes = UnsafeMutablePointer<Int8>( bytes )
                    
                    while true {
                        let start = strstr( bytes, sbytes ) - UnsafeMutablePointer<Int8>( bytes )
                        if start < 0 {
                            out.append( String.fromCString( bytes )! )
                            break
                        }
                        bytes[start] = 0
                        out.append( String.fromCString( bytes )! )
                        bytes += start + Int(strlen( sbytes ))
                    }
                }
            }
            
            return out
        }
        
        func rangeOfString( str: String ) -> Range<Int>? {
            var start = -1
            withCString { (bytes) in
                str.withCString { (sbytes) in
                    start = strstr( bytes, sbytes ) - UnsafeMutablePointer<Int8>( bytes )
                }
            }
            return start < 0 ? nil : start..<start+str.utf8.count
        }
        
        func substringToIndex( index: Int ) -> String {
            var out = self
            withCString { (bytes) in
                let bytes = UnsafeMutablePointer<Int8>(bytes)
                bytes[index] = 0
                out = String.fromCString( bytes )!
            }
            return out
        }
        
        func substringFromIndex( index: Int ) -> String {
            var out = self
            withCString { (bytes) in
                out = String.fromCString( bytes+index )!
            }
            return out
        }
        
    }
#endif


var _module_dispatch = true

#if os(Linux)
    import Glibc
    
    typealias dispatch_queue_t = Int
    
    public let DISPATCH_QUEUE_CONCURRENT = 0, DISPATCH_QUEUE_PRIORITY_HIGH = 0, DISPATCH_QUEUE_PRIORITY_LOW = 0, DISPATCH_QUEUE_PRIORITY_BACKGROUND = 0, DISPATCH_QUEUE_SERIAL = 0
    
    public func dispatch_get_global_queue( type: Int, _ flags: Int ) -> dispatch_queue_t {
        return type
    }
    
    public func dispatch_queue_create( name: String, _ type: Int ) -> dispatch_queue_t {
        return type
    }
    
    public func dispatch_sync( queue: Int, _ block: () -> () ) {
        block()
    }
    
    private class pthreadBlock {<commit_msg>Made dispatch_queue_t type public<commit_after>var _module_nsstring = true

#if os(Linux)
    import Foundation
    import Glibc
    
    private let O = "0".ord, A = "A".ord, percent = "%".ord
    
    private func unhex( char: Int8 ) -> Int8 {
        return char < A ? char - O : char - A + 10
    }
    
    extension String {
        
        var ord: Int8 {
            return Int8(utf8.first!)
        }
        
        var stringByRemovingPercentEncoding: String? {
            var arr = [Int8]( count: 100000, repeatedValue: 0 )
            var out = UnsafeMutablePointer<Int8>( arr )
            
            withCString { (bytes) in
                var bytes = UnsafeMutablePointer<Int8>(bytes)
                
                while out < &arr + arr.count {
                    let start = strchr( bytes, Int32(percent) ) - UnsafeMutablePointer<Int8>( bytes )
                    
                    let extralen = start < 0 ? Int(strlen( bytes )) : start + 1
                    let required = out - UnsafeMutablePointer<Int8>(arr) + extralen
                    if required > arr.count {
                        var newarr = [Int8]( count: Int(Double(required) * 1.5), repeatedValue: 0 )
                        strcpy( &newarr, arr )
                        arr = newarr
                        out = &arr + Int(strlen( arr ))
                    }
                    
                    if start < 0 {
                        strcat( out, bytes )
                        break
                    }
                    
                    bytes[start] = 0
                    strcat( out, bytes )
                    bytes += start + 3
                    out += start + 1
                    out[-1] = (unhex( bytes[-2] ) << 4) + unhex( bytes[-1] )
                }
            }
            
            return String.fromCString( arr )
        }
        
        func stringByAddingPercentEscapesUsingEncoding( encoding: UInt ) -> String? {
            return self
        }
        
        func stringByTrimmingCharactersInSet( cset: NSCharacterSet ) -> String {
            return self
        }
        
        func componentsSeparatedByString( sep: String ) -> [String] {
            var out = [String]()
            
            withCString { (bytes) in
                sep.withCString { (sbytes) in
                    var bytes = UnsafeMutablePointer<Int8>( bytes )
                    
                    while true {
                        let start = strstr( bytes, sbytes ) - UnsafeMutablePointer<Int8>( bytes )
                        if start < 0 {
                            out.append( String.fromCString( bytes )! )
                            break
                        }
                        bytes[start] = 0
                        out.append( String.fromCString( bytes )! )
                        bytes += start + Int(strlen( sbytes ))
                    }
                }
            }
            
            return out
        }
        
        func rangeOfString( str: String ) -> Range<Int>? {
            var start = -1
            withCString { (bytes) in
                str.withCString { (sbytes) in
                    start = strstr( bytes, sbytes ) - UnsafeMutablePointer<Int8>( bytes )
                }
            }
            return start < 0 ? nil : start..<start+str.utf8.count
        }
        
        func substringToIndex( index: Int ) -> String {
            var out = self
            withCString { (bytes) in
                let bytes = UnsafeMutablePointer<Int8>(bytes)
                bytes[index] = 0
                out = String.fromCString( bytes )!
            }
            return out
        }
        
        func substringFromIndex( index: Int ) -> String {
            var out = self
            withCString { (bytes) in
                out = String.fromCString( bytes+index )!
            }
            return out
        }
        
    }
#endif


var _module_dispatch = true

#if os(Linux)
    import Glibc
    
    public typealias dispatch_queue_t = Int
    
    public let DISPATCH_QUEUE_CONCURRENT = 0, DISPATCH_QUEUE_PRIORITY_HIGH = 0, DISPATCH_QUEUE_PRIORITY_LOW = 0, DISPATCH_QUEUE_PRIORITY_BACKGROUND = 0, DISPATCH_QUEUE_SERIAL = 0
    
    public func dispatch_get_global_queue( type: Int, _ flags: Int ) -> dispatch_queue_t {
        return type
    }
    
    public func dispatch_queue_create( name: String, _ type: Int ) -> dispatch_queue_t {
        return type
    }
    
    public func dispatch_sync( queue: Int, _ block: () -> () ) {
        block()
    }
    
    private class pthreadBlock {
{'fraction_non_alphanumeric': 0.08368655580915117, 'fraction_numerical': 0.011085751548744702, 'mean_word_length': 1.1385080176620963, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5290237', 'n_tokens_mistral': 2538, 'n_tokens_neox': 2376, 'n_words': 610}