<commit_before>//
//  UTS46+Loading.swift
//  icumap2code
//
//  Created by <NAME> on 2020-05-08.
//

import Foundation
import Compression
import os

extension UTS46 {

	private static func parseHeader(from data: Data) throws -> Header? {
		let headerData = data.prefix(12)

		guard headerData.count == 12 else { throw UTS46Error.badSize }

		return Header(rawValue: headerData)
	}

	static func load(from url: URL) throws {
		let fileData = try Data(contentsOf: url)

		guard let header = try? parseHeader(from: fileData) else { return }

		guard header.version == 1 else { throw UTS46Error.unknownVersion }

		let offset = header.dataOffset

		guard fileData.count > offset else { throw UTS46Error.badSize }

		let compressedData = fileData[offset...]
<commit_msg>Verify file version<commit_after>//
//  UTS46+Loading.swift
//  icumap2code
//
//  Created by <NAME> on 2020-05-08.
//

import Foundation
import Compression
import os

extension UTS46 {

	private static func parseHeader(from data: Data) throws -> Header? {
		let headerData = data.prefix(12)

		guard headerData.count == 12 else { throw UTS46Error.badSize }

		return Header(rawValue: headerData)
	}

	static func load(from url: URL) throws {
		let fileData = try Data(contentsOf: url)

		guard let header = try? parseHeader(from: fileData) else { return }

		guard header.version == 1 else { throw UTS46Error.unknownVersion }

		let offset = header.dataOffset

		guard fileData.count > offset else { throw UTS46Error.badSize }

		switch header.version {
			case 1:
				break;
			default:
				if #available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) {
					os_log("Unrecognized version found; assuming 1.", type: .debug)
				} else {
					print("Unrecognized version found; assuming 1.")
				}
		}

		let compressedData = fileData[offset...]

{'fraction_non_alphanumeric': 0.10633608815426997, 'fraction_numerical': 0.03471074380165289, 'mean_word_length': 3.461916461916462, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4716641', 'n_tokens_mistral': 678, 'n_tokens_neox': 564, 'n_words': 176}