<commit_before>#!/usr/bin/swift

import Foundation

func containerXMLData() -> Data {
    let container = XMLElement(name: "container")
    let root = XMLDocument(rootElement: container)
    let version = XMLElement(kind: .attribute)
    version.name = "version"
    version.stringValue = "1.0"
    container.addAttribute(version)

    let namespace = XMLElement(kind: .attribute)
    namespace.name = "xmlns"
    namespace.stringValue = "urn:oasis:names:tc:opendocument:xmlns:container"
    container.addAttribute(namespace)

    let rootfiles = XMLElement(name: "rootfiles")
    let rootfile = XMLElement(kind: .element, options: .nodeCompactEmptyElement)
    rootfile.name = "rootfile"
    let fullPath = XMLElement(kind: .attribute)
    fullPath.name = "full-path"
    fullPath.stringValue = "content.opf"
    rootfile.addAttribute(fullPath)
    let mediaType = XMLElement(kind: .attribute)
    mediaType.name = "media-type"
    mediaType.stringValue = "application/oebps-package+xml"
    rootfile.addAttribute(mediaType)


    rootfiles.addChild(rootfile)
    container.addChild(rootfiles)

    root.version = "1.0"
    root.isStandalone = true

    return root.xmlData(withOptions: Int(XMLNode.Options.nodePrettyPrint.rawValue))
}

let fm = FileManager.default

let currentDir = URL(fileURLWithPath: fm.currentDirectoryPath).appendingPathComponent("result", isDirectory: true)
if fm.fileExists(atPath: currentDir.absoluteString) == false {
    try! fm.createDirectory(at: currentDir,
                            withIntermediateDirectories: true,
                            attributes: nil)
    dump("created directory: \(currentDir)")
}

let contents = try! fm.contentsOfDirectory(at: currentDir, includingPropertiesForKeys: nil, options: [])

if contents.isEmpty == false {
    dump("removing \(contents.count) previous items")

    for item in contents{
        try! fm.removeItem(at: item)
    }
}

let address = "http://www.dataorienteddesign.com/dodmain/node{#0}.html"<commit_msg>content.opf WIP<commit_after>#!/usr/bin/swift

import Foundation

func containerXMLData() -> Data {
    let container = XMLElement(name: "container")
    let root = XMLDocument(rootElement: container)
    let version = XMLElement(kind: .attribute)
    version.name = "version"
    version.stringValue = "1.0"
    container.addAttribute(version)

    let namespace = XMLElement(kind: .attribute)
    namespace.name = "xmlns"
    namespace.stringValue = "urn:oasis:names:tc:opendocument:xmlns:container"
    container.addAttribute(namespace)

    let rootfiles = XMLElement(name: "rootfiles")
    let rootfile = XMLElement(kind: .element, options: .nodeCompactEmptyElement)
    rootfile.name = "rootfile"
    let fullPath = XMLElement(kind: .attribute)
    fullPath.name = "full-path"
    fullPath.stringValue = "content.opf"
    rootfile.addAttribute(fullPath)
    let mediaType = XMLElement(kind: .attribute)
    mediaType.name = "media-type"
    mediaType.stringValue = "application/oebps-package+xml"
    rootfile.addAttribute(mediaType)


    rootfiles.addChild(rootfile)
    container.addChild(rootfiles)

    root.version = "1.0"
    root.isStandalone = true

    return root.xmlData(withOptions: Int(XMLNode.Options.nodePrettyPrint.rawValue))
}

func packageXMLData() -> Data {
    let package = XMLElement(name: "package")
    let root = XMLDocument(rootElement: package)
    root.version = "1.0"
    root.isStandalone = true
    let version = XMLElement(kind: .attribute)
    version.name = "version"
    version.stringValue = "1.0"
    version.kind
    package.addAttribute(version)

    let metadata = XMLElement(name: "metadata")
    let title = XMLElement(name: "dc:title", stringValue: "dod")
    let creator = XMLElement(name: "dc:creator", stringValue: "<NAME>")
    metadata.addChild(title)
    metadata.addChild(creator)

    package.addChild(metadata)

    let namespace = XMLElement(kind: .namespace)
    namespace.name = ""
    namespace.stringValue = "http://www.idpf.org/2007/opf"
    package.addNamespace(namespace)

    let manifest = XMLElement(name: "manifest")

    func manifestItem(withName name: String, index: Int) -> XMLElement {
        let item = XMLElement(name: "item")
        if name.hasSuffix("png") {
            item.setAttributesAs(["href": "images/\(name)", "media-type": "image/png", "id": "id\(index)"])
        } else {
            item.setAttributesAs(["href": "text/\(name)", "media-type": "application/xhtml+xml", "id": "id\(index)"])
        }
        return item
    }

    let img = manifestItem(withName: "20.png", index: 5)
    let text = manifestItem(withName: "20.html", index: 10)

    manifest.addChild(img)
    manifest.addChild(text)

    package.addChild(manifest)

    return package.xmlData(withOptions: Int(XMLNode.Options.nodePrettyPrint.rawValue))
}

let fm = FileManager.default

let currentDir = URL(fileURLWithPath: fm.currentDirectoryPath).appendingPathComponent("result", isDirectory: true)
if fm.fileExists(atPath: currentDir.absoluteString) == false {
    try! fm.createDirectory(at: currentDir,
                            withIntermediateDirectories: true,
                            attributes: nil)
    dump("created directory: \(currentDir)")
}

let contents = try! fm.contentsOfDirectory(at: currentDir, includingPropertiesForKeys: nil, options: [])

if contents.isEmpty == false {
    dump("removing \(contents.count) previous items")

    for item in contents{
        try! fm.removeItem(at: item)
    }
}

let address = "http://www.dataorienteddesign.com/dodmain/node{#0}.html"
{'fraction_non_alphanumeric': 0.12371690977849811, 'fraction_numerical': 0.0045020709526382135, 'mean_word_length': 3.679022746419545, 'pattern_counts': {'":': 6, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3047246', 'n_tokens_mistral': 1739, 'n_tokens_neox': 1710, 'n_words': 340}