<commit_before>package se.kb.libris.whelks.component

import groovy.util.logging.Slf4j as Log

import se.kb.libris.whelks.*

@Log
class DiskStorage implements Storage {
    def storageDir = "./storage/"
    Whelk whelk
    boolean enabled = true

    String id = "diskstorage"

    DiskStorage() {
        def env = System.getenv()
        def whelk_storage = (env["PROJECT_HOME"] ? env["PROJECT_HOME"] : System.getProperty("user.home")) + "/whelk_storage"
        setStorageDir(whelk_storage)
    }

    DiskStorage(def directoryName) {
        setStorageDir(directoryName)
    }

    def void setWhelk(se.kb.libris.whelks.Whelk w) { this.whelk = w }

    def void enable() {this.enabled = true}
    def void disable() {this.enabled = false}

    @Override
    OutputStream getOutputStreamFor(Document doc) {
        def filename = doc.identifier.toString()
        log.debug "${this.class.name} storing file $filename in $storageDir"
        def fullpath = storageDir + "/" + filename
        def path = fullpath.substring(0, fullpath.lastIndexOf("/"))
        log.debug "PATH: $path"
        new File(path).mkdirs()
        File file = new File("$storageDir/$filename")
        return file.newOutputStream()
        //file.write(new String(d.data))
    }

    LookupResult lookup(Key key) {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    Document get(URI uri) {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    Iterable<Document> getAll() {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    void store(Document doc) {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    void store(Iterable<Document> docs) {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    void delete(URI uri) {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    def init() {
        log.debug "Making sure directory $storageDir exists ..."
        new File(storageDir).mkdirs()
    }

    def setStorageDir(String directoryName) { 
        storageDir = directoryName 
        init()
    }

    def deprecated_add(Document d) {
        def filename = (d.identifier ? d.identifier.toString() : _create_filename())
        log.debug "${this.class.name} storing file $filename in $storageDir"
        def fullpath = storageDir + "/" + filename
        def path = fullpath.substring(0, fullpath.lastIndexOf("/"))
        log.debug "PATH: $path"
        new File(path).mkdirs()
        File file = new File("$storageDir/$filename")
        file.write(new String(d.data))
        d.identifier = new URI(filename)
    }

    Document retrieve(URI u, raw=false) {
        def s 
        def filename = u.toString()
        File f = new File("$storageDir/$filename")
        try {
            return this.whelk.createDocument().withIdentifier(u).withContentType("content/type").withData(f.text.getBytes())
        } catch (FileNotFoundException fnfe) {
            return null
        }
    }

    def _create_filename() {
        def pool = ['a'..'z','A'..'Z',0..9,'_'].flatten()
        Random rand = new Random(System.currentTimeMillis())

        def passChars = (0..10).collect { pool[rand.nextInt(pool.size())] }
        passChars.join()
    }
}<commit_msg>Implemented simple DiskStorage.<commit_after>package se.kb.libris.whelks.component

import groovy.util.logging.Slf4j as Log

import se.kb.libris.whelks.*
import se.kb.libris.whelks.basic.*

@Log
class DiskStorage implements Storage {
    def storageDir = "./storage/"
    Whelk whelk
    boolean enabled = true

    String id = "diskstorage"

    DiskStorage() {
        def env = System.getenv()
        def whelk_storage = (env["PROJECT_HOME"] ? env["PROJECT_HOME"] : System.getProperty("user.home")) + "/whelk_storage"
        setStorageDir(whelk_storage)
    }

    DiskStorage(String directoryName) {
        setStorageDir(directoryName)
    }

    def void setWhelk(se.kb.libris.whelks.Whelk w) { this.whelk = w }

    def void enable() {this.enabled = true}
    def void disable() {this.enabled = false}

    @Override
    OutputStream getOutputStreamFor(Document doc) {
        def filename = doc.identifier.toString()
        log.debug "${this.class.name} storing file $filename in $storageDir"
        def fullpath = storageDir + "/" + filename
        def path = fullpath.substring(0, fullpath.lastIndexOf("/"))
        log.debug "PATH: $path"
        new File(path).mkdirs()
        File file = new File("$storageDir/$filename")
        return file.newOutputStream()
        //file.write(new String(d.data))
    }

    LookupResult lookup(Key key) {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    Document get(URI uri) {
        def filename = uri.toString()
        File f = new File("$storageDir/$filename")
        try {
            return new BasicDocument(f.text)
        } catch (FileNotFoundException fnfe) {
            return null
        }
    }

    Iterable<Document> getAll() {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    void store(Document doc) {
        def filename = doc.identifier.toString() 
        log.debug "${this.class.name} storing file $filename in $storageDir"
        def fullpath = storageDir + "/" + filename
        def path = fullpath.substring(0, fullpath.lastIndexOf("/"))
        log.debug "PATH: $path"
        new File(path).mkdirs()
        File file = new File("$storageDir/$filename")
        file.write(new String(doc.toJson()))
    }

    void store(Iterable<Document> docs) {
        docs.each {
            store(it)
        }
    }

    void delete(URI uri) {
        throw new UnsupportedOperationException("Not supported yet.")
    }

    def init() {
        log.debug "Making sure directory $storageDir exists ..."
        new File(storageDir).mkdirs()
    }

    def setStorageDir(String directoryName) { 
        storageDir = directoryName 
        init()
    }
}
{'fraction_non_alphanumeric': 0.11928131758442855, 'fraction_numerical': 0.0018299783729828648, 'mean_word_length': 2.544811320754717, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '134618', 'n_tokens_mistral': 1784, 'n_tokens_neox': 1709, 'n_words': 412}