<commit_before>// diskdatabank.js
//
// On-disk implementation of Databank interface
//
// Copyright 2011, StatusNet Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var databank = require('../databank'),
    Databank = databank.Databank,
    DatabankError = databank.DatabankError,
    AlreadyExistsError = databank.AlreadyExistsError,
    NoSuchThingError = databank.NoSuchThingError,
    crypto = require('crypto'),
    fs = require('fs'),
    path = require('path');

var DiskDatabank = function(params) {
    this.dir = params.dir || '/var/lib/diskdatabank/';
    this.mode = params.mode || 0755;
    this.hashDepth = params.hashDepth || 3;
    this.schema = params.schema || {};
};

DiskDatabank.prototype = new Databank();

DiskDatabank.prototype.toFilename = function(type, id) {
    var hash = this.toHash(id);

    return path.join(this.toDirname(type, id), hash + '.json');
};

DiskDatabank.prototype.toHash = function(id) {

    var hash = crypto.createHash('md5'),
        str, data;

    data = "" + id;

    hash.update(data);
    str = hash.digest('base64');

    // Make it a little more FS-safe

    str = str.replace(/\+/g, '-');
    str = str.replace(/\//g, '_');
    str = str.replace(/=/g, '');

    return str;
};

DiskDatabank.prototype.toDirname = function(type, id) {
    var n;
    var dirname = path.join(this.dir, type);
    var hash = this.toHash(id);

    for (n = 0; n < Math.min(this.hashDepth, hash.length); n++) {
        dirname = path.join(dirname, hash.substr(0, n + 1));
    }

    return dirname;
};

DiskDatabank.prototype.connect = function(params, onCompletion) {
    this.ensureDir(this.dir, onCompletion);
};

DiskDatabank.prototype.disconnect = function(onCompletion) {
    // Always succeed
    onCompletion(null);
};

DiskDatabank.prototype.create = function(type, id, value, onCompletion) {
    var dirname = this.toDirname(type, id),
        filename = this.toFilename(type, id);

    this.ensureDir(dirname, function(err) {
	if (err) {
	    onCompletion(err, null);
	} else {
            fs.stat(filename, function(err, stats) {
		if (err && err.code == 'ENOENT') {
                    fs.writeFile(filename, JSON.stringify(value), 'utf8', function(err) {
			if (err) {
                            onCompletion(err, null);
			} else {
                            onCompletion(null, value);
                    }
                    });
		} else {
                    onCompletion(new AlreadyExistsError(type, id), null);
		}
            });
	}
    });
};

DiskDatabank.prototype.update = function(type, id, value, onCompletion) {
    var filename = this.toFilename(type, id);

    fs.stat(filename, function(err, stats) {
        if (err && err.code == 'ENOENT') {
            onCompletion(new NoSuchThingError(type, id), null);
        } else {
            fs.writeFile(filename, JSON.stringify(value), 'utf8', function(err) {
                if (err) {
                    onCompletion(err, null);
                } else {
                    onCompletion(null, value);
                }
            });
        }
    });
};

DiskDatabank.prototype.read = function(type, id, onCompletion) {
    var filename = this.toFilename(type, id);

    fs.stat(filename, function(err, stats) {
        if (err && err.code == 'ENOENT') {
            onCompletion(new NoSuchThingError(type, id), null);
        } else {
            fs.readFile(filename, function(err, data) {
                if (err) {
                    onCompletion(err, null);
                } else {
                    onCompletion(null, JSON.parse(data));
                }
            });
        }
    });
};

DiskDatabank.prototype.del = function(type, id, onCompletion) {
    var filename = this.toFilename(type, id);
    fs.unlink(filename, function(err) {
        if (err) {
            if (err.code == 'ENOENT') {
                onCompletion(new NoSuchThingError(type, id), null);
            } else {
                onCompletion(err, null);
            }
        } else {
            onCompletion(null);
        }
    });
};

DiskDatabank.prototype.ensureDir = function(dir, onCompletion) {

    var lastSlash = dir.lastIndexOf('/'),
        parent = dir.substr(0, lastSlash),
        base = dir.substr(lastSlash + 1),
        bank = this,
        ensureOne = function(dir, onCompletion) {
            fs.stat(dir, function(err, stats) {
                if (err) {
                    if (err.code == 'ENOENT') {
                        fs.mkdir(dir, bank.mode, function(err) {
                            if (err) {
                                onCompletion(err);
                            } else {
                                onCompletion(null);
                            }
                        });
                    } else {
                        onCompletion(err);
                    }
                } else {
                    onCompletion(null);
                }
            });
        };

    if (parent) {
        // recursively ensure parent
        this.ensureDir(parent, function(err) {
            if (err) {
                onCompletion(err);
            } else {
                ensureOne(dir, onCompletion);<commit_msg>more tolerant of existence of directory<commit_after>// diskdatabank.js
//
// On-disk implementation of Databank interface
//
// Copyright 2011, StatusNet Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

var databank = require('../databank'),
    Databank = databank.Databank,
    DatabankError = databank.DatabankError,
    AlreadyExistsError = databank.AlreadyExistsError,
    NoSuchThingError = databank.NoSuchThingError,
    crypto = require('crypto'),
    fs = require('fs'),
    path = require('path');

var DiskDatabank = function(params) {
    this.dir = params.dir || '/var/lib/diskdatabank/';
    this.mode = params.mode || 0755;
    this.hashDepth = params.hashDepth || 3;
    this.schema = params.schema || {};
};

DiskDatabank.prototype = new Databank();

DiskDatabank.prototype.toFilename = function(type, id) {
    var hash = this.toHash(id);

    return path.join(this.toDirname(type, id), hash + '.json');
};

DiskDatabank.prototype.toHash = function(id) {

    var hash = crypto.createHash('md5'),
        str, data;

    data = "" + id;

    hash.update(data);
    str = hash.digest('base64');

    // Make it a little more FS-safe

    str = str.replace(/\+/g, '-');
    str = str.replace(/\//g, '_');
    str = str.replace(/=/g, '');

    return str;
};

DiskDatabank.prototype.toDirname = function(type, id) {
    var n;
    var dirname = path.join(this.dir, type);
    var hash = this.toHash(id);

    for (n = 0; n < Math.min(this.hashDepth, hash.length); n++) {
        dirname = path.join(dirname, hash.substr(0, n + 1));
    }

    return dirname;
};

DiskDatabank.prototype.connect = function(params, onCompletion) {
    this.ensureDir(this.dir, onCompletion);
};

DiskDatabank.prototype.disconnect = function(onCompletion) {
    // Always succeed
    onCompletion(null);
};

DiskDatabank.prototype.create = function(type, id, value, onCompletion) {
    var dirname = this.toDirname(type, id),
        filename = this.toFilename(type, id);

    this.ensureDir(dirname, function(err) {
	if (err) {
	    onCompletion(err, null);
	} else {
            fs.stat(filename, function(err, stats) {
		if (err && err.code == 'ENOENT') {
                    fs.writeFile(filename, JSON.stringify(value), 'utf8', function(err) {
			if (err) {
                            onCompletion(err, null);
			} else {
                            onCompletion(null, value);
                    }
                    });
		} else {
                    onCompletion(new AlreadyExistsError(type, id), null);
		}
            });
	}
    });
};

DiskDatabank.prototype.update = function(type, id, value, onCompletion) {
    var filename = this.toFilename(type, id);

    fs.stat(filename, function(err, stats) {
        if (err && err.code == 'ENOENT') {
            onCompletion(new NoSuchThingError(type, id), null);
        } else {
            fs.writeFile(filename, JSON.stringify(value), 'utf8', function(err) {
                if (err) {
                    onCompletion(err, null);
                } else {
                    onCompletion(null, value);
                }
            });
        }
    });
};

DiskDatabank.prototype.read = function(type, id, onCompletion) {
    var filename = this.toFilename(type, id);

    fs.stat(filename, function(err, stats) {
        if (err && err.code == 'ENOENT') {
            onCompletion(new NoSuchThingError(type, id), null);
        } else {
            fs.readFile(filename, function(err, data) {
                if (err) {
                    onCompletion(err, null);
                } else {
                    onCompletion(null, JSON.parse(data));
                }
            });
        }
    });
};

DiskDatabank.prototype.del = function(type, id, onCompletion) {
    var filename = this.toFilename(type, id);
    fs.unlink(filename, function(err) {
        if (err) {
            if (err.code == 'ENOENT') {
                onCompletion(new NoSuchThingError(type, id), null);
            } else {
                onCompletion(err, null);
            }
        } else {
            onCompletion(null);
        }
    });
};

DiskDatabank.prototype.ensureDir = function(dir, onCompletion) {

    var lastSlash = dir.lastIndexOf('/'),
        parent = dir.substr(0, lastSlash),
        base = dir.substr(lastSlash + 1),
        bank = this,
        ensureOne = function(dir, onCompletion) {
            fs.stat(dir, function(err, stats) {
                if (err) {
                    if (err.code == 'ENOENT') {
                        fs.mkdir(dir, bank.mode, function(err) {
                            if (err && err.code == 'EEXIST') {
                                onCompletion(null);
                            } else if (err) {
                                onCompletion(err);
                            } else {
                                onCompletion(null);
                            }
                        });
                    } else {
                        onCompletion(err);
                    }
                } else {
                    onCompletion(null);
                }
            });
        };

    if (parent) {
        // recursively ensure parent
        this.ensureDir(parent, function(err) {
            if (err) {
                onCompletion(err);
            } else {
                ensureOne(dir, onCompletion);
{'fraction_non_alphanumeric': 0.1308038808038808, 'fraction_numerical': 0.003984753984753985, 'mean_word_length': 1.8165406196633325, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '190695', 'n_tokens_mistral': 3431, 'n_tokens_neox': 3272, 'n_words': 758}