<commit_before>/**
 * Created by tigran on 1/30/15.
 */

mongoose = require("mongoose");
require("./model/city");
var config = require("./config")
    , options = { server: { socketOptions: { keepAlive: 1, connectTimeoutMS: 10 } }};

var city_json = require("./geonames_dump.json")
    , async = require("async");

mongoose.connection.on("error",function(err) {
    console.log(err);
    console.log("Hello from error event! ")
});

function GetField(model, id) {
    for(var i in city_json)
    {
        if(city_json[i]["model"] != model) continue;
        if(city_json[i]["pk"] == id) return city_json[i];
    }
    return false;
}

var db = mongoose.connect(config.MONGO_CONNECTION_STRING, function (error, res) {
    if (error) {
        console.log(error);
    }
});

mongoose.connection.on("open",function(err) {
    var cities = [], tmp = [], k=0;
    for(var i=0; i<= 20; i++)
    {
        k++;
        if(city_json[k]["model"] != "cities.city") {
            i--;
            continue;
        }
        tmp.push(city_json[i]);
        console.log(tmp[i]);
    }
    async.forEach(tmp, function (cc, callback){
        if(cc["model"] != "cities.city") {
            callback();
            return;
        }
        var region = GetField("cities.region", cc["fields"]["region"]);
        if(!region || !("fields" in region && "country" in region["fields"]) || !("fields" in region && "name" in region["fields"])) {
            callback();
            return;
        }
        var country = GetField("cities.country", region["fields"]["country"]);
        if(!country || !("fields" in country && "name" in region["fields"])) {
            callback();
            return;
        }
        var locs = cc["fields"]["location"].replace("POINT (", "").replace(")", "").split(" ");

        if(!region || !country) {
            callback();
            return;
        }
        var City = mongoose.model('city');
        var c = new City({
            region: region["fields"]["name"],
            country: country["fields"]["name"],
            population: cc["fields"]["population"],
            slug: cc["fields"]["slug"],
            name: cc["fields"]["name"],
            lat: locs[0],
            lng: locs[1]
        });

        cities.push(c);
        console.log(c.name);
        callback(); // tell async that the iterator has completed
    }, function(err) {
        var save = function() {
            var c = cities.pop();
            if (typeof c == "undefined") return;
            c.save(function (err) {<commit_msg>clear some comments<commit_after>/**
 * Created by tigran on 1/30/15.
 */

mongoose = require("mongoose");
require("./model/city");
var config = require("./config")
    , options = { server: { socketOptions: { keepAlive: 1, connectTimeoutMS: 10 } }};

var city_json = require("./geonames_dump.json")
    , async = require("async");

mongoose.connection.on("error",function(err) {
    console.log(err);
    console.log("Hello from error event! ")
});

function GetField(model, id) {
    for(var i in city_json)
    {
        if(city_json[i]["model"] != model) continue;
        if(city_json[i]["pk"] == id) return city_json[i];
    }
    return false;
}

var db = mongoose.connect(config.MONGO_CONNECTION_STRING, function (error, res) {
    if (error) {
        console.log(error);
    }
});

mongoose.connection.on("open",function(err) {
    var cities = [], tmp = [], k=0;
    for(var i=0; i<= 20; i++)
    {
        k++;
        if(city_json[k]["model"] != "cities.city") {
            i--;
            continue;
        }
        tmp.push(city_json[i]);
        console.log(tmp[i]);
    }
    async.forEach(tmp, function (cc, callback){
        if(cc["model"] != "cities.city") {
            callback();
            return;
        }
        var region = GetField("cities.region", cc["fields"]["region"]);
        if(!region || !("fields" in region && "country" in region["fields"]) || !("fields" in region && "name" in region["fields"])) {
            callback();
            return;
        }
        var country = GetField("cities.country", region["fields"]["country"]);
        if(!country || !("fields" in country && "name" in region["fields"])) {
            callback();
            return;
        }
        var locs = cc["fields"]["location"].replace("POINT (", "").replace(")", "").split(" ");

        if(!region || !country) {
            callback();
            return;
        }
        var City = mongoose.model('city');
        var c = new City({
            region: region["fields"]["name"],
            country: country["fields"]["name"],
            population: cc["fields"]["population"],
            slug: cc["fields"]["slug"],
            name: cc["fields"]["name"],
            lat: locs[0],
            lng: locs[1]
        });

        cities.push(c);
        console.log(c.name);
        callback();
    }, function(err) {
        var save = function() {
            var c = cities.pop();
            if (typeof c == "undefined") return;
            c.save(function (err) {
{'fraction_non_alphanumeric': 0.18841729472431576, 'fraction_numerical': 0.005553351844506148, 'mean_word_length': 2.1697045883092394, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6670188', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1558, 'n_words': 327}