<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@authors: <NAME> <EMAIL>
          <NAME>. <EMAIL>

@license:  GNU AFFERO GENERAL PUBLIC LICENSE

Caliope Server is the web server of Caliope's Framework
Copyright (C) 2013 Fundación Correlibre

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''
#system, and standard library
import os
import sys

from datetime import datetime

#Model imports
from cid.model.SIIMModel import RegistroPredioCatastroTipo2


def main(argv):
    if len(argv) is not 1:
        print "Usage import_predios.py predios.csv"
    else:
        print "Preparing to import from "
        print argv
        importPredios(argv[0])


def parseDateFromTwoDigitYear(dumbdate):
    day, month, year = map(int, dumbdate.split('/'))
    if year <= 13:
        year += 2000
    else:
        year += 1900
    return datetime(year, month, day)


def importPredios(filename):
    ins = open(filename, "r")
    line = ins.readline()
    header = map(lambda f: f.strip('\n').strip('"').lower(), line.split('|'))
    print header
    for line in ins:
        fields = map(lambda f: f.strip('\n').strip('"'),
                     line.replace(',', '.').split('|'))
        node = RegistroPredioCatastroTipo2()
        record = {}
        map(lambda k, v: record.update({k: v}), header, fields)
        map(lambda k, v: setattr(node, k, v), header, fields)
        node.fecha_documento = parseDateFromTwoDigitYear(record['fecha_documento'])
        node.save()


if __name__ == '__main__':
    main(sys.argv[1:])
<commit_msg>Added UniqueProperty exception for duplicated unique fields<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-
'''
@authors: <NAME> <EMAIL>
          <NAME>. <EMAIL>

@license:  GNU AFFERO GENERAL PUBLIC LICENSE

Caliope Server is the web server of Caliope's Framework
Copyright (C) 2013 Fundación Correlibre

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''
#system, and standard library
import os
import sys

from datetime import datetime

#neomodel exceptiosn

from neomodel.exception import UniqueProperty

#Model imports
from cid.model.SIIMModel import RegistroPredioCatastroTipo2


def main(argv):
    if len(argv) is not 1:
        print "Usage import_predios.py predios.csv"
    else:
        print "Preparing to import from "
        print argv
        importPredios(argv[0])


def parseDateFromTwoDigitYear(dumbdate):
    day, month, year = map(int, dumbdate.split('/'))
    if year <= 13:
        year += 2000
    else:
        year += 1900
    return datetime(year, month, day)


def importPredios(filename):
    ins = open(filename, "r")
    line = ins.readline()
    header = map(lambda f: f.strip('\n').strip('"').lower(), line.split('|'))
    print header
    for line in ins:    
        fields = map(lambda f: f.strip('\n').strip('"'),
                     line.replace(',', '.').split('|'))
        node = RegistroPredioCatastroTipo2()
        record = {}
        map(lambda k, v: record.update({k: v}), header, fields)
        map(lambda k, v: setattr(node, k, v), header, fields)
        node.fecha_documento = parseDateFromTwoDigitYear(record['fecha_documento'])
        try:
            node.save()
        except UniqueProperty:
            print record


if __name__ == '__main__':
    main(sys.argv[1:])

{'fraction_non_alphanumeric': 0.10387323943661972, 'fraction_numerical': 0.009242957746478873, 'mean_word_length': 2.9973614775725594, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4489507', 'n_tokens_mistral': 1381, 'n_tokens_neox': 1318, 'n_words': 491}