<commit_before># -*- coding: UTF-8 -*-
## Copyright 2009-2010 <NAME>
## This file is part of the TimTools project.
## TimTools is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 3 of the License, or
## (at your option) any later version.
## TimTools is distributed in the hope that it will be useful, 
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
## GNU General Public License for more details.
## You should have received a copy of the GNU General Public License
## along with TimTools; if not, see <http://www.gnu.org/licenses/>.

"""

Performs a database reset and initial import of your TIM data. 
Mandatory argument is the path to your TIM data directory.

"""

import logging
logger = logging.getLogger(__name__)

import os
import sys
import datetime

from dateutil import parser as dateparser

from django.conf import settings
from django.core.management import call_command
from django.db import IntegrityError
from django.core.exceptions import ValidationError
from django.core.management.base import BaseCommand, CommandError

#~ from lino import lino_site
from lino.utils import dbfreader
from lino.utils import dblogger
#~ from lino import diag

from lino.modlib.contacts.utils import name2kw, street2kw, join_words
from lino.utils.instantiator import Instantiator

from lino.tools import resolve_model, obj2str
import lino

from lino.utils import confirm, iif
from lino.core.coretools import app_labels

Activity = resolve_model('dsbe.Activity')
Country = resolve_model('countries.Country')
City = resolve_model('countries.City')
Person = resolve_model('contacts.Person')
Company = resolve_model('contacts.Company')


def store(kw,**d):
    for k,v in d.items():
        #~ if v is not None:
        if v:
            kw[k] = v

#~ def convert_username(name):
    #~ return name.lower()
  
def convert_sex(v):
    if v in ('W','F'): return 'F'
    if v == 'M': return 'M'
    return None
      
def isolang(x):
    if x == 'K' : return 'et'
    if x == 'E' : return 'en'
    if x == 'D' : return 'de'
    if x == 'F' : return 'fr'
    if x == 'N' : return 'nl'
      
def is_company(data):
    # wer eine nationalregisternummer hat ist eine Person, selbst wenn er auch eine MWst-Nummer hat.
    if data.get('NB2',False):
        return False
    if data.get('NOTVA',False):
    #~ if data.get('NOTVA',False):
        return True
    return False
    
def store_date(row,obj,rowattr,objattr):
    v = row[rowattr]
    if v:
        if isinstance(v,basestring):
            v = dateparser.parse(v)
        setattr(obj,objattr,v)
      
      
def ADR_id(cIdAdr):
    assert len(cIdAdr) == 3
    #~ assert [cIdAdr:-3] == '000'
    try:
        return 199000+int(cIdAdr)
    except ValueError,e:
        return None

     
def country2kw(row,kw):
    # for both PAR and ADR
    
    if row.has_key('PROF'):
        activity = row['PROF']
        if activity:
            try:
                activity = Activity.objects.get(pk=activity)
            except Activity.DoesNotExist:
                activity = Activity(id=activity,name=activity)
                activity.save(force_insert=True)
            kw.update(activity=activity)
        
    country = row['PAYS']
    if country:
        try:
            country = Country.objects.get(short_code__exact=country)
        except Country.DoesNotExist:
            country = Country(isocode=country,name=country,short_code=country)
            country.save()
        kw.update(country=country)
    
    store(kw,
      phone=row['TEL'],
      fax=row['FAX'],
      email=row['EMAIL'],
      )
    
    zip_code = row['CP']
    if zip_code:
        kw.update(zip_code=zip_code)
        try:
            city = City.objects.get(
              country=country,
              zip_code__exact=zip_code,
              )
            kw.update(city=city)
        except City.DoesNotExist,e:
            city = City(zip_code=zip_code,name=zip_code,country=country)
            city.save()
            kw.update(city=city)
            #~ logger.warning("%s-%s : %s",row['PAYS'],row['CP'],e)
        except City.MultipleObjectsReturned,e:
            dblogger.warning("%s-%s : %s",row['PAYS'],row['CP'],e)
      
def pxs2person(row,person):
  
    kw = {}
    store(kw,
      card_number=row['CARDNUMBER'],
      birth_place=row['BIRTHPLACE'],
      sex=convert_sex(row['SEXE'])
    )
    for k,v in kw.items():
        setattr(person,k,v)
        
    person.is_active = iif(row['IDPRT']=='I',False,True)
    if row['IDPRT'] == 'S':
        person.is_cpas = True
    elif row['IDPRT'] == 'A':
        person.is_senior = True
        
    if row['IDMUT']:
        try:
            person.health_insurance = Company.objects.get(pk=ADR_id(row['IDMUT']))
        except ValueError,e:
            dblogger.warning(u"%s : invalid health_insurance %r",obj2str(person),row['IDMUT'])
        except Company.DoesNotExist,e:
            dblogger.warning(u"%s : health_insurance %s not found",obj2str(person),row['IDMUT'])
  
    if row['APOTHEKE']:
        try:
            person.pharmacy = Company.objects.get(pk=int(row['APOTHEKE']))
        except ValueError,e:
            dblogger.warning(u"%s : invalid pharmacy %r",obj2str(person),row['APOTHEKE'])
        except Company.DoesNotExist,e:
            dblogger.warning(u"%s : pharmacy %s not found",obj2str(person),row['APOTHEKE'])
            
    nat = row['NATIONALIT']
    if nat:
        try:
            country = Country.objects.get(short_code__exact=nat)
        except Country.DoesNotExist:
            country = Country(isocode=nat,name=nat,short_code=nat)
            country.save()
        person.nationality=country
        
    store_date(row,person,'GEBDAT','birth_date')
    store_date(row,person,'VALID1','valid_from')
    store_date(row,person,'VALID2','valid_until')
            
        
def try_full_clean(i):
    while True:
        try:
            i.full_clean()
        except ValidationError,e:
            if not hasattr(e, 'message_dict'):
                raise<commit_msg>20110106<commit_after># -*- coding: UTF-8 -*-
## Copyright 2009-2010 <NAME>
## This file is part of the TimTools project.
## TimTools is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 3 of the License, or
## (at your option) any later version.
## TimTools is distributed in the hope that it will be useful, 
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
## GNU General Public License for more details.
## You should have received a copy of the GNU General Public License
## along with TimTools; if not, see <http://www.gnu.org/licenses/>.

"""

Performs a database reset and initial import of your TIM data. 
Mandatory argument is the path to your TIM data directory.

"""

import logging
logger = logging.getLogger(__name__)

import os
import sys
import datetime

from dateutil import parser as dateparser

from django.conf import settings
from django.core.management import call_command
from django.db import IntegrityError
from django.core.exceptions import ValidationError
from django.core.management.base import BaseCommand, CommandError

#~ from lino import lino_site
from lino.utils import dbfreader
from lino.utils import dblogger
#~ from lino import diag

from lino.modlib.contacts.utils import name2kw, street2kw, join_words
from lino.utils.instantiator import Instantiator

from lino.tools import resolve_model, obj2str
import lino

from lino.utils import confirm, iif
from lino.core.coretools import app_labels

Activity = resolve_model('dsbe.Activity')
Country = resolve_model('countries.Country')
City = resolve_model('countries.City')
Person = resolve_model('contacts.Person')
Company = resolve_model('contacts.Company')


def store(kw,**d):
    for k,v in d.items():
        #~ if v is not None:
        if v:
            kw[k] = v

#~ def convert_username(name):
    #~ return name.lower()
  
def convert_sex(v):
    if v in ('W','F'): return 'F'
    if v == 'M': return 'M'
    return None
      
def isolang(x):
    if x == 'K' : return 'et'
    if x == 'E' : return 'en'
    if x == 'D' : return 'de'
    if x == 'F' : return 'fr'
    if x == 'N' : return 'nl'
      
def is_company(data):
    # wer eine nationalregisternummer hat ist eine Person, selbst wenn er auch eine MWst-Nummer hat.
    if data.get('NB2',False):
        return False
    if data.get('NOTVA',False):
    #~ if data.get('NOTVA',False):
        return True
    return False
    
def store_date(row,obj,rowattr,objattr):
    v = row[rowattr]
    if v:
        if isinstance(v,basestring):
            v = dateparser.parse(v)
        setattr(obj,objattr,v)
      
      
def ADR_id(cIdAdr):
    assert len(cIdAdr) == 3
    #~ assert [cIdAdr:-3] == '000'
    try:
        return 199000+int(cIdAdr)
    except ValueError,e:
        return None

     
def country2kw(row,kw):
    # for both PAR and ADR
    
    if row.has_key('PROF'):
        activity = row['PROF']
        if activity:
            try:
                activity = Activity.objects.get(pk=activity)
            except Activity.DoesNotExist:
                activity = Activity(id=activity,name=activity)
                activity.save(force_insert=True)
            kw.update(activity=activity)
        
    country = row['PAYS']
    if country:
        try:
            country = Country.objects.get(short_code__exact=country)
        except Country.DoesNotExist:
            country = Country(isocode=country,name=country,short_code=country)
            country.save()
        kw.update(country=country)
    
    store(kw,
      phone=row['TEL'],
      fax=row['FAX'],
      email=row['EMAIL'],
      )
    
    zip_code = row['CP']
    if zip_code:
        kw.update(zip_code=zip_code)
        try:
            city = City.objects.get(
              country=country,
              zip_code__exact=zip_code,
              )
            kw.update(city=city)
        except City.DoesNotExist,e:
            city = City(zip_code=zip_code,name=zip_code,country=country)
            city.save()
            kw.update(city=city)
            #~ logger.warning("%s-%s : %s",row['PAYS'],row['CP'],e)
        except City.MultipleObjectsReturned,e:
            dblogger.warning("%s-%s : %s",row['PAYS'],row['CP'],e)
      
def pxs2person(row,person):
  
    kw = {}
    store(kw,
      card_number=row['CARDNUMBER'],
      birth_place=row['BIRTHPLACE'],
      sex=convert_sex(row['SEXE'])
    )
    for k,v in kw.items():
        setattr(person,k,v)
        
    person.is_active = iif(row['IDPRT']=='I',False,True)
    if row['IDPRT'] == 'S':
        person.is_cpas = True
    elif row['IDPRT'] == 'A':
        person.is_senior = True
        
    if row['IDMUT']:
        try:
            person.health_insurance = Company.objects.get(pk=ADR_id(row['IDMUT']))
        except ValueError,e:
            dblogger.warning(u"%s : invalid health_insurance %r",obj2str(person),row['IDMUT'])
        except Company.DoesNotExist,e:
            dblogger.warning(u"%s : health_insurance %s not found",obj2str(person),row['IDMUT'])
  
    if row['APOTHEKE']:
        try:
            person.pharmacy = Company.objects.get(pk=int(row['APOTHEKE']))
        except ValueError,e:
            dblogger.warning(u"%s : invalid pharmacy %r",obj2str(person),row['APOTHEKE'])
        except Company.DoesNotExist,e:
            dblogger.warning(u"%s : pharmacy %s not found",obj2str(person),row['APOTHEKE'])
            
    nat = row['NATIONALIT']
    if nat:
        try:
            country = Country.objects.get(short_code__exact=nat)
        except Country.DoesNotExist:
            country = Country(isocode=nat,name=nat,short_code=nat)
            country.save()
        person.nationality=country
        
    store_date(row,person,'GEBDAT','birth_date')
    store_date(row,person,'VALID1','card_valid_from')
    store_date(row,person,'VALID2','card_valid_until')
            
        
def try_full_clean(i):
    while True:
        try:
            i.full_clean()
        except ValidationError,e:
            if not hasattr(e, 'message_dict'):
                raise
{'fraction_non_alphanumeric': 0.11792186870720903, 'fraction_numerical': 0.005960531614981877, 'mean_word_length': 2.6421237899677323, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7583583', 'n_tokens_mistral': 3970, 'n_tokens_neox': 3872, 'n_words': 971}