<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2008-2010  <NAME>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.


from weboob.backends.aum.pages.base import PageBase
from weboob.tools.ordereddict import OrderedDict

from copy import deepcopy
from logging import warning
import re

class FieldBase:

    def __init__(self, key):
        self.key = key

    def put_value(self, d, value):
        raise NotImplementedError

class FieldString(FieldBase):
    def put_value(self, d, value):
        d[self.key] = unicode(value)

class FieldList(FieldBase):
    def put_value(self, d, value):
        d[self.key] = value.split(', ')

class FieldWideList(FieldBase):

    def put_value(self, d, value):
        d[self.key] += [value]

class FieldOld(FieldBase):
    regexp = re.compile(u'([0-9]+) ans( \(Née le  ([0-9]+) ([^ ]+) ([0-9]+)\))?')
    month2i = ['', 'janvier', u'février', 'mars', 'avril', 'mai', 'juin', 'juillet', u'août', 'septembre', 'octobre', 'novembre', u'décembre']

    def put_value(self, d, value):
        m = self.regexp.match(value)
        if not m:
            return

        d[self.key] = int(m.group(1))
        if not m.group(2):
            return

        try:
            d['birthday'] = (int(m.group(3)),
                             self.month2i.index(m.group(4)),
                             int(m.group(5)))
        except ValueError, e:
            print str(e)

class FieldLocation(FieldBase):
    location = re.compile('(.*) \(([0-9]{5})\), (.*)')

    def __init__(self):
        FieldBase.__init__(self, '')
    def put_value(self, d, value):
        # TODO: determine distance, or something like
        m = self.location.match(value)
        if m:
            d['location'] = m.group(1)
            d['zipcode'] = int(m.group(2))
            d['country'] = m.group(3)
        else:
            warning('Unable to parse the location "%s"' % value)
            d['location'] = unicode(value)<commit_msg>better parse location in aum<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2008-2010  <NAME>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.


from weboob.backends.aum.pages.base import PageBase
from weboob.tools.ordereddict import OrderedDict

from copy import deepcopy
from logging import warning
import re

class FieldBase:

    def __init__(self, key):
        self.key = key

    def put_value(self, d, value):
        raise NotImplementedError

class FieldString(FieldBase):
    def put_value(self, d, value):
        d[self.key] = unicode(value)

class FieldList(FieldBase):
    def put_value(self, d, value):
        d[self.key] = value.split(', ')

class FieldWideList(FieldBase):

    def put_value(self, d, value):
        d[self.key] += [value]

class FieldOld(FieldBase):
    regexp = re.compile(u'([0-9]+) ans( \(Née le  ([0-9]+) ([^ ]+) ([0-9]+)\))?')
    month2i = ['', 'janvier', u'février', 'mars', 'avril', 'mai', 'juin', 'juillet', u'août', 'septembre', 'octobre', 'novembre', u'décembre']

    def put_value(self, d, value):
        m = self.regexp.match(value)
        if not m:
            return

        d[self.key] = int(m.group(1))
        if not m.group(2):
            return

        try:
            d['birthday'] = (int(m.group(3)),
                             self.month2i.index(m.group(4)),
                             int(m.group(5)))
        except ValueError, e:
            print str(e)

class FieldLocation(FieldBase):
    location = re.compile('(?P<location>.+?)( \((?P<zipcode>[0-9]{5})\))?, (?P<country>.*)')

    def __init__(self):
        FieldBase.__init__(self, '')
    def put_value(self, d, value):
        # TODO: determine distance, or something like
        m = self.location.match(value)
        if m:
            for field in ('country', 'location', 'zipcode'):
                d[field] = m.groupdict().get(field)
        else:
            warning('Unable to parse the location "%s"' % value)
            d['location'] = unicode(value)
{'fraction_non_alphanumeric': 0.1349728892331526, 'fraction_numerical': 0.016072811773818745, 'mean_word_length': 2.6892857142857145, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3382396', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1598, 'n_words': 510}