<commit_before># Import the database object (db) from the main application module
from app import db, Base
from .utils import country_code
from utils.get_or_insert import get_or_insert

class View(Base):
    __tablename__ = 'view'

    # Object
    object = db.Column(db.Integer, db.ForeignKey('object.id',
                                                 ondelete='CASCADE',
                                                 onupdate='CASCADE'))

    # View Type
    type = db.Column(db.Enum('page', 'raw', 'download', 'redirect'), nullable=True)

    # Viewer
    ip = db.Column(db.String(39))
    country_code = db.Column(db.Integer, db.ForeignKey('country_code.id',
                                                       ondelete='CASCADE',
                                                       onupdate='CASCADE'))
    referrer = db.Column(db.Integer, db.ForeignKey('referrer.id',
                                                   ondelete='CASCADE',
                                                   onupdate='CASCADE'))
    user_agent = db.Column(db.Integer, db.ForeignKey('user_agent.id',
                                                     ondelete='CASCADE',
                                                     onupdate='CASCADE'))

    def __init__(self, object, type, ip, referrer, user_agent):
        self.object = object
        self.type = type
        self.ip = ip
        self.country_code = get_or_insert(CountryCode, CountryCode.country_code, country_code(ip)).id
        self.referrer = get_or_insert(Referrer, Referrer.referrer, referrer).id
        self.user_agent = get_or_insert(UserAgent, UserAgent.user_agent, user_agent).id

    def __repr__(self):
        return '<View %s/%s>' % (self.object, self.type)


class CountryCode(Base):
    __tablename__ = 'country_code'

    country_code = db.Column(db.String(2), nullable=True, unique=True)

    def __init__(self, country_code):
        self.country_code = country_code

    def __repr__(self):
        return '<country_code %s>' % (self.country_code)


class Referrer(Base):
    __tablename__ = 'referrer'

    referrer = db.Column(db.String(255), nullable=True, unique=True)

    def __init__(self, referrer):
        self.referrer = referrer

    def __repr__(self):
        return '<referrer "%s">' % (self.referrer)

<commit_msg>do not normalize empty values<commit_after># Import the database object (db) from the main application module
from app import db, Base
from .utils import country_code as country_code_from_ip
from utils.get_or_insert import get_or_insert

class View(Base):
    __tablename__ = 'view'

    # Object
    object = db.Column(db.Integer, db.ForeignKey('object.id',
                                                 ondelete='CASCADE',
                                                 onupdate='CASCADE'))

    # View Type
    type = db.Column(db.Enum('page', 'raw', 'download', 'redirect'), nullable=True)

    # Viewer
    ip = db.Column(db.String(39))
    country_code = db.Column(db.Integer, db.ForeignKey('country_code.id',
                                                       ondelete='CASCADE',
                                                       onupdate='CASCADE'))
    referrer = db.Column(db.Integer, db.ForeignKey('referrer.id',
                                                   ondelete='CASCADE',
                                                   onupdate='CASCADE'))
    user_agent = db.Column(db.Integer, db.ForeignKey('user_agent.id',
                                                     ondelete='CASCADE',
                                                     onupdate='CASCADE'))

    def __init__(self, object, type, ip, referrer, user_agent):
        self.object = object
        self.type = type
        self.ip = ip

        country_code = country_code_from_ip(ip)
        if country_code:
            self.country_code = get_or_insert(CountryCode, CountryCode.country_code, country_code).id
        else:
            self.country_code = None

        if referrer:
            self.referrer = get_or_insert(Referrer, Referrer.referrer, referrer).id
        else:
            self.referrer = None

        if user_agent:
            self.user_agent = get_or_insert(UserAgent, UserAgent.user_agent, user_agent).id
        else:
            self.user_agent = None


    def __repr__(self):
        return '<View %s/%s>' % (self.object, self.type)


class CountryCode(Base):
    __tablename__ = 'country_code'

    country_code = db.Column(db.String(2), nullable=True, unique=True)

    def __init__(self, country_code):
        self.country_code = country_code

    def __repr__(self):
        return '<country_code %s>' % (self.country_code)


class Referrer(Base):
    __tablename__ = 'referrer'

    referrer = db.Column(db.String(255), nullable=True, unique=True)

    def __init__(self, referrer):
        self.referrer = referrer

    def __repr__(self):
        return '<referrer "%s">' % (self.referrer)


{'fraction_non_alphanumeric': 0.10489086499595797, 'fraction_numerical': 0.002425222312045271, 'mean_word_length': 1.842619184376795, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7087022', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1366, 'n_words': 283}