<commit_before># No shebang line, this module is meant to be imported
#
# This file is part of PyFarm.
# Copyright (C) 2008-2012 <NAME>
#
# PyFarm is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyFarm is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PyFarm.  If not, see <http://www.gnu.org/licenses/>.

import uuid
import logging
import datetime
import sqlalchemy as sql

from pyfarm.preferences import prefs
from pyfarm.db.session import ENGINE
from pyfarm.datatypes.enums import State, DEFAULT_JOBTYPES, DEFAULT_GROUPS, DEFAULT_SOFTWARE

from twisted.python import log

HOSTNAME_LENGTH = 255
IPV4_LENGTH = 16

# create global metadata object and bind the engine
metadata = sql.MetaData()
metadata.bind = ENGINE

def getuuid():
    return str(uuid.uuid4())
# end getuuid

# HOSTS TABLE ATTRIBUTES
# hold - if True the given client cannot accept jobs until hold is False again
# frames - csv list of frames current running (from frames.c.id)
# cpus - number of cpus on the system
# online - True if we are able to reach and communicate with the host.  This
#          value can only be set if:
#           - the host shuts down
#           - an operation times out and we are unable to ping the host
#             after an exception is raised
# software - csv list of software that the host can run
# jobtypes - csv list of jobtypes that the host can run
# os - the operating system of the host
#      0 - linux
#      1 - mac
#      2 - window
#      3 - other/unknown
hosts = sql.Table('pyfarm_hosts', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('uuid', sql.String(36), default=getuuid),
    sql.Column('hostname', sql.String(HOSTNAME_LENGTH), nullable=False),
    sql.Column('port', sql.Integer, nullable=False),
    sql.Column('master', sql.String(HOSTNAME_LENGTH), default=None),
    sql.Column('ip', sql.String(IPV4_LENGTH)),
    sql.Column('subnet', sql.String(IPV4_LENGTH)),
    sql.Column('os', sql.Integer),
    sql.Column('ram_total', sql.Integer),
    sql.Column('ram_usage', sql.Integer),
    sql.Column('swap_total', sql.Integer),
    sql.Column('swap_usage', sql.Integer),
    sql.Column('cpu_count', sql.Integer),
    sql.Column('online', sql.Boolean, nullable=False, default=True),
    sql.Column('groups', sql.PickleType, default=DEFAULT_GROUPS),
    sql.Column('software', sql.PickleType, default=DEFAULT_SOFTWARE),
    sql.Column('jobtypes', sql.PickleType, default=DEFAULT_JOBTYPES)
)

# MASTERS TABLE ATTRIBUTES
# queue - queue management and accepting new jobs
# assignment - sending jobs to hosts when enabled
masters = sql.Table('pyfarm_masters', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('uuid', sql.String(36), default=getuuid),
    sql.Column('hostname', sql.String(HOSTNAME_LENGTH), nullable=False),
    sql.Column('port', sql.Integer, nullable=False),
    sql.Column('ip', sql.String(IPV4_LENGTH)),
    sql.Column('subnet', sql.String(IPV4_LENGTH)),
    sql.Column('online', sql.Boolean, default=False),
    sql.Column('queue', sql.Boolean, default=True),
    sql.Column('assignment', sql.Boolean, default=True)
)

# create jobs table
jobs = sql.Table('pyfarm_jobs', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('uuid', sql.String(36), default=getuuid),
    sql.Column('state', sql.Integer, default=0),
    sql.Column('priority', sql.Integer, default=prefs.get('jobsystem.priority-default')),

    # frame range declaration
    sql.Column('start_frame', sql.Float),
    sql.Column('end_frame', sql.Float),
    sql.Column('by_frame', sql.Float),

    # frame statistics
    sql.Column('count_success', sql.Integer, default=0),
    sql.Column('count_failed', sql.Integer, default=0),
    sql.Column('count_running', sql.Integer, default=0),
    sql.Column('count_total', sql.Integer, default=0),
    sql.Column('frame_longest', sql.Float, default=0),
    sql.Column('frame_shortest', sql.Float, default=0),
    sql.Column('frame_average', sql.Float, default=0),

    # timers
    sql.Column('time_start', sql.DateTime, default=None),
    sql.Column('time_end', sql.DateTime, default=None),
    sql.Column('time_elapsed', sql.DateTime, default=None),
    sql.Column('time_submitted', sql.DateTime, default=datetime.datetime.now),

    # job setup
    # isolate - if True this job must run by itself (no other jobs on host)
    # cpus - number of cpus required to be free on the client
    # enviro - pickle of an environment dictionary
    sql.Column('notes', sql.String(512, convert_unicode=True), default=""),
    sql.Column('enviro', sql.PickleType),
    sql.Column('data', sql.PickleType),
    sql.Column('user', sql.String(256)),
    sql.Column('software', sql.PickleType, default=None),
    sql.Column('jobtype', sql.String(64)),
    sql.Column('ram', sql.Integer, default=None),
    sql.Column('cpus', sql.Integer, default=None),
    sql.Column('requeue_failed', sql.Boolean,
               default=prefs.get('jobtypes.defaults.requeue-failed')),
    sql.Column('requeue_max', sql.Integer,
               default=prefs.get('jobtypes.defaults.requeue-max'))
)

# create frames table
# uuid - uuid of job on client
frames = sql.Table('pyfarm_frames', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('uuid', sql.String(36), default=getuuid),
    sql.Column('jobid', sql.Integer, sql.ForeignKey(jobs.c.id)),
    sql.Column('priority', sql.Integer, default=prefs.get('jobsystem.priority-default')),
    sql.Column('assigned_by', sql.Integer, sql.ForeignKey(masters.c.id), default=None),
    sql.Column('hostname', sql.Integer, sql.ForeignKey(hosts.c.id), default=None),
    sql.Column('frame', sql.Float),
    sql.Column('order', sql.Integer, default=0),
    sql.Column('state', sql.Integer, default=State.QUEUED),
    sql.Column('attempts', sql.Integer, default=0),
    sql.Column('ram', sql.Integer, default=None),
    sql.Column('time_start', sql.DateTime, default=None),
    sql.Column('time_end', sql.DateTime, default=None),
    sql.Column('time_submitted', sql.DateTime, default=datetime.datetime.now),
    sql.Column('dependencies', sql.PickleType, default=[])
)

# scratch table used for various testing
foobar = sql.Table('pyfarm_foobar', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
    sql.Column('bool', sql.Boolean, default=False),
    sql.Column('int', sql.Integer),
    sql.Column('string', sql.String(36))
)

def init(rebuild=False):
    '''initializes the tables according the the preferences'''
    if rebuild or prefs.get('database.setup.rebuild'):
        log.msg('dropping all tables before rebuilding', level=logging.WARNING)
        metadata.drop_all()

    metadata.create_all()
# end init<commit_msg>commenting out uuid for now<commit_after># No shebang line, this module is meant to be imported
#
# This file is part of PyFarm.
# Copyright (C) 2008-2012 <NAME>
#
# PyFarm is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# PyFarm is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with PyFarm.  If not, see <http://www.gnu.org/licenses/>.

import uuid
import logging
import datetime
import sqlalchemy as sql

from pyfarm.preferences import prefs
from pyfarm.db.session import ENGINE
from pyfarm.datatypes.enums import State, DEFAULT_JOBTYPES, DEFAULT_GROUPS, DEFAULT_SOFTWARE

from twisted.python import log

HOSTNAME_LENGTH = 255
IPV4_LENGTH = 16

# create global metadata object and bind the engine
metadata = sql.MetaData()
metadata.bind = ENGINE

#def getuuid():
#    '''generates the string representation of a uuid for use in columns'''
#    return str(uuid.uuid4())
## end getuuid


# HOSTS TABLE ATTRIBUTES
# hold - if True the given client cannot accept jobs until hold is False again
# frames - csv list of frames current running (from frames.c.id)
# cpus - number of cpus on the system
# online - True if we are able to reach and communicate with the host.  This
#          value can only be set if:
#           - the host shuts down
#           - an operation times out and we are unable to ping the host
#             after an exception is raised
# software - csv list of software that the host can run
# jobtypes - csv list of jobtypes that the host can run
# os - the operating system of the host
#      0 - linux
#      1 - mac
#      2 - window
#      3 - other/unknown
hosts = sql.Table('pyfarm_hosts', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
#    sql.Column('uuid', sql.String(36), default=getuuid),
    sql.Column('hostname', sql.String(HOSTNAME_LENGTH), nullable=False),
    sql.Column('port', sql.Integer, nullable=False),
    sql.Column('master', sql.String(HOSTNAME_LENGTH), default=None),
    sql.Column('ip', sql.String(IPV4_LENGTH)),
    sql.Column('subnet', sql.String(IPV4_LENGTH)),
    sql.Column('os', sql.Integer),
    sql.Column('ram_total', sql.Integer),
    sql.Column('ram_usage', sql.Integer),
    sql.Column('swap_total', sql.Integer),
    sql.Column('swap_usage', sql.Integer),
    sql.Column('cpu_count', sql.Integer),
    sql.Column('online', sql.Boolean, nullable=False, default=True),
    sql.Column('groups', sql.PickleType, default=DEFAULT_GROUPS),
    sql.Column('software', sql.PickleType, default=DEFAULT_SOFTWARE),
    sql.Column('jobtypes', sql.PickleType, default=DEFAULT_JOBTYPES)
)

# MASTERS TABLE ATTRIBUTES
# queue - queue management and accepting new jobs
# assignment - sending jobs to hosts when enabled
masters = sql.Table('pyfarm_masters', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
#    sql.Column('uuid', sql.String(36), default=getuuid),
    sql.Column('hostname', sql.String(HOSTNAME_LENGTH), nullable=False),
    sql.Column('port', sql.Integer, nullable=False),
    sql.Column('ip', sql.String(IPV4_LENGTH)),
    sql.Column('subnet', sql.String(IPV4_LENGTH)),
    sql.Column('online', sql.Boolean, default=False),
    sql.Column('queue', sql.Boolean, default=True),
    sql.Column('assignment', sql.Boolean, default=True)
)

# create jobs table
jobs = sql.Table('pyfarm_jobs', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
#    sql.Column('uuid', sql.String(36), default=getuuid),
    sql.Column('state', sql.Integer, default=0),
    sql.Column('priority', sql.Integer, default=prefs.get('jobsystem.priority-default')),

    # frame range declaration
    sql.Column('start_frame', sql.Float),
    sql.Column('end_frame', sql.Float),
    sql.Column('by_frame', sql.Float),

    # frame statistics
    sql.Column('count_success', sql.Integer, default=0),
    sql.Column('count_failed', sql.Integer, default=0),
    sql.Column('count_running', sql.Integer, default=0),
    sql.Column('count_total', sql.Integer, default=0),
    sql.Column('frame_longest', sql.Float, default=0),
    sql.Column('frame_shortest', sql.Float, default=0),
    sql.Column('frame_average', sql.Float, default=0),

    # timers
    sql.Column('time_start', sql.DateTime, default=None),
    sql.Column('time_end', sql.DateTime, default=None),
    sql.Column('time_elapsed', sql.DateTime, default=None),
    sql.Column('time_submitted', sql.DateTime, default=datetime.datetime.now),

    # job setup
    # isolate - if True this job must run by itself (no other jobs on host)
    # cpus - number of cpus required to be free on the client
    # enviro - pickle of an environment dictionary
    sql.Column('notes', sql.String(512, convert_unicode=True), default=""),
    sql.Column('enviro', sql.PickleType),
    sql.Column('data', sql.PickleType),
    sql.Column('user', sql.String(256)),
    sql.Column('software', sql.PickleType, default=None),
    sql.Column('jobtype', sql.String(64)),
    sql.Column('ram', sql.Integer, default=None),
    sql.Column('cpus', sql.Integer, default=None),
    sql.Column('requeue_failed', sql.Boolean,
               default=prefs.get('jobtypes.defaults.requeue-failed')),
    sql.Column('requeue_max', sql.Integer,
               default=prefs.get('jobtypes.defaults.requeue-max'))
)

# create frames table
# uuid - uuid of job on client
frames = sql.Table('pyfarm_frames', metadata,
    sql.Column('id', sql.Integer, autoincrement=True, primary_key=True),
#    sql.Column('uuid', sql.String(36), default=getuuid),
    sql.Column('jobid', sql.Integer, sql.ForeignKey(jobs.c.id)),
    sql.Column('priority', sql.Integer, default=prefs.get('jobsystem.priority-default')),
    sql.Column('assigned_by', sql.Integer, sql.ForeignKey(masters.c.id), default=None),
    sql.Column('hostname', sql.Integer, sql.ForeignKey(hosts.c.id), default=None),
    sql.Column('frame', sql.Float),
    sql.Column('order', sql.Integer, default=0),
    sql.Column('state', sql.Integer, default=State.QUEUED),
    sql.Column('attempts', sql.Integer, default=0),
    sql.Column('ram', sql.Integer, default=None),
    sql.Column('time_start', sql.DateTime, default=None),
    sql.Column('time_end', sql.DateTime, default=None),
    sql.Column('time_submitted', sql.DateTime, default=datetime.datetime.now),
    sql.Column('dependencies', sql.PickleType, default=[])
)


def init(rebuild=False):
    '''initializes the tables according the the preferences'''
    if rebuild or prefs.get('database.setup.rebuild'):
        log.msg('dropping all tables before rebuilding', level=logging.WARNING)
        metadata.drop_all()

    metadata.create_all()
# end init
{'fraction_non_alphanumeric': 0.13574660633484162, 'fraction_numerical': 0.007100591715976331, 'mean_word_length': 4.2719266055045875, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2632176', 'n_tokens_mistral': 4486, 'n_tokens_neox': 4384, 'n_words': 1196}