<commit_before>#!flask/bin/python
import os
import argparse
import datetime
from dateutil.tz import tzutc
import libtaxii as t
import libtaxii.messages as tm
import libtaxii.clients as tc
from sqlalchemy import create_engine, Column, ForeignKey
from sqlalchemy import Integer, String, Date, Text, Boolean
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relation, sessionmaker, class_mapper
from json import dumps
import warnings


PID_FILE = '/tmp/taxii_daemon.pid'
PROXY_ENABLED = True
PROXY_SCHEME = 'http'
PROXY_STRING = '127.0.0.1:8008'
CB_XML_112608 = 'http://www.w3.org/TR/xml'
#TAXII_SERVICE_HOST = 'misplab.cert.europa.eu'
#TAXII_SERVICE_PORT = 80
#TAXII_SERVICE_PATH = '/taxii/inbox'
TAXII_SERVICE_HOST = '127.0.0.1'
TAXII_SERVICE_PORT = 4242
TAXII_SERVICE_PATH = '/inbox'

# only required if you get data from a DB
Base = declarative_base()
db = create_engine('mysql://root@127.0.0.1:3309/CTI_db_test')
Session = sessionmaker(bind=db)


def is_process_running(pid):
    try:
        os.kill(pid, 0)
    except OSError:
        return
    else:
        return pid


def check_process(path):
    if os.path.exists(path):
        pid = int(open(path).read())
        if is_process_running(pid):
            print "Process %d is still running" % pid
            raise SystemExit
        else:
            os.remove(path)
    pid = os.getpid()
    open(path, 'w').write(str(pid))
    return pid


def serialize(model, exclude_fields=[]):
    warnings.warn('deprecated', DeprecationWarning)
    """Transforms a model into a dictionary which can be dumped to JSON."""
    columns = [c.key for c in class_mapper(model.__class__).columns if c not in exclude_fields]
    return dict((c, getattr(model, c).__str__()) if type(getattr(model, c)) is datetime.date
                else (c, getattr(model, c)) for c in columns)


def create_inbox_message(data, content_binding=CB_XML_112608):
    """Creates TAXII message from data"""
    xml_content_block1 = tm.ContentBlock(
        content_binding=content_binding,
        content=data,
        timestamp_label=datetime.datetime.now(tzutc()))

    msg_id = tm.generate_message_id()

    inbox_message = tm.InboxMessage(
        message_id=msg_id,
        content_blocks=[xml_content_block1])
    return msg_id, inbox_message.to_xml()


def load_db_data():
    session = Session()

    #serialized_events = [
    #    serialize(event)
    #    for event in session.query(Event).filter(Event.published == 1)
    #]
    #return dumps(serialized_events)
    events = session.query(Event).filter(Event.published == 1, Event.published != '').limit(1).all()
    all = [dumps(e.serialize) for e in events]
    #for x in all:
    #    print x


class Event(Base):
    __tablename__ = 'events'

    id = Column(Integer, primary_key=True, autoincrement=True)
    date = Column(Date, default=datetime.datetime.now())
    risk = Column(String)
    info = Column(Text)
    published = Column(Integer)
    uuid = Column(String)
    attributes = relation('Attribute', backref='event')

    @property
    def serialize(self):
        """Return object data in easily serializeable format"""
        return {
            self.uuid: {
                #'id': self.id,
                "date": self.date.__str__(),
                "risk": self.risk,
                "info": self.info,
                "published": self.published,
                "uuid": self.uuid,
                "attributes": self.serialize_attributes
            }
        }
<commit_msg>dumps<commit_after>#!flask/bin/python
import os
import argparse
import datetime
from dateutil.tz import tzutc
import libtaxii as t
import libtaxii.messages as tm
import libtaxii.clients as tc
from sqlalchemy import create_engine, Column, ForeignKey
from sqlalchemy import Integer, String, Date, Text, Boolean
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relation, sessionmaker, class_mapper
from json import dumps
import warnings
from flask import jsonify


PID_FILE = '/tmp/taxii_daemon.pid'
PROXY_ENABLED = True
PROXY_SCHEME = 'http'
PROXY_STRING = '127.0.0.1:8008'
CB_XML_112608 = 'http://www.w3.org/TR/xml'
#TAXII_SERVICE_HOST = 'misplab.cert.europa.eu'
#TAXII_SERVICE_PORT = 80
#TAXII_SERVICE_PATH = '/taxii/inbox'
TAXII_SERVICE_HOST = '127.0.0.1'
TAXII_SERVICE_PORT = 4242
TAXII_SERVICE_PATH = '/inbox'

# only required if you get data from a DB
Base = declarative_base()
db = create_engine('mysql://root@127.0.0.1:3309/CTI_db_test')
Session = sessionmaker(bind=db)


def is_process_running(pid):
    try:
        os.kill(pid, 0)
    except OSError:
        return
    else:
        return pid


def check_process(path):
    if os.path.exists(path):
        pid = int(open(path).read())
        if is_process_running(pid):
            print "Process %d is still running" % pid
            raise SystemExit
        else:
            os.remove(path)
    pid = os.getpid()
    open(path, 'w').write(str(pid))
    return pid


def serialize(model, exclude_fields=[]):
    warnings.warn('deprecated', DeprecationWarning)
    """Transforms a model into a dictionary which can be dumped to JSON."""
    columns = [c.key for c in class_mapper(model.__class__).columns if c not in exclude_fields]
    return dict((c, getattr(model, c).__str__()) if type(getattr(model, c)) is datetime.date
                else (c, getattr(model, c)) for c in columns)


def create_inbox_message(data, content_binding=CB_XML_112608):
    """Creates TAXII message from data"""
    xml_content_block1 = tm.ContentBlock(
        content_binding=content_binding,
        content=data,
        timestamp_label=datetime.datetime.now(tzutc()))

    msg_id = tm.generate_message_id()

    inbox_message = tm.InboxMessage(
        message_id=msg_id,
        content_blocks=[xml_content_block1])
    return msg_id, inbox_message.to_xml()


def load_db_data():
    session = Session()

    #serialized_events = [
    #    serialize(event)
    #    for event in session.query(Event).filter(Event.published == 1)
    #]
    #return dumps(serialized_events)
    events = session.query(Event).filter(Event.published == 1, Event.published != '').limit(2).all()
    all = [e.serialize for e in events]
    return dumps(all)

class Event(Base):
    __tablename__ = 'events'

    id = Column(Integer, primary_key=True, autoincrement=True)
    date = Column(Date, default=datetime.datetime.now())
    risk = Column(String)
    info = Column(Text)
    published = Column(Integer)
    uuid = Column(String)
    attributes = relation('Attribute', backref='event')

    @property
    def serialize(self):
        """Return object data in easily serializeable format"""
        return {
            self.uuid: {
                #'id': self.id,
                "date": self.date.__str__(),
                "risk": self.risk,
                "info": self.info,
                "published": self.published,
                "uuid": self.uuid,
                "attributes": self.serialize_attributes
            }
        }

{'fraction_non_alphanumeric': 0.11226522481502561, 'fraction_numerical': 0.014513375071143995, 'mean_word_length': 3.0771461716937356, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5347177', 'n_tokens_mistral': 2344, 'n_tokens_neox': 2259, 'n_words': 542}