# Path: config.py
# CERTITUDE_DATABASE = "sqlite:///data.db"
# 
# SLEEP = 5 # second interval between database poll
# 
# MIN_RESCAN_INTERVAL = 300 # min second interval between two consecutive scans on same IP address
# 
# Path: config.py
# IOC_MODE = 'flat'           # flat | logic ## DO NOT USE "logic" for now !!!
# 
# IOC_KEEPFILES = False       # True | False
# 
# Path: config.py
# IOC_CONFIDENTIAL_DIRECTORY = 'DR_PLUS'
# 
# IOC_COMPONENT_ROOT = os.path.join('components','scanner')
# 
# IOC_TEMP_DIR = os.path.join('components','scanner','tmp')
# 
# Path: helpers/helpers.py
# def hashPassword(password):
# def checksum(data):
# def verifyPassword(p):
#     MIN = False
#     MAJ = False
#     NUM = False
#     SPEC = False
#             MIN = True
#             MAJ = True
#             NUM = True
#             SPEC = True
# 
# Path: helpers/misc_models.py
# class ConfigurationProfile(Base):
#     __tablename__ = 'configuration_profiles'
# 
#     id = Column(Integer, primary_key=True)
#     name = Column(String)
#     host_confidential = Column(Boolean, default=False)
#     ioc_list = Column(String)
#     yara_list = Column(String)
# 
# class WindowsCredential(Base):
#     __tablename__ = 'windows_crendentials'
# 
#     id = Column(Integer, primary_key=True)
#     domain = Column(String)
#     login = Column(String)
#     encrypted_password = Column(String)
# 
# class XMLIOC(Base):
#     __tablename__ = 'xmliocs'
# 
#     id = Column(Integer, primary_key=True)
#     name = Column(String)
#     date_added = Column(DateTime, default=datetime.now)
#     xml_content = Column(String)
# 
# class Batch(Base):
#     __tablename__ = 'batches'
# 
#     id = Column(Integer, primary_key = True)
#     name = Column(String)
#     configuration_profile_id = Column(Integer)
#     windows_credential_id = Column(Integer)
# 
# class GlobalConfig(Base):
#     __tablename__ = 'global_config'
# 
#     id = Column(Integer, primary_key=True)
#     key = Column(String)
#     value = Column(String)
# 
# class User(Base):
#     __tablename__ = 'users'
# 
#     id = Column(Integer, primary_key=True)
#     username = Column(String)
#     password = Column(String)
#     email = Column(String)
#     active = Column(Boolean)
#     encrypted_master_key = Column(String)
#     b64_kdf_salt = Column(String)
# 
# Path: helpers/results_models.py
# class Result(Base):
#     __tablename__ = 'resultats'
# 
#     id = Column(Integer, primary_key=True)
#     tache_id = Column(Integer, ForeignKey('queue.id'))
#     finished = Column(DateTime, default=datetime.now)
# 
#     up = Column(Boolean)
#     blocked = Column(Boolean)
#     ip = Column(String)
#     hostname = Column(String)
#     smbreachable = Column(Boolean)
# 
# class IOCDetection(Base):
#     __tablename__ = 'iocdetections'
# 
#     id = Column(Integer, primary_key=True)
#     result_id = Column(Integer, ForeignKey('resultats.id'))
#     resultat = relationship(Result,
#         backref='iocdetections',)
#     xmlioc_id = Column(Integer)
#     indicator_id = Column(String) # Length should be 62
#     indicator_data = Column(String)


import base64
import datetime
import getpass
import json
import logging
import os
import re
import subprocess
import sys
import socket
import time
import traceback
import uuid
import helpers.crypto as crypto
import helpers.hashscan_modules as scan_modules
import openioc.openiocparser as openiocparser
import openioc.ioc as ioc
import remotecmd
import shutil
from lxml import objectify
from optparse import OptionParser
from sqlalchemy import create_engine, or_, func
from sqlalchemy.orm import sessionmaker
from threading import Lock
from config import CERTITUDE_DATABASE, SLEEP, MIN_RESCAN_INTERVAL
from config import IOC_MODE, IOC_KEEPFILES
from config import IOC_CONFIDENTIAL_DIRECTORY, IOC_COMPONENT_ROOT, IOC_TEMP_DIR
from helpers.helpers import hashPassword, checksum, threadname
from helpers.misc_models import ConfigurationProfile, WindowsCredential, XMLIOC, Batch, GlobalConfig, User
from helpers.queue_models import Task
from helpers.results_models import Result, IOCDetection
    # Retrieves evaluators for current mode
    FLAT_MODE = (IOC_MODE == 'flat')
    allowedElements = {}
    evaluatorList = scan_modules.flatEvaluatorList if FLAT_MODE else scan_modules.logicEvaluatorList

    for name, classname in evaluatorList.items():
        allowedElements[name] = classname.evalList

    # Parse XML Ioc into IOC trees according to what we can do
    for xmlioc in allXmlIocs:

        content = base64.b64decode(xmlioc.xml_content)
        oip = openiocparser.OpenIOCParser(content, allowedElements, FLAT_MODE, fromString=True)
        oip.parse()
        iocTree = oip.getTree()

        # Trees may be stripped from non valid elements
        if iocTree is not None:
            treeByIoc[xmlioc.id] = {'name':xmlioc.name, 'tree':iocTree}

    # Each configuration profile has a set of trees
    tree_by_cp = {cpid: {i:treeByIoc[i] for i in ioclist} for (cpid, ioclist) in iocByConfigurationProfile.items()}

    halt = False
    task = None
    batchquery = None

    # Batch filtering
    if batch is not None:
        loggingscan.info('Filtering for batch "%s"' % batch)

# Path: config.py
# CERTITUDE_DATABASE = "sqlite:///data.db"
# 
# SLEEP = 5 # second interval between database poll
# 
# MIN_RESCAN_INTERVAL = 300 # min second interval between two consecutive scans on same IP address
# 
# Path: config.py
# IOC_MODE = 'flat'           # flat | logic ## DO NOT USE "logic" for now !!!
# 
# IOC_KEEPFILES = False       # True | False
# 
# Path: config.py
# IOC_CONFIDENTIAL_DIRECTORY = 'DR_PLUS'
# 
# IOC_COMPONENT_ROOT = os.path.join('components','scanner')
# 
# IOC_TEMP_DIR = os.path.join('components','scanner','tmp')
# 
# Path: helpers/helpers.py
# def hashPassword(password):
# def checksum(data):
# def verifyPassword(p):
#     MIN = False
#     MAJ = False
#     NUM = False
#     SPEC = False
#             MIN = True
#             MAJ = True
#             NUM = True
#             SPEC = True
# 
# Path: helpers/misc_models.py
# class ConfigurationProfile(Base):
#     __tablename__ = 'configuration_profiles'
# 
#     id = Column(Integer, primary_key=True)
#     name = Column(String)
#     host_confidential = Column(Boolean, default=False)
#     ioc_list = Column(String)
#     yara_list = Column(String)
# 
# class WindowsCredential(Base):
#     __tablename__ = 'windows_crendentials'
# 
#     id = Column(Integer, primary_key=True)
#     domain = Column(String)
#     login = Column(String)
#     encrypted_password = Column(String)
# 
# class XMLIOC(Base):
#     __tablename__ = 'xmliocs'
# 
#     id = Column(Integer, primary_key=True)
#     name = Column(String)
#     date_added = Column(DateTime, default=datetime.now)
#     xml_content = Column(String)
# 
# class Batch(Base):
#     __tablename__ = 'batches'
# 
#     id = Column(Integer, primary_key = True)
#     name = Column(String)
#     configuration_profile_id = Column(Integer)
#     windows_credential_id = Column(Integer)
# 
# class GlobalConfig(Base):
#     __tablename__ = 'global_config'
# 
#     id = Column(Integer, primary_key=True)
#     key = Column(String)
#     value = Column(String)
# 
# class User(Base):
#     __tablename__ = 'users'
# 
#     id = Column(Integer, primary_key=True)
#     username = Column(String)
#     password = Column(String)
#     email = Column(String)
#     active = Column(Boolean)
#     encrypted_master_key = Column(String)
#     b64_kdf_salt = Column(String)
# 
# Path: helpers/results_models.py
# class Result(Base):
#     __tablename__ = 'resultats'
# 
#     id = Column(Integer, primary_key=True)
#     tache_id = Column(Integer, ForeignKey('queue.id'))
#     finished = Column(DateTime, default=datetime.now)
# 
#     up = Column(Boolean)
#     blocked = Column(Boolean)
#     ip = Column(String)
#     hostname = Column(String)
#     smbreachable = Column(Boolean)
# 
# class IOCDetection(Base):
#     __tablename__ = 'iocdetections'
# 
#     id = Column(Integer, primary_key=True)
#     result_id = Column(Integer, ForeignKey('resultats.id'))
#     resultat = relationship(Result,
#         backref='iocdetections',)
#     xmlioc_id = Column(Integer)
#     indicator_id = Column(String) # Length should be 62
#     indicator_data = Column(String)


# Path: components/scanner/hashscan_queue.py
import base64
import datetime
import getpass
import json
import logging
import os
import re
import subprocess
import sys
import socket
import time
import traceback
import uuid
import helpers.crypto as crypto
import helpers.hashscan_modules as scan_modules
import openioc.openiocparser as openiocparser
import openioc.ioc as ioc
import remotecmd
import shutil
from lxml import objectify
from optparse import OptionParser
from sqlalchemy import create_engine, or_, func
from sqlalchemy.orm import sessionmaker
from threading import Lock
from config import CERTITUDE_DATABASE, SLEEP, MIN_RESCAN_INTERVAL
from config import IOC_MODE, IOC_KEEPFILES
from config import IOC_CONFIDENTIAL_DIRECTORY, IOC_COMPONENT_ROOT, IOC_TEMP_DIR
from helpers.helpers import hashPassword, checksum, threadname
from helpers.misc_models import ConfigurationProfile, WindowsCredential, XMLIOC, Batch, GlobalConfig, User
from helpers.queue_models import Task
from helpers.results_models import Result, IOCDetection

    # Retrieves evaluators for current mode
    FLAT_MODE = (IOC_MODE == 'flat')
    allowedElements = {}
    evaluatorList = scan_modules.flatEvaluatorList if FLAT_MODE else scan_modules.logicEvaluatorList

    for name, classname in evaluatorList.items():
        allowedElements[name] = classname.evalList

    # Parse XML Ioc into IOC trees according to what we can do
    for xmlioc in allXmlIocs:

        content = base64.b64decode(xmlioc.xml_content)
        oip = openiocparser.OpenIOCParser(content, allowedElements, FLAT_MODE, fromString=True)
        oip.parse()
        iocTree = oip.getTree()

        # Trees may be stripped from non valid elements
        if iocTree is not None:
            treeByIoc[xmlioc.id] = {'name':xmlioc.name, 'tree':iocTree}

    # Each configuration profile has a set of trees
    tree_by_cp = {cpid: {i:treeByIoc[i] for i in ioclist} for (cpid, ioclist) in iocByConfigurationProfile.items()}

    halt = False
    task = None
    batchquery = None

    # Batch filtering
    if batch is not None:
        loggingscan.info('Filtering for batch "%s"' % batch)

        batchquery = session.query(Batch).filter( Batch.name == batch).first()