<commit_before># -*- coding: utf-8 -*-
"""
    Security Knowledge Framework is an expert system application 
    that uses OWASP Application Security Verification Standard, code examples,
    helps developers in pre-development and post-development.  
    Copyright (C) 2015  <NAME>, <NAME>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program. If not, see <http://www.gnu.org/licenses/>.
"""

import os, markdown, datetime, string, base64, re, sys, re, requests, mimetypes, smtplib
from OpenSSL import SSL, rand
from docx import Document
from BeautifulSoup import BeautifulSoup
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.shared import Inches
from functools import wraps 
from sqlite3 import dbapi2 as sqlite3
from flask.ext.bcrypt import Bcrypt
from flask import Flask, request, session, g, redirect, url_for, abort, \
     render_template, flash, Markup, make_response
     
     

# create the application
app = Flask(__name__)

"""Set up bcrypt for passwords encrypting"""
bcrypt = Bcrypt(app)

def add_response_headers(headers={}):
    """This decorator adds the headers passed in to the response"""
    def decorator(f):
        @wraps(f)
        def decorated_function(*args, **kwargs):
            resp = make_response(f(*args, **kwargs))
            h = resp.headers
            for header, value in headers.items():
                h[header] = value
            return resp
        return decorated_function
    return decorator

def security(f):
    """This decorator passes multiple security headers and checks log file to block users"""
    return add_response_headers({'X-Frame-Options': 'deny', 'X-XSS-Protection': '1', 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'no-store, no-cache','Strict-Transport-Security': 'max-age=16070400; includeSubDomains', 'Server': 'Security Knowledge Framework'})(f)

def check_token():
    """Checks the submitted CSRF token"""
    if not session.get('csrf_token') == request.form['csrf_token']:
        log("User supplied not valid CSRF token", "FAIL", "HIGH")
        session.clear()
        return abort(500)(f)

def generate_pass():
    chars = string.letters + string.digits + '+/'
    assert 256 % len(chars) == 0  # non-biased later modulo
    PWD_LEN = 12
    generated_pass = ''.join(chars[ord(c) % len(chars)] for c in os.urandom(PWD_LEN))
    return generated_pass

def random_token(tokenBytes):
    #Create random token
    rand.cleanup()
    Random_token_raw = rand.bytes(int(tokenBytes))
    Random_token = base64.b64encode(Random_token_raw)
    result = re.sub("==", "", Random_token)
    return result

def log(message, value, threat):
    """Create log file and write events triggerd by the user
    The variables: message can be everything, value contains FAIL or SUCCESS and threat LOW MEDIUM HIGH"""
    now = datetime.datetime.now()
    dateLog = now.strftime("%Y-%m")
    dateTime = now.strftime("%Y-%m-%d %H:%M") 
    ip = request.remote_addr
    try:
        file = open('logs/'+dateLog+'.txt', 'a+')
    except IOError:
        # If not exists, create the file
        file = open('logs/'+dateLog+'.txt', 'w+')
    file.write(dateTime +' '+ message +' ' + ' ' + value + ' ' + threat + ' ' +ip + "\r\n")
    file.close() 

def blockUsers():
    """Check the log file and based on the FAIL items block a user"""
    dateLog  = datetime.datetime.now().strftime("%Y-%m")
    count = 0
    try:
        read = open(os.path.join(app.root_path, 'logs/'+dateLog+'.txt'), 'a+')
    except IOError:
        # If not exists, create the file
        read = open(os.path.join(app.root_path, 'logs/'+dateLog+'.txt'), 'w+')
    for line in read:
        match = re.search('FAIL', line)
        # If-statement after search() tests if it succeeded
        if match:                      
            count += 1   
            str(count) 
            if count > 11:
                sys.exit('Due to to many FAILED logs in your logging file we have the suspicion your application has been under attack by hackers. Please check your log files to validate and take repercussions. After validation clear your log or simply change the FAIL items to another value.')            
                                
def valAlphaNum(value, countLevel):
    match = re.findall(r"[^ a-zA-Z0-9_.-]", value)
    if match:
        log("User supplied not an a-zA-Z0-9 value", "FAIL", "MEDIUM")
        countAttempts(countLevel)
        abort(406)
        return False
    else:
        return True

def valNum(value, countLevel):
    match = re.findall(r'[a-zA-Z_]', str(value))
    if match:
        log("malicious input found", "FAIL", "MEDIUM")
        countAttempts(countLevel)
        abort(406)
        return False
    else:
        return True
        
def encodeInput(html):
    """Encode evil chars..."""
    result = re.sub('"', "&quot;", html)
    result = re.sub("'", "&#39;", result)
    result = re.sub("&", "&amp;", result)
    result = re.sub("<", "&lt;", result)
    result = re.sub(">", "&gt;", result)
    log("User supplied input was encoded", "SUCCESS", "NULL")
    return result
    
#not tested yet, made draft did not needed it so far
def whiteList(allowed, input, countlevel):
    splitted = string.split(allowed, ',')
    bool = False
    for val in splitted:
        if val in input:
            bool = True
    if bool == False:
        log("User is tampering whitelist values", "FAIL", "HIGH")
        countAttempts(countLevel)
        abort(401)
    if bool == True:
        return bool


#secret key for flask internal session use
rand.cleanup()
secret_key = rand.bytes(512)

mimetypes.add_type('image/svg+xml', '.svg')
bindaddr = '127.0.0.1';

# Load default config and override config from an environment variable
# You can also replace password with static password:  PASSWORD='<PASSWORD>'
app.config.update(dict(
    DATABASE=os.path.join(app.root_path, 'skf.db'),
    DEBUG=False,
    SECRET_KEY=secret_key,
    SESSION_COOKIE_SECURE=True,
    SESSION_COOKIE_HTTPONLY = True
))

@app.context_processor
def inject_year():
    return dict(year=datetime.datetime.now().strftime("%Y"))

def connect_db():
    """Connects to the specific database."""
    rv = sqlite3.connect(app.config['DATABASE'])
    rv.row_factory = sqlite3.Row
    return rv


def init_db():
    """Initializes the database."""
    db = get_db()
    with app.open_resource('schema.sql') as f:
        db.cursor().executescript(f.read())
    db.commit()


@app.cli.command('initdb')
def initdb_command():
    """Creates the database tables."""
    init_db()
    print('Initialized the database.')


def get_db():
    """Opens a new database connection if there is none yet for the
    current application context.
    """
    if not hasattr(g, 'sqlite_db'):
        g.sqlite_db = connect_db()
    return g.sqlite_db

def get_filepaths(directory):
    """
    This function will generate the file names in a directory 
    tree by walking the tree either top-down or bottom-up. For each 
    directory in the tree rooted at directory top (including top itself), 
    it yields a 3-tuple (dirpath, dirnames, filenames).
    """
    file_paths = [] 
    for root, directories, files in os.walk(directory):
        for filename in files:
            filepath = os.path.join(root, filename)
            file_paths.append(filepath)
    return file_paths  

def get_num(x):
    """get numbers from a string"""
    return int(''.join(ele for ele in x if ele.isdigit()))

@app.teardown_appcontext
def close_db(error):
    """Closes the database again at the end of the request."""
    if hasattr(g, 'sqlite_db'):
        g.sqlite_db.close()

def check_version():
    try:
        r = requests.get("http://raw.githubusercontent.com/blabla1337/skf-flask/master/setup.py")
        items_remote = r.content.split(",") 
        version_remote = items_remote[1]
        version_remote = version_remote.replace(version_remote[:14], '')
        version_remote = version_remote[:-1]
        with open ("version.txt", "r") as myfile:
            version_local = myfile.read().replace('\n', '')

        if version_local == version_remote:
            return True
        else:
            return False
    except:
        return False

def get_version():
    with open ("version.txt", "r") as myfile:
        version_final = myfile.read().replace('\n', '')
    return version_final
        
def projects_functions_techlist():
    """get list of technology used for creating project functions"""
    if not session.get('logged_in'):
        abort(401)
    db = get_db()
    cur = db.execute('SELECT techID, techName, vulnID from techhacks ORDER BY techID DESC')
    entries = cur.fetchall()
    return entries 

@app.route('/')
@security
def show_landing():
    """show the loging page and set default code language"""
    rand.cleanup()
    csrf_token_raw = rand.bytes(128)
    csrf_token = base64.b64encode(csrf_token_raw)
    session['csrf_token'] = csrf_token
    session['code_lang'] = "php"

    return render_template('login.html', csrf_token=session['csrf_token'])

@app.route('/dashboard', methods=['GET'])
@security
def dashboard():
    """show the landing page"""
    if not session.get('logged_in'):
        log("User with no valid session tries access to page /dashboard", "FAIL", "HIGH")
        abort(401)
    permissions("read")
    version_check = check_version()
    version = get_version()
    return render_template('dashboard.html', version=version, version_check=version_check)

@app.route('/first-login', methods=['GET'])
@security
def first_login():
    version_check = check_version()
    version = get_version()
    return render_template('first-login.html', version=version, version_check=version_check)

"""create account for a user"""
@app.route('/create-account', methods=['GET', 'POST'])
@security
def create_account():
    """validate the login data for access dashboard page"""
    error = None
    db = get_db()
    db.commit()
    if request.method == 'POST':
        """Username, password, token, email from form"""
        token  = request.form['token']
        email  = request.form['email']
        password  = request.form['password']
        
        #hash the password with Bcrypt, does autosalt
        hashed = bcrypt.generate_password_hash(password)
      
        #Do DB query also check for access
        cur = db.execute('SELECT accessToken, userID from users where email=? AND accessToken=?',
                            [email, token])
        check = cur.fetchall()<commit_msg>Update skf.py<commit_after># -*- coding: utf-8 -*-
"""
    Security Knowledge Framework is an expert system application 
    that uses OWASP Application Security Verification Standard, code examples,
    helps developers in pre-development and post-development.  
    Copyright (C) 2015  <NAME>, <NAME>

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program. If not, see <http://www.gnu.org/licenses/>.
"""

import os, markdown, datetime, string, base64, re, sys, re, requests, mimetypes, smtplib
from OpenSSL import SSL, rand
from docx import Document
from BeautifulSoup import BeautifulSoup
from docx.enum.text import WD_ALIGN_PARAGRAPH
from docx.shared import Inches
from functools import wraps 
from sqlite3 import dbapi2 as sqlite3
from flask.ext.bcrypt import Bcrypt
from flask import Flask, request, session, g, redirect, url_for, abort, \
     render_template, flash, Markup, make_response
     
     

# create the application
app = Flask(__name__)

"""Set up bcrypt for passwords encrypting"""
bcrypt = Bcrypt(app)

def add_response_headers(headers={}):
    """This decorator adds the headers passed in to the response"""
    def decorator(f):
        @wraps(f)
        def decorated_function(*args, **kwargs):
            resp = make_response(f(*args, **kwargs))
            h = resp.headers
            for header, value in headers.items():
                h[header] = value
            return resp
        return decorated_function
    return decorator

def security(f):
    """This decorator passes multiple security headers and checks log file to block users"""
    return add_response_headers({'X-Frame-Options': 'deny', 'X-XSS-Protection': '1', 'X-Content-Type-Options': 'nosniff', 'Cache-Control': 'no-store, no-cache','Strict-Transport-Security': 'max-age=16070400; includeSubDomains', 'Server': 'Security Knowledge Framework'})(f)

def check_token():
    """Checks the submitted CSRF token"""
    if not session.get('csrf_token') == request.form['csrf_token']:
        log("User supplied not valid CSRF token", "FAIL", "HIGH")
        session.clear()
        return abort(500)(f)

def generate_pass():
    chars = string.letters + string.digits + '+/'
    assert 256 % len(chars) == 0  # non-biased later modulo
    PWD_LEN = 12
    generated_pass = ''.join(chars[ord(c) % len(chars)] for c in os.urandom(PWD_LEN))
    return generated_pass

def random_token(tokenBytes):
    #Create random token
    rand.cleanup()
    Random_token_raw = rand.bytes(int(tokenBytes))
    Random_token = base64.b64encode(Random_token_raw)
    result = re.sub("==", "", Random_token)
    return result

def log(message, value, threat):
    """Create log file and write events triggerd by the user
    The variables: message can be everything, value contains FAIL or SUCCESS and threat LOW MEDIUM HIGH"""
    now = datetime.datetime.now()
    dateLog = now.strftime("%Y-%m")
    dateTime = now.strftime("%Y-%m-%d %H:%M") 
    ip = request.remote_addr
    try:
        file = open('logs/'+dateLog+'.txt', 'a+')
    except IOError:
        # If not exists, create the file
        file = open('logs/'+dateLog+'.txt', 'w+')
    file.write(dateTime +' '+ message +' ' + ' ' + value + ' ' + threat + ' ' +ip + "\r\n")
    file.close() 

def blockUsers():
    """Check the log file and based on the FAIL items block a user"""
    dateLog  = datetime.datetime.now().strftime("%Y-%m")
    count = 0
    try:
        read = open(os.path.join(app.root_path, 'logs/'+dateLog+'.txt'), 'a+')
    except IOError:
        # If not exists, create the file
        read = open(os.path.join(app.root_path, 'logs/'+dateLog+'.txt'), 'w+')
    for line in read:
        match = re.search('FAIL', line)
        # If-statement after search() tests if it succeeded
        if match:                      
            count += 1   
            str(count) 
            if count > 11:
                sys.exit('Due to to many FAILED logs in your logging file we have the suspicion your application has been under attack by hackers. Please check your log files to validate and take repercussions. After validation clear your log or simply change the FAIL items to another value.')            
                                
def valAlphaNum(value, countLevel):
    match = re.findall(r"[^ a-zA-Z0-9_.-]", value)
    if match:
        log("User supplied not an a-zA-Z0-9 value", "FAIL", "MEDIUM")
        countAttempts(countLevel)
        abort(406)
        return False
    else:
        return True

def valNum(value, countLevel):
    match = re.findall(r'[a-zA-Z_]', str(value))
    if match:
        log("malicious input found", "FAIL", "MEDIUM")
        countAttempts(countLevel)
        abort(406)
        return False
    else:
        return True
        
def encodeInput(html):
    """Encode evil chars..."""
    result = re.sub('"', "&quot;", html)
    result = re.sub("'", "&#39;", result)
    result = re.sub("&", "&amp;", result)
    result = re.sub("<", "&lt;", result)
    result = re.sub(">", "&gt;", result)
    log("User supplied input was encoded", "SUCCESS", "NULL")
    return result
    
#not tested yet, made draft did not needed it so far
def whiteList(allowed, input, countlevel):
    splitted = string.split(allowed, ',')
    bool = False
    for val in splitted:
        if val in input:
            bool = True
    if bool == False:
        log("User is tampering whitelist values", "FAIL", "HIGH")
        countAttempts(countLevel)
        abort(401)
    if bool == True:
        return bool


#secret key for flask internal session use
rand.cleanup()
secret_key = rand.bytes(512)

mimetypes.add_type('image/svg+xml', '.svg')
bindaddr = '127.0.0.1';

# Load default config and override config from an environment variable
# You can also replace password with static password:  PASSWORD='<PASSWORD>'
app.config.update(dict(
    DATABASE=os.path.join(app.root_path, 'skf.db'),
    DEBUG=False,
    SECRET_KEY=secret_key,
    SESSION_COOKIE_SECURE=True,
    SESSION_COOKIE_HTTPONLY = True
))

@app.context_processor
def inject_year():
    return dict(year=datetime.datetime.now().strftime("%Y"))

def connect_db():
    """Connects to the specific database."""
    rv = sqlite3.connect(app.config['DATABASE'])
    rv.row_factory = sqlite3.Row
    return rv


def init_db():
    """Initializes the database."""
    db = get_db()
    with app.open_resource('schema.sql') as f:
        db.cursor().executescript(f.read())
    db.commit()


@app.cli.command('initdb')
def initdb_command():
    """Creates the database tables."""
    init_db()
    print('Initialized the database.')


def get_db():
    """Opens a new database connection if there is none yet for the
    current application context.
    """
    if not hasattr(g, 'sqlite_db'):
        g.sqlite_db = connect_db()
    return g.sqlite_db

def get_filepaths(directory):
    """
    This function will generate the file names in a directory 
    tree by walking the tree either top-down or bottom-up. For each 
    directory in the tree rooted at directory top (including top itself), 
    it yields a 3-tuple (dirpath, dirnames, filenames).
    """
    file_paths = [] 
    for root, directories, files in os.walk(directory):
        for filename in files:
            filepath = os.path.join(root, filename)
            file_paths.append(filepath)
    return file_paths  

def get_num(x):
    """get numbers from a string"""
    return int(''.join(ele for ele in x if ele.isdigit()))

@app.teardown_appcontext
def close_db(error):
    """Closes the database again at the end of the request."""
    if hasattr(g, 'sqlite_db'):
        g.sqlite_db.close()

def check_version():
    try:
        r = requests.get("http://raw.githubusercontent.com/blabla1337/skf-flask/master/setup.py")
        items_remote = r.content.split(",") 
        version_remote = items_remote[1]
        version_remote = version_remote.replace(version_remote[:14], '')
        version_remote = version_remote[:-1]
        with open ("version.txt", "r") as myfile:
            version_local = myfile.read().replace('\n', '')

        if version_local == version_remote:
            return True
        else:
            return False
    except:
        return False

def get_version():
    with open ("version.txt", "r") as myfile:
        version_final = myfile.read().replace('\n', '')
    return version_final
        
def projects_functions_techlist():
    """get list of technology used for creating project functions"""
    if not session.get('logged_in'):
        abort(401)
    db = get_db()
    cur = db.execute('SELECT techID, techName, vulnID from techhacks ORDER BY techID DESC')
    entries = cur.fetchall()
    return entries 

@app.route('/')
@security
def show_landing():
    """show the loging page and set default code language"""
    rand.cleanup()
    csrf_token_raw = rand.bytes(128)
    csrf_token = base64.b64encode(csrf_token_raw)
    session['csrf_token'] = csrf_token
    session['code_lang'] = "php"

    return render_template('login.html', csrf_token=session['csrf_token'])

@app.route('/dashboard', methods=['GET'])
@security
def dashboard():
    """show the landing page"""
    if not session.get('logged_in'):
        log("User with no valid session tries access to page /dashboard", "FAIL", "HIGH")
        abort(401)
    permissions("read")
    version_check = check_version()
    version = get_version()
    return render_template('dashboard.html', version=version, version_check=version_check)

@app.route('/first-login', methods=['GET'])
@security
def first_login():
    version_check = check_version()
    version = get_version()
    return render_template('first-login.html', version=version, version_check=version_check)

"""create account for a user"""
@app.route('/create-account', methods=['GET', 'POST'])
@security
def create_account():
    """validate the login data for access dashboard page"""
    error = None
    db = get_db()
    db.commit()
    if request.method == 'POST':
        """Username, password, token, email from form"""
        token  = request.form['token']
        email  = request.form['email']
        password  = request.form['password']
        
        #hash the password with Bcrypt, does autosalt
        hashed = bcrypt.generate_password_hash(password,14)
      
        #Do DB query also check for access
        cur = db.execute('SELECT accessToken, userID from users where email=? AND accessToken=?',
                            [email, token])
        check = cur.fetchall()
{'fraction_non_alphanumeric': 0.1213629575998914, 'fraction_numerical': 0.007783157608941581, 'mean_word_length': 3.0116173534216735, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5844857', 'n_tokens_mistral': 6726, 'n_tokens_neox': 6518, 'n_words': 2076}