<commit_before>"""
mod_upload Controllers
===================
In this module, we are trying to add functionality regarding upload
and finalizing of samples through html or ftp. Reporting issues along
with submitting sample
"""
import base64
import hashlib
import json
import os
import traceback
import magic
import mimetypes

import shutil
import requests

from flask import Blueprint, g, make_response, render_template, request,  redirect, url_for, flash
from werkzeug.utils import secure_filename

from decorators import template_renderer, get_menu_entries
from git import Repo, InvalidGitRepositoryError, GitCommandError
from mod_auth.controllers import login_required, check_access_rights
from mod_auth.models import Role, User
from mod_home.models import CCExtractorVersion
from mod_sample.models import Sample, ForbiddenExtension, ForbiddenMimeType, Issue
from mod_upload.forms import UploadForm, DeleteQueuedSampleForm, \
    FinishQueuedSampleForm
from .models import Upload, QueuedSample, UploadLog, FTPCredentials, Platform

mod_upload = Blueprint('upload', __name__)


class QueuedSampleNotFoundException(Exception):

    def __init__(self, message):
        Exception.__init__(self)
        self.message = message


@mod_upload.before_app_request
def before_app_request():
    g.menu_entries['upload'] = {
        'title': 'Upload',
        'icon': 'upload',
        'route': 'upload.index'
    }
    config_entries = get_menu_entries(
        g.user, 'Platform mgmt', 'cog', all_entries=[
            {'title': 'Upload manager', 'icon': 'upload', 'route':
                'upload.index_admin', 'access': [Role.admin]}
        ]
    )
    if 'config' in g.menu_entries and 'entries' in config_entries:
        g.menu_entries['config']['entries'] = config_entries['entries'] + g.menu_entries['config']['entries']
    else:
        g.menu_entries['config'] = config_entries


@mod_upload.errorhandler(QueuedSampleNotFoundException)
@template_renderer('upload/queued_sample_not_found.html', 404)
def not_found(error):
    return {
        'message': error.message
    }


@mod_upload.route('/')
@login_required
@template_renderer()
def index():
    return {
        'queue': QueuedSample.query.filter(QueuedSample.user_id == g.user.id).all(),
        'messages': UploadLog.query.filter(UploadLog.user_id == g.user.id).order_by(UploadLog.id.desc()).all()
    }


@mod_upload.route('/manage')
@login_required
@check_access_rights([Role.admin])
@template_renderer()
def index_admin():
    return {
        'queue': QueuedSample.query.all(),
        'messages': UploadLog.query.order_by(UploadLog.id.desc()).all()
    }


def make_github_issue(title, body=None, labels=None):
    """
    Create an issue on github.com using the given parameters.

    :param title: The issue title
    :type title: str
    :param body: The content in the body
    :type body: str
    :param labels: The labels to add to the issue
    :type labels: list[str]
    :return: void
    :rtype: void
    """
    from run import config
    # Our url to create issues via POST
    url = 'https://api.github.com/repos/{org}/{repo}/issues'.format(
        org=config.get('GITHUB_OWNER', ''),
        repo=config.get('GITHUB_REPOSITORY', '')
    )
    session = requests.Session()
    session.auth = (g.user.email, g.user.github_token)
    # Create an authenticated session to create the issue
    # Create our issue
    issue = {'title': title,
             'body': body,
             'labels': labels}
    # Add the issue to our repository
    r = session.post(url, json.dumps(issue))

    if r.status_code == 201:
        return r.content

    return 'ERROR'


@mod_upload.route('/ftp')
@login_required
@template_renderer()
def ftp_index():
    from run import config

    credentials = FTPCredentials.query.filter(FTPCredentials.user_id == g.user.id).first()

    if credentials is None:
        credentials = FTPCredentials(g.user.id)
        g.db.add(credentials)
        g.db.commit()

    return {
        'host': config.get('SERVER_NAME', ''),
        'port': config.get('FTP_PORT', ''),
        'username': credentials.user_name,
        'password': <PASSWORD>
    }


@mod_upload.route('/ftp/filezilla')
@login_required
def ftp_filezilla():
    from run import config

    credentials = FTPCredentials.query.filter(FTPCredentials.user_id == g.user.id).first()

    if credentials is None:
        credentials = FTPCredentials(g.user.id)
        g.db.add(credentials)
        g.db.commit()

    response = make_response(
        render_template(
            'upload/filezilla_template.xml',
            host=config.get('SERVER_NAME', ''),
            port=config.get('FTP_PORT', ''),
            username=credentials.user_name,
            password=<PASSWORD>(<PASSWORD>())
        )
    )
    response.headers['Content-Description'] = 'File Transfer'
    response.headers['Cache-Control'] = 'no-cache'
    response.headers['Content-Type'] = 'text/xml'
    response.headers['Content-Disposition'] = 'attachment; filename=FileZilla.xml'

    return response


@mod_upload.route('/new', methods=['GET', 'POST'])
@login_required
@template_renderer()
def upload():
    from run import config
    form = UploadForm()
    if form.validate_on_submit():
        # Process uploaded file
        uploaded_file = request.files[form.file.name]
        if uploaded_file:
            filename = secure_filename(uploaded_file.filename)
            temp_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TempFiles', filename)
            # Save to temporary location
            uploaded_file.save(temp_path)
            # Get hash and check if it's already been submitted
            file_hash = create_hash_for_sample(temp_path)<commit_msg>Add missing decode call<commit_after>"""
mod_upload Controllers
===================
In this module, we are trying to add functionality regarding upload
and finalizing of samples through html or ftp. Reporting issues along
with submitting sample
"""
import base64
import hashlib
import json
import os
import traceback
import magic
import mimetypes

import shutil
import requests

from flask import Blueprint, g, make_response, render_template, request,  redirect, url_for, flash
from werkzeug.utils import secure_filename

from decorators import template_renderer, get_menu_entries
from git import Repo, InvalidGitRepositoryError, GitCommandError
from mod_auth.controllers import login_required, check_access_rights
from mod_auth.models import Role, User
from mod_home.models import CCExtractorVersion
from mod_sample.models import Sample, ForbiddenExtension, ForbiddenMimeType, Issue
from mod_upload.forms import UploadForm, DeleteQueuedSampleForm, \
    FinishQueuedSampleForm
from .models import Upload, QueuedSample, UploadLog, FTPCredentials, Platform

mod_upload = Blueprint('upload', __name__)


class QueuedSampleNotFoundException(Exception):

    def __init__(self, message):
        Exception.__init__(self)
        self.message = message


@mod_upload.before_app_request
def before_app_request():
    g.menu_entries['upload'] = {
        'title': 'Upload',
        'icon': 'upload',
        'route': 'upload.index'
    }
    config_entries = get_menu_entries(
        g.user, 'Platform mgmt', 'cog', all_entries=[
            {'title': 'Upload manager', 'icon': 'upload', 'route':
                'upload.index_admin', 'access': [Role.admin]}
        ]
    )
    if 'config' in g.menu_entries and 'entries' in config_entries:
        g.menu_entries['config']['entries'] = config_entries['entries'] + g.menu_entries['config']['entries']
    else:
        g.menu_entries['config'] = config_entries


@mod_upload.errorhandler(QueuedSampleNotFoundException)
@template_renderer('upload/queued_sample_not_found.html', 404)
def not_found(error):
    return {
        'message': error.message
    }


@mod_upload.route('/')
@login_required
@template_renderer()
def index():
    return {
        'queue': QueuedSample.query.filter(QueuedSample.user_id == g.user.id).all(),
        'messages': UploadLog.query.filter(UploadLog.user_id == g.user.id).order_by(UploadLog.id.desc()).all()
    }


@mod_upload.route('/manage')
@login_required
@check_access_rights([Role.admin])
@template_renderer()
def index_admin():
    return {
        'queue': QueuedSample.query.all(),
        'messages': UploadLog.query.order_by(UploadLog.id.desc()).all()
    }


def make_github_issue(title, body=None, labels=None):
    """
    Create an issue on github.com using the given parameters.

    :param title: The issue title
    :type title: str
    :param body: The content in the body
    :type body: str
    :param labels: The labels to add to the issue
    :type labels: list[str]
    :return: void
    :rtype: void
    """
    from run import config
    # Our url to create issues via POST
    url = 'https://api.github.com/repos/{org}/{repo}/issues'.format(
        org=config.get('GITHUB_OWNER', ''),
        repo=config.get('GITHUB_REPOSITORY', '')
    )
    session = requests.Session()
    session.auth = (g.user.email, g.user.github_token)
    # Create an authenticated session to create the issue
    # Create our issue
    issue = {'title': title,
             'body': body,
             'labels': labels}
    # Add the issue to our repository
    r = session.post(url, json.dumps(issue))

    if r.status_code == 201:
        return r.content

    return 'ERROR'


@mod_upload.route('/ftp')
@login_required
@template_renderer()
def ftp_index():
    from run import config

    credentials = FTPCredentials.query.filter(FTPCredentials.user_id == g.user.id).first()

    if credentials is None:
        credentials = FTPCredentials(g.user.id)
        g.db.add(credentials)
        g.db.commit()

    return {
        'host': config.get('SERVER_NAME', ''),
        'port': config.get('FTP_PORT', ''),
        'username': credentials.user_name,
        'password': <PASSWORD>
    }


@mod_upload.route('/ftp/filezilla')
@login_required
def ftp_filezilla():
    from run import config

    credentials = FTPCredentials.query.filter(FTPCredentials.user_id == g.user.id).first()

    if credentials is None:
        credentials = FTPCredentials(g.user.id)
        g.db.add(credentials)
        g.db.commit()

    response = make_response(
        render_template(
            'upload/filezilla_template.xml',
            host=config.get('SERVER_NAME', ''),
            port=config.get('FTP_PORT', ''),
            username=credentials.user_name,
            password=<PASSWORD>(<PASSWORD>()).decode()
        )
    )
    response.headers['Content-Description'] = 'File Transfer'
    response.headers['Cache-Control'] = 'no-cache'
    response.headers['Content-Type'] = 'text/xml'
    response.headers['Content-Disposition'] = 'attachment; filename=FileZilla.xml'

    return response


@mod_upload.route('/new', methods=['GET', 'POST'])
@login_required
@template_renderer()
def upload():
    from run import config
    form = UploadForm()
    if form.validate_on_submit():
        # Process uploaded file
        uploaded_file = request.files[form.file.name]
        if uploaded_file:
            filename = secure_filename(uploaded_file.filename)
            temp_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TempFiles', filename)
            # Save to temporary location
            uploaded_file.save(temp_path)
            # Get hash and check if it's already been submitted
            file_hash = create_hash_for_sample(temp_path)
{'fraction_non_alphanumeric': 0.12051662448730256, 'fraction_numerical': 0.0013962823981150188, 'mean_word_length': 3.399232245681382, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3796374', 'n_tokens_mistral': 3466, 'n_tokens_neox': 3394, 'n_words': 806}