<commit_before>import flask as fk
from corrdb.common.core import setup_app
from corrdb.common.models import UserModel
from corrdb.common.models import ProjectModel
from corrdb.common.models import ApplicationModel
from corrdb.common.models import TrafficModel
from corrdb.common.models import StatModel  
from corrdb.common.models import AccessModel
import tarfile
from StringIO import StringIO
from io import BytesIO
import zipfile
import json
import time
import boto3
import traceback 
import datetime

import requests
from datetime import date, timedelta
from functools import update_wrapper
from calendar import monthrange
import time

app = setup_app(__name__)

s3 =  boto3.resource('s3')

S3_BUCKET = app.config['S3_BUCKET']

# Stormpath

from flask.ext.stormpath import StormpathManager

stormpath_manager = StormpathManager(app)

from datetime import date, timedelta
from functools import update_wrapper


def get_week_days(year, week):
    d = date(year,1,1)
    if(d.weekday()>3):
        d = d+timedelta(7-d.weekday())
    else:
        d = d - timedelta(d.weekday())
    dlt = timedelta(days = (week-1)*7)
    return d + dlt,  d + dlt + timedelta(days=6)

def find_week_days(year, current):
    index  = 1
    while True:
        if index == 360:
            break
        interval = get_week_days(year, index)
        if current > interval[0] and current < interval[1]:
            return interval
        index +=1

class InMemoryZip(object):
    def __init__(self):
        # Create the in-memory file-like object
        self.in_memory_zip = StringIO()

    def append(self, filename_in_zip, file_contents):
        '''Appends a file with name filename_in_zip and contents of 
        file_contents to the in-memory zip.'''
        # Get a handle to the in-memory zip in append mode
        zf = zipfile.ZipFile(self.in_memory_zip, "a", zipfile.ZIP_DEFLATED, False)

        # Write the file to the in-memory zip
        zf.writestr(filename_in_zip, file_contents)

        # Mark the files as having been created on Windows so that
        # Unix permissions are not inferred as 0000
        for zfile in zf.filelist:
            zfile.create_system = 0        

        return self

    def read(self):
        '''Returns a string with the contents of the in-memory zip.'''
        self.in_memory_zip.seek(0)
        return self.in_memory_zip.read()

    def writetofile(self, filename):
        '''Writes the in-memory zip to a file.'''
        f = file(filename, "w")
        f.write(self.read())
        f.close()


def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, basestring):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, basestring):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and fk.request.method == 'OPTIONS':
                resp = app.make_default_options_response()
            else:
                resp = fk.make_response(f(*args, **kwargs))
            if not attach_to_all and fk.request.method != 'OPTIONS':
                return resp

            h = resp.headers

            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

    return [memory_file, record.environment.bundle['location'].split("/")[-1].split(".")[0]+".zip"]

def delete_project_files(project):
    from corrdb.common.models import ProjectModel
    from corrdb.common.models import RecordModel
    from corrdb.common.models import EnvironmentModel
    from corrdb.common.models import FileModel

    # print s3_files
    # project resources
    for _file in project.resources:
        file_ = FileModel.objects.with_id(_file)
        if file_:
            # print file_.to_json()
            result = s3_delete_file(file_.group, file_.storage)
            if result:
                logStat(deleted=True, file_obj=file_)
                file_.delete()

    # project records resources
    for record in project.records:
        result = delete_record_files(record)
        if result:
            logStat(deleted=True, record=record)
            record.delete()

    for environment_id in project.history:
        _environment = EnvironmentModel.objects.with_id(environment_id)
        if _environment.bundle["scope"] == "local":
            s3_bundles.delete_key(_environment.bundle.location)
            result = s3_delete_file('bundle', _environment.bundle.location)
            if result:
                logStat(deleted=True, bundle=_environment.bundle)
                logStat(deleted=True, environment=_environment)
                _environment.bundle.delete()
                _environment.delete()
        else:
            logStat(deleted=True, environment=_environment)
            _environment.delete()

def cloud_response(code, title, content):
    import flask as fk
    response = {'code':code, 'title':title, 'content':content}
    # print response
    return fk.Response(json.dumps(response, sort_keys=True, indent=4, separators=(',', ': ')), mimetype='application/json')

def delete_record_files(record):
    # s3_files = s3.Bucket('reproforge-files')

    from corrdb.common.models import RecordModel
    from corrdb.common.models import FileModel
    final_result = True
    for _file_id in record.resources:
        _file = FileModel.objects.with_id(_file_id)
        result = delete_record_file(_file)
        if not result:
            final_result = result
    return final_result

def delete_record_file(record_file):
    result = s3_delete_file(record_file.group, record_file.storage)
    if result:
        logStat(deleted=True, file_obj=record_file)
        record_file.delete()
    return result

def s3_get_file(group='', key=''):
    file_buffer = StringIO()
    print 'corr-{0}s/{1}'.format(group,key)
    try:
        obj = None
        if key != '':
            obj = s3.Object(bucket_name=S3_BUCKET, key='corr-{0}s/{1}'.format(group,key))
        else:
            if group == 'picture' or group == 'logo':
                obj = s3.Object(bucket_name=S3_BUCKET, key='corr-{0}s/default-{1}.png'.format(group,key))
    except:
        print 'corr-{0}s/{1}'.format(group,key)
        print traceback.print_exc()
        if group == 'picture' or group == 'logo':
            obj = s3.Object(bucket_name=S3_BUCKET, key='corr-logos/default-{0}.png'.format(group))

    try:
        res = obj.get()
        print res['Body']
        file_buffer.write(res['Body'].read())
        file_buffer.seek(0)
        return file_buffer
    except:
        print 'corr-{0}s/{1}'.format(group,key)
        print traceback.print_exc()
        return None

def s3_upload_file(file_meta=None, file_obj=None):
    if file_meta != None and file_obj != None:
        if file_meta.location == 'local':
            dest_filename = file_meta.storage
            try:
                group = 'corr-resources'
                if file_meta.group != 'descriptive':
                    group = 'corr-%ss'%file_meta.group
                print group
                s3_files = s3.Bucket(S3_BUCKET)
                s3_files.put_object(Key='{0}/{1}'.format(group, dest_filename), Body=file_obj.read())
                return [True, "File uploaded successfully"]
            except:
                return [False, traceback.format_exc()]
        else:
            return [False, "Cannot upload a file that is remotely set. It has to be local targeted."]
    else:
        return [False, "file meta data does not exist or file content is empty."]

def s3_delete_file(group='', key=''):
    deleted = False
    if key not in ["default-logo.png", "default-picture.png"]:<commit_msg>debugging file uploades.<commit_after>import flask as fk
from corrdb.common.core import setup_app
from corrdb.common.models import UserModel
from corrdb.common.models import ProjectModel
from corrdb.common.models import ApplicationModel
from corrdb.common.models import TrafficModel
from corrdb.common.models import StatModel  
from corrdb.common.models import AccessModel
import tarfile
from StringIO import StringIO
from io import BytesIO
import zipfile
import json
import time
import boto3
import traceback 
import datetime

import requests
from datetime import date, timedelta
from functools import update_wrapper
from calendar import monthrange
import time

app = setup_app(__name__)

s3 =  boto3.resource('s3')

S3_BUCKET = app.config['S3_BUCKET']

# Stormpath

from flask.ext.stormpath import StormpathManager

stormpath_manager = StormpathManager(app)

from datetime import date, timedelta
from functools import update_wrapper


def get_week_days(year, week):
    d = date(year,1,1)
    if(d.weekday()>3):
        d = d+timedelta(7-d.weekday())
    else:
        d = d - timedelta(d.weekday())
    dlt = timedelta(days = (week-1)*7)
    return d + dlt,  d + dlt + timedelta(days=6)

def find_week_days(year, current):
    index  = 1
    while True:
        if index == 360:
            break
        interval = get_week_days(year, index)
        if current > interval[0] and current < interval[1]:
            return interval
        index +=1

class InMemoryZip(object):
    def __init__(self):
        # Create the in-memory file-like object
        self.in_memory_zip = StringIO()

    def append(self, filename_in_zip, file_contents):
        '''Appends a file with name filename_in_zip and contents of 
        file_contents to the in-memory zip.'''
        # Get a handle to the in-memory zip in append mode
        zf = zipfile.ZipFile(self.in_memory_zip, "a", zipfile.ZIP_DEFLATED, False)

        # Write the file to the in-memory zip
        zf.writestr(filename_in_zip, file_contents)

        # Mark the files as having been created on Windows so that
        # Unix permissions are not inferred as 0000
        for zfile in zf.filelist:
            zfile.create_system = 0        

        return self

    def read(self):
        '''Returns a string with the contents of the in-memory zip.'''
        self.in_memory_zip.seek(0)
        return self.in_memory_zip.read()

    def writetofile(self, filename):
        '''Writes the in-memory zip to a file.'''
        f = file(filename, "w")
        f.write(self.read())
        f.close()


def crossdomain(origin=None, methods=None, headers=None,
                max_age=21600, attach_to_all=True,
                automatic_options=True):
    if methods is not None:
        methods = ', '.join(sorted(x.upper() for x in methods))
    if headers is not None and not isinstance(headers, basestring):
        headers = ', '.join(x.upper() for x in headers)
    if not isinstance(origin, basestring):
        origin = ', '.join(origin)
    if isinstance(max_age, timedelta):
        max_age = max_age.total_seconds()

    def get_methods():
        if methods is not None:
            return methods

        options_resp = app.make_default_options_response()
        return options_resp.headers['allow']

    def decorator(f):
        def wrapped_function(*args, **kwargs):
            if automatic_options and fk.request.method == 'OPTIONS':
                resp = app.make_default_options_response()
            else:
                resp = fk.make_response(f(*args, **kwargs))
            if not attach_to_all and fk.request.method != 'OPTIONS':
                return resp

            h = resp.headers

            h['Access-Control-Allow-Origin'] = origin
            h['Access-Control-Allow-Methods'] = get_methods()
            h['Access-Control-Max-Age'] = str(max_age)
            if headers is not None:
                h['Access-Control-Allow-Headers'] = headers
            return resp

        f.provide_automatic_options = False
        return update_wrapper(wrapped_function, f)
    return decorator

    return [memory_file, record.environment.bundle['location'].split("/")[-1].split(".")[0]+".zip"]

def delete_project_files(project):
    from corrdb.common.models import ProjectModel
    from corrdb.common.models import RecordModel
    from corrdb.common.models import EnvironmentModel
    from corrdb.common.models import FileModel

    # print s3_files
    # project resources
    for _file in project.resources:
        file_ = FileModel.objects.with_id(_file)
        if file_:
            # print file_.to_json()
            result = s3_delete_file(file_.group, file_.storage)
            if result:
                logStat(deleted=True, file_obj=file_)
                file_.delete()

    # project records resources
    for record in project.records:
        result = delete_record_files(record)
        if result:
            logStat(deleted=True, record=record)
            record.delete()

    for environment_id in project.history:
        _environment = EnvironmentModel.objects.with_id(environment_id)
        if _environment.bundle["scope"] == "local":
            s3_bundles.delete_key(_environment.bundle.location)
            result = s3_delete_file('bundle', _environment.bundle.location)
            if result:
                logStat(deleted=True, bundle=_environment.bundle)
                logStat(deleted=True, environment=_environment)
                _environment.bundle.delete()
                _environment.delete()
        else:
            logStat(deleted=True, environment=_environment)
            _environment.delete()

def cloud_response(code, title, content):
    import flask as fk
    response = {'code':code, 'title':title, 'content':content}
    # print response
    return fk.Response(json.dumps(response, sort_keys=True, indent=4, separators=(',', ': ')), mimetype='application/json')

def delete_record_files(record):
    # s3_files = s3.Bucket('reproforge-files')

    from corrdb.common.models import RecordModel
    from corrdb.common.models import FileModel
    final_result = True
    for _file_id in record.resources:
        _file = FileModel.objects.with_id(_file_id)
        result = delete_record_file(_file)
        if not result:
            final_result = result
    return final_result

def delete_record_file(record_file):
    result = s3_delete_file(record_file.group, record_file.storage)
    if result:
        logStat(deleted=True, file_obj=record_file)
        record_file.delete()
    return result

def s3_get_file(group='', key=''):
    file_buffer = StringIO()
    print 'corr-{0}s/{1}'.format(group,key)
    try:
        obj = None
        if key != '':
            obj = s3.Object(bucket_name=S3_BUCKET, key='corr-{0}s/{1}'.format(group,key))
        else:
            if group == 'picture' or group == 'logo':
                obj = s3.Object(bucket_name=S3_BUCKET, key='corr-{0}s/default-{1}.png'.format(group,key))
    except:
        print 'corr-{0}s/{1}'.format(group,key)
        print traceback.print_exc()
        if group == 'picture' or group == 'logo':
            obj = s3.Object(bucket_name=S3_BUCKET, key='corr-logos/default-{0}.png'.format(group))

    try:
        res = obj.get()
        print res
        file_buffer.write(res['Body'].read())
        file_buffer.seek(0)
        return file_buffer
    except:
        print 'corr-{0}s/{1}'.format(group,key)
        print traceback.print_exc()
        return None

def s3_upload_file(file_meta=None, file_obj=None):
    if file_meta != None and file_obj != None:
        if file_meta.location == 'local':
            dest_filename = file_meta.storage
            try:
                group = 'corr-resources'
                if file_meta.group != 'descriptive':
                    group = 'corr-%ss'%file_meta.group
                print group
                s3_files = s3.Bucket(S3_BUCKET)
                s3_files.put_object(Key='{0}/{1}'.format(group, dest_filename), Body=file_obj.read())
                return [True, "File uploaded successfully"]
            except:
                return [False, traceback.format_exc()]
        else:
            return [False, "Cannot upload a file that is remotely set. It has to be local targeted."]
    else:
        return [False, "file meta data does not exist or file content is empty."]

def s3_delete_file(group='', key=''):
    deleted = False
    if key not in ["default-logo.png", "default-picture.png"]:
{'fraction_non_alphanumeric': 0.10254142343298543, 'fraction_numerical': 0.008019340763016688, 'mean_word_length': 2.5555555555555554, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3684477', 'n_tokens_mistral': 5102, 'n_tokens_neox': 4955, 'n_words': 1263}