<commit_before>import os
import shutil
import subprocess
import tempfile
import traceback
import zipfile
import json

from celery import task

from django.conf import settings
from django.utils.timezone import now

import apptools.addr2lines
from ide.utils import link_or_copy
from ide.utils.sdk import generate_wscript_file, generate_jshint_file, generate_v2_manifest_dict, \
    generate_simplyjs_manifest_dict
from utils.keen_helper import send_keen_event

from ide.models.build import  BuildResult
from ide.models.files import SourceFile, ResourceFile

__author__ = 'katharine'


@task(ignore_result=True, acks_late=True)
def run_compile(build_result):
    build_result = BuildResult.objects.get(pk=build_result)
    project = build_result.project
    source_files = SourceFile.objects.filter(project=project)
    resources = ResourceFile.objects.filter(project=project)

    if project.sdk_version == '1':
        build_result.state = BuildResult.STATE_FAILED
        build_result.finished = now()
        build_result.save()
        return

    # Assemble the project somewhere
    base_dir = tempfile.mkdtemp(dir=os.path.join(settings.CHROOT_ROOT, 'tmp') if settings.CHROOT_ROOT else None)
    print "Compiling in %s" % base_dir

    try:
        os.makedirs(build_result.get_dir())
    except OSError:
        pass

    try:
        if project.project_type == 'native':
            # Create symbolic links to the original files
            # Source code
            src_dir = os.path.join(base_dir, 'src')
            os.mkdir(src_dir)
            for f in source_files:
                abs_target = os.path.abspath(os.path.join(src_dir, f.file_name))
                if not abs_target.startswith(src_dir):
                    raise Exception("Suspicious filename: %s" % f.file_name)
                abs_target_dir = os.path.dirname(abs_target)
                if not os.path.exists(abs_target_dir):
                    print "Creating directory %s." % abs_target_dir
                    os.makedirs(abs_target_dir)
                link_or_copy(os.path.abspath(f.local_filename), abs_target)

            # Resources
            resource_root = 'resources'
            os.makedirs(os.path.join(base_dir, resource_root, 'images'))
            os.makedirs(os.path.join(base_dir, resource_root, 'fonts'))
            os.makedirs(os.path.join(base_dir, resource_root, 'data'))

            print "Writing out manifest"
            manifest_dict = generate_v2_manifest_dict(project, resources)
            open(os.path.join(base_dir, 'appinfo.json'), 'w').write(json.dumps(manifest_dict))

            for f in resources:
                target_dir = os.path.abspath(os.path.join(base_dir, resource_root, ResourceFile.DIR_MAP[f.kind]))
                abs_target = os.path.abspath(os.path.join(target_dir, f.file_name))
                if not abs_target.startswith(target_dir):
                    raise Exception("Suspicious filename: %s" % f.file_name)
                print "Added %s %s" % (f.kind, f.local_filename)
                link_or_copy(os.path.abspath(f.local_filename), abs_target)


            # Reconstitute the SDK<commit_msg>Handle files that have never been saved (fixes #57).<commit_after>import os
import shutil
import subprocess
import tempfile
import traceback
import zipfile
import json

from celery import task

from django.conf import settings
from django.utils.timezone import now

import apptools.addr2lines
from ide.utils import link_or_copy
from ide.utils.sdk import generate_wscript_file, generate_jshint_file, generate_v2_manifest_dict, \
    generate_simplyjs_manifest_dict
from utils.keen_helper import send_keen_event

from ide.models.build import  BuildResult
from ide.models.files import SourceFile, ResourceFile

__author__ = 'katharine'


@task(ignore_result=True, acks_late=True)
def run_compile(build_result):
    build_result = BuildResult.objects.get(pk=build_result)
    project = build_result.project
    source_files = SourceFile.objects.filter(project=project)
    resources = ResourceFile.objects.filter(project=project)

    if project.sdk_version == '1':
        build_result.state = BuildResult.STATE_FAILED
        build_result.finished = now()
        build_result.save()
        return

    # Assemble the project somewhere
    base_dir = tempfile.mkdtemp(dir=os.path.join(settings.CHROOT_ROOT, 'tmp') if settings.CHROOT_ROOT else None)
    print "Compiling in %s" % base_dir

    try:
        os.makedirs(build_result.get_dir())
    except OSError:
        pass

    try:
        if project.project_type == 'native':
            # Create symbolic links to the original files
            # Source code
            src_dir = os.path.join(base_dir, 'src')
            os.mkdir(src_dir)
            for f in source_files:
                abs_target = os.path.abspath(os.path.join(src_dir, f.file_name))
                if not abs_target.startswith(src_dir):
                    raise Exception("Suspicious filename: %s" % f.file_name)
                abs_target_dir = os.path.dirname(abs_target)
                if not os.path.exists(abs_target_dir):
                    print "Creating directory %s." % abs_target_dir
                    os.makedirs(abs_target_dir)
                try:
                    link_or_copy(os.path.abspath(f.local_filename), abs_target)
                except OSError as err:
                    if err.errno == 2:
                        open(abs_target, 'w').close() # create the file if it's missing.

            # Resources
            resource_root = 'resources'
            os.makedirs(os.path.join(base_dir, resource_root, 'images'))
            os.makedirs(os.path.join(base_dir, resource_root, 'fonts'))
            os.makedirs(os.path.join(base_dir, resource_root, 'data'))

            print "Writing out manifest"
            manifest_dict = generate_v2_manifest_dict(project, resources)
            open(os.path.join(base_dir, 'appinfo.json'), 'w').write(json.dumps(manifest_dict))

            for f in resources:
                target_dir = os.path.abspath(os.path.join(base_dir, resource_root, ResourceFile.DIR_MAP[f.kind]))
                abs_target = os.path.abspath(os.path.join(target_dir, f.file_name))
                if not abs_target.startswith(target_dir):
                    raise Exception("Suspicious filename: %s" % f.file_name)
                print "Added %s %s" % (f.kind, f.local_filename)
                link_or_copy(os.path.abspath(f.local_filename), abs_target)


            # Reconstitute the SDK
{'fraction_non_alphanumeric': 0.09057471264367815, 'fraction_numerical': 0.001685823754789272, 'mean_word_length': 2.623542476401999, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4581303', 'n_tokens_mistral': 2027, 'n_tokens_neox': 1942, 'n_words': 409}