<commit_before>import tempfile, shutil
import os
import re
import subprocess
import time
import datetime
import csv
import json, yaml
import string

from bson.objectid import ObjectId
from bson import json_util
from dateutil.parser import parse
from django.conf import settings
from hashlib import md5

from crits.core.class_mapper import class_from_value
from crits.core.exceptions import ZipFileError
from crits.core.mongo_tools import get_file

def get_file_fs(sample_md5):
    """
    Read a file from the filesystem. The path to the file is:

    /data/files/<md5[:2]>/<md5[2:4]>/<md5>

    :param sample_md5: The MD5 of the file to read off of disk.
    :type sample_md5: str
    :returns: str
    """

    try:
        fin = open('/data/files/%s/%s/%s' % (sample_md5[:2],
                                             sample_md5[2:4],
                                             sample_md5),
                   'rb')
        data = fin.read()
        fin.close()
    except:
        raise
    return data

def put_file_fs(data):
    """
    Write a file to the filesystem. The path to write the file to is:

    /data/files/<md5[:2]>/<md5[2:4]>/<md5>

    :param data: The data of the file to write.
    :type data: str
    :returns: str (the md5 of the file written)
    """

    a = md5()
    a.update(data)
    sample_md5 = a.hexdigest()
    try:
        fout = open('/data/files/%s/%s/%s' % (sample_md5[:2],
                                              sample_md5[2:4],
                                              sample_md5),
                    'wb')
        fout.write(data)
        fout.close()
    except:
        raise
    return sample_md5

def create_zip(files, pw_protect=True):
    """
    Create a zip file. Creates a temporary directory to write files to on disk
    using :class:`tempfile`. Uses /usr/bin/zip as the zipping mechanism
    currently. Will password protect the zip file as a default. The password for
    the zip file defaults to "<PASSWORD>", but it can be changed in the config 
    under zip7_password.

    :param files: The files to add to the zip file.
    :type files: list of files which are in the format of a list or tuple of
                 (<filename>, <data>).
    :param pw_protect: To password protect the zip file or not.
    :type pw_protect: boolean
    :returns: :class:`crits.core.exceptions.ZipFileError`, str
    """

    dumpdir = ""
    try:
        # Zip can take data from stdin to compress, but
        # you can't define the filenames within the archive,
        # they show up as "-".  Therefore, we need to write
        # out the file, compress it and return the zip.
        # Save the sample as a file in a temp directory
        # NOTE: the following line was causing a "permission denied" exception.
        # Removed dir arg.
        crits_config = CRITsConfig.objects().first()
        if crits_config:
            zip7_password = crits_config.zip7_password
        else:
            zip7_password = settings.ZIP7_PASSWORD
        dumpdir = tempfile.mkdtemp() #dir=temproot
        #write out binary files
        for f in files:
            filename = f[0]
            file_data = f[1]

            # make sure our desired path doesn't already exist (some files may
            # have the same name but different data)
            path = dumpdir + "/" + filename.encode("utf-8")
            i = 1
            tmp = path
            while os.path.exists(tmp):
                tmp = path+"("+str(i)+")"
                i += 1
<commit_msg>After all local import is required<commit_after>import tempfile, shutil
import os
import re
import subprocess
import time
import datetime
import csv
import json, yaml
import string

from bson.objectid import ObjectId
from bson import json_util
from dateutil.parser import parse
from django.conf import settings
from hashlib import md5

from crits.core.class_mapper import class_from_value
from crits.core.exceptions import ZipFileError
from crits.core.mongo_tools import get_file

def get_file_fs(sample_md5):
    """
    Read a file from the filesystem. The path to the file is:

    /data/files/<md5[:2]>/<md5[2:4]>/<md5>

    :param sample_md5: The MD5 of the file to read off of disk.
    :type sample_md5: str
    :returns: str
    """

    try:
        fin = open('/data/files/%s/%s/%s' % (sample_md5[:2],
                                             sample_md5[2:4],
                                             sample_md5),
                   'rb')
        data = fin.read()
        fin.close()
    except:
        raise
    return data

def put_file_fs(data):
    """
    Write a file to the filesystem. The path to write the file to is:

    /data/files/<md5[:2]>/<md5[2:4]>/<md5>

    :param data: The data of the file to write.
    :type data: str
    :returns: str (the md5 of the file written)
    """

    a = md5()
    a.update(data)
    sample_md5 = a.hexdigest()
    try:
        fout = open('/data/files/%s/%s/%s' % (sample_md5[:2],
                                              sample_md5[2:4],
                                              sample_md5),
                    'wb')
        fout.write(data)
        fout.close()
    except:
        raise
    return sample_md5

def create_zip(files, pw_protect=True):
    """
    Create a zip file. Creates a temporary directory to write files to on disk
    using :class:`tempfile`. Uses /usr/bin/zip as the zipping mechanism
    currently. Will password protect the zip file as a default. The password for
    the zip file defaults to "inf<PASSWORD>", but it can be changed in the config 
    under zip7_password.

    :param files: The files to add to the zip file.
    :type files: list of files which are in the format of a list or tuple of
                 (<filename>, <data>).
    :param pw_protect: To password protect the zip file or not.
    :type pw_protect: boolean
    :returns: :class:`crits.core.exceptions.ZipFileError`, str
    """

    dumpdir = ""
    try:
        # Zip can take data from stdin to compress, but
        # you can't define the filenames within the archive,
        # they show up as "-".  Therefore, we need to write
        # out the file, compress it and return the zip.
        # Save the sample as a file in a temp directory
        # NOTE: the following line was causing a "permission denied" exception.
        # Removed dir arg.
        from crits.config.config import CRITsConfig
        crits_config = CRITsConfig.objects().first()
        if crits_config:
            zip7_password = crits_config.zip7_password
        else:
            zip7_password = settings.ZIP7_PASSWORD
        dumpdir = tempfile.mkdtemp() #dir=temproot
        #write out binary files
        for f in files:
            filename = f[0]
            file_data = f[1]

            # make sure our desired path doesn't already exist (some files may
            # have the same name but different data)
            path = dumpdir + "/" + filename.encode("utf-8")
            i = 1
            tmp = path
            while os.path.exists(tmp):
                tmp = path+"("+str(i)+")"
                i += 1

{'fraction_non_alphanumeric': 0.08863252673044457, 'fraction_numerical': 0.012099043331457513, 'mean_word_length': 1.949792531120332, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7452660', 'n_tokens_mistral': 2100, 'n_tokens_neox': 2037, 'n_words': 741}