<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import logging
import math
import os

from django.conf import settings
from django.core.files.uploadedfile import InMemoryUploadedFile

import django_rq
import nanoscope
from PIL import Image, ImageDraw, ImageFont
import six

from core.tasks import AsyncDjangoFile


logger = logging.getLogger(__name__)


@django_rq.job
def process_nanoscope_file(raw_file):
    scan_number = int(os.path.splitext(raw_file.name)[-1][1:])
    if '_' in raw_file.name:
        location = os.path.splitext(raw_file.name)[0].split('_')[-1]
        if location not in 'rRcCfFeE':
            location = 'c'
    else:
        location = 'c'
    logger.debug(
        'Extracted scan_number={}, location={}'.format(scan_number, location))
    raw = six.BytesIO(raw_file.read())
    raw.mode = 'b'
    scan = nanoscope.read(raw, encoding='cp1252')
    logger.debug('Read in file {}'.format(raw_file.name))

    processed_files = [
        AsyncDjangoFile(raw_file,
                        dict(image_type='Raw', state='raw',
                             rms=0.0, zrange=0.0,
                             size=0.0, scan_number=scan_number,
                             content_type='application/octet-stream'))]
    for img in scan:
        processed_image = _create_scan_png(img, raw_file.name, scan_number)
        logger.debug('Created image file for {} scan'.format(img.type))
        scan_size = math.sqrt(img.scan_area)
        processed_files.append(
            AsyncDjangoFile(processed_image,
                            dict(image_type=img.type, state='extracted',
                                 rms=img.rms, zrange=img.zrange,
                                 size=scan_size, scan_number=scan_number)))
    processed_files[0].size = scan_size
    return processed_files


def _create_scan_png(scan, filename, scan_number):
    """
    Create standard png with afm and summary information.
    """
    image = Image.fromarray(scan.colorize())
    if image.size != (512, 512):
        logger.debug('Image not correct size: {}, should be 512 x 512'.format(image.size))<commit_msg>Added missing processing steps to afm.<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals

import logging
import math
import os

from django.conf import settings
from django.core.files.uploadedfile import InMemoryUploadedFile

import django_rq
import nanoscope
from PIL import Image, ImageDraw, ImageFont
import six

from core.tasks import AsyncDjangoFile


logger = logging.getLogger(__name__)


@django_rq.job
def process_nanoscope_file(raw_file):
    scan_number = int(os.path.splitext(raw_file.name)[-1][1:])
    if '_' in raw_file.name:
        location = os.path.splitext(raw_file.name)[0].split('_')[-1]
        if location not in 'rRcCfFeE':
            location = 'c'
    else:
        location = 'c'
    logger.debug(
        'Extracted scan_number={}, location={}'.format(scan_number, location))
    raw = six.BytesIO(raw_file.read())
    raw.mode = 'b'
    scan = nanoscope.read(raw, encoding='cp1252')
    logger.debug('Read in file {}'.format(raw_file.name))

    processed_files = [
        AsyncDjangoFile(raw_file,
                        dict(image_type='Raw', state='raw',
                             rms=0.0, zrange=0.0,
                             size=0.0, scan_number=scan_number,
                             content_type='application/octet-stream'))]
    for img in scan:
        img.process()
        logger.debug('Processed {} scan'.format(img.type))
        processed_image = _create_scan_png(img, raw_file.name, scan_number)
        logger.debug('Created image file for {} scan'.format(img.type))
        scan_size = math.sqrt(img.scan_area)
        processed_files.append(
            AsyncDjangoFile(processed_image,
                            dict(image_type=img.type, state='extracted',
                                 rms=img.rms, zrange=img.zrange,
                                 size=scan_size, scan_number=scan_number)))
    processed_files[0].kwargs['size'] = scan_size
    return processed_files


def _create_scan_png(scan, filename, scan_number):
    """
    Create standard png with afm and summary information.
    """
    image = Image.fromarray(scan.colorize())
    if image.size != (512, 512):
        logger.debug('Image not correct size: {}, should be 512 x 512'.format(image.size))
{'fraction_non_alphanumeric': 0.11278703286807744, 'fraction_numerical': 0.012606933813597478, 'mean_word_length': 2.597570850202429, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7605513', 'n_tokens_mistral': 1375, 'n_tokens_neox': 1326, 'n_words': 285}