<commit_before>__author__ = 'dhan'

import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../..")
from slideatlas.uploader import MongoUploaderWrapper, MongoUploaderPtiff, MongoUploaderPyramid

import logging
logger = logging.getLogger("tasks.dicer")

from bson import ObjectId

from .common import celeryapp


@celeryapp.task()
def process_file(args):
    """
    Deliver the file to appropriate
    """
    resp = {}
    resp["args"] = args
    logger.info("Got task: " + str(args))
    # if args.input is None:
    #     # Get the file from
    #     # Create a unique folder name
    #     temp_folder_name = str(ObjectId())
    #     logger.info("Using " + temp_folder_name)

    # ext = os.path.splitext(args.input)[1][1:]
    # logger.warning("Extension: " + ext)

    # if ext in ["ptif", ]:
    #     logger.info("Got a PTIF")
    #     MongoUploaderPtiff(args)

    # elif ext in ["jp2", ]:
    #     logger.info("Got a " + args.input)
    #     MongoUploaderWrapper(args)

    # elif ext in ["scn", "ndpi", "svs", "tif", "jpg", "png", "bif"]:
    #     logger.info("Got a " + args.input)
    #     MongoUploaderPyramid(args)

    # else:
    #     # Check if the input is a dir
    #     logger.error("Unsupported file: " + args.input[-4:])

    logger.info(str(resp))
    return resp<commit_msg>Processing the file as celery task<commit_after>__author__ = 'dhan'

import os
import sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../..")

from slideatlas.uploader import MongoUploaderPyramid
from slideatlas.models import Session
from slideatlas.api import apiv2

import logging
logger = logging.getLogger("tasks.dicer")

from bson import ObjectId
from bson.objectid import InvalidId

from .common import celeryapp, flaskapp


@celeryapp.task()
def process_file(args):
    """
    Deliver the file to appropriate
    """
    resp = {}
    resp["args"] = args
    logger.info("Got task: " + str(args))

    # Try if the input file is a file
    if not os.path.isfile(args["input"]):
        try:
            file_id = ObjectId(args["input"])
        except InvalidId:
            # logger.error("Raising exception !")
            raise Exception("Input should be a file or an gridfs id in the collection's default imagestore")

        with flaskapp.app_context():
            # Locate the session
            session = Session.objects.get(id=ObjectId(args["session"]))
            logger.info("session: %s" % (session.to_son()))
            datadb, filefs, afile = apiv2.SessionAttachmentItemAPI._fetch_attachment(session, "imagefiles", file_id)

            # Create a temporary location
            temp_folder_name = str(ObjectId())
            logger.info("Using temporary location: " + temp_folder_name)
            newpath = os.path.join(".", temp_folder_name)
            os.makedirs(newpath)

            # Get the file from gridfs
            newinput = os.path.join(newpath, afile.name)
            ofile = open(newinput, "wb")
            ofile.write(afile.read())
            ofile.close()

            args["input"] = newinput

        # Choose the reader based on file extension
        ext = os.path.splitext(args["input"])[1][1:]
        logger.warning("Extension: " + ext)

        #TODO: Test and enable other files too
        # if ext in ["ptif", ]:
        #     logger.info("Got a PTIF")
        #     MongoUploaderPtiff(args)

        # elif ext in ["jp2", ]:
        #     logger.info("Got a " + args.input)
        #     MongoUploaderWrapper(args)

        if ext in ["scn", "ndpi", "svs", "tif", "jpg", "png", "bif"]:
            logger.info("Got a " + args["input"])
            MongoUploaderPyramid(args)
        else:
            raise Exception("Unsupported file: " + ext)

    logger.info(str(resp))
    return resp
{'fraction_non_alphanumeric': 0.13110877561744613, 'fraction_numerical': 0.002364687335785602, 'mean_word_length': 2.5217391304347827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3711830', 'n_tokens_mistral': 1235, 'n_tokens_neox': 1192, 'n_words': 292}