<commit_before>import asyncio
import random
import telepot
import dropbox
import tempfile
import shutil
import myconf
import os
import hashlib
from moviepy.editor import VideoFileClip
import datetime

import vimeo
from sizefield.utils import parse_size

from tgbot.models import RequestedSign
from dictionary.models import BSLEntry
from videolibrary.models import SourceVideo

@asyncio.coroutine
def upload_to_dropbox(file_object, file_name, directory="/bslparlour/videos/tg_uploads/"):
    """A simple function to upload a thing to dropbox."""
    dbox_client = dropbox.client.DropboxClient(myconf.dbox_master_key)
    print("Uploading to dropbox...")
    retval = dbox_client.put_file(
        os.path.join(directory, file_name), file_object)
    print("Uploaded to dropbox...")
    return retval
    
@asyncio.coroutine
def upload_to_vimeo(file_name):
    print("Uploading to vimeo...")
    v = vimeo.VimeoClient(
        token=myconf.vimeo_access_token,
        key=myconf.vimeo_client_identifier,
        secret=myconf.vimeo_client_secret
        )
    return v.upload(file_name)
    print("Uploaded to vimeo.")
 

class Charles(telepot.helper.ChatHandler):
    def __init__(self, seed_tuple, timeout):
        super(Charles, self).__init__(seed_tuple, timeout)

    @asyncio.coroutine
    def on_message(self, msg):
        try:
            if msg["text"] == "/requestsign" or msg['text'] == "/tosign":
                yield from self.request_sign(msg)
            elif msg["text"] == "/listsigns":
                yield from self.list_signs(msg)
            elif msg["text"] == "/signwhat":
                yield from self.sign_what(msg)
            elif msg["text"] == "/uploadsign":
                yield from self.upload_sign(msg)
        except KeyError:
            # If there is no "text", just carry on.
            pass

    @asyncio.coroutine
    def fetch_video_from_user(self, video_json, video_file):
        retval = yield from self._bot.downloadFile(video_json["video"]["file_id"], video_file)
        return retval
        
    @asyncio.coroutine
    def upload_sign(self, msg):
        video_json = yield from self.request_info("Send your video!")

        prefix =   datetime.datetime.strftime(
            datetime.datetime.now(),
            "%Y%m%d%H%M%S",
        )
        with tempfile.NamedTemporaryFile(prefix=prefix, suffix=".mp4") as f:
            try:
                yield from self.fetch_video_from_user(video_json, f.file)
            except KeyError:
                yield from self.sender.sendMessage("Sorry, I must recieve a file.")
                return
            
            yield from self.sender.sendMessage("Processing video...")

            f.seek(0)
            dropbox_json = yield from upload_to_dropbox(f.file, f.name.split("/")[-1])

            vimeo_uri = yield from upload_to_vimeo(f.name)
            
            # TODO add logger stuffs for return things
        
            # get file hash
            # TODO make function
            f.seek(0)
            file_hash = hashlib.sha224()
            BUFSIZE = 65536
            buf = f.read(BUFSIZE)
            while len(buf) > 0:
                file_hash.update(buf)
                buf = f.read(BUFSIZE)
                
            source_video = SourceVideo.objects.create(
                sha224=file_hash.hexdigest(),
                filename=f.name.split("/")[-1],
                dropbox_directory="/bslparlour/videos/tg_uploads",
                mime_type=dropbox_json["mime_type"],
                size=parse_size(dropbox_json["size"]),
                vimeo_uri=int(vimeo_uri.split("/")[-1]),
            )

        yield from self.sender.sendMessage(
            "Added to Dropbox, Vimeo, and SourceVideo table")

        # gloss = self.request_info("What's the gloss for this sign?")
        # # Check for additional gloss
        # # Confirm if gloss already present
        # bsl_entry = BSLEntry.objects.create(
        #     # blah blah blah
        # )
        
        # tweet_entry =           # blah blah


 
    @asyncio.coroutine
    def sign_what(self, msg):
        a = yield from self.request_info("What would you like to know the sign for?")
        a = a["text"]
        try:
            bsl_entry = BSLEntry.objects.get(gloss=a.upper())
        except:
            bsl_entries = BSLEntry.objects.filter(gloss__contains=a.upper())
            bsl_entry = bsl_entries[0]
        yield from self.sender.sendMessage(bsl_entry.gloss)
        source_video = bsl_entry.source_videos.all()[0]
        
        yield from self._bot.sendChatAction(self.chat_id, 'upload_video')

        dbox_client = dropbox.client.DropboxClient(myconf.dbox_master_key)
        temp_dir = tempfile.mkdtemp()
        try:
            f, metadata = dbox_client.get_file_and_metadata(
                os.path.join(
                    source_video.dropbox_directory, 
                    source_video.filename))
            print("Got video path")
            temp_video_file_path = os.path.join(temp_dir, source_video.filename)
            print("Writing file")
            with open(temp_video_file_path, "wb") as out:
                out.write(f.read())
            print("Converting file")
            v = VideoFileClip(temp_video_file_path)
            v.write_videofile(temp_video_file_path+".mp4")
            print("Sending file")
            yield from self.sender.sendVideo(
                open(temp_video_file_path+".mp4", "rb"))
            
        except Exception as e:
            # TODO something clever
            raise(e)
        finally:
            shutil.rmtree(temp_dir)

            
   
    @asyncio.coroutine
    def list_signs(self, msg):
        yield from self.sender.sendMessage(
            "\n".join(
                [str(x) for x in RequestedSign.objects.all()]))

    @asyncio.coroutine
    def request_info(self, prompt, timeout=30):
        yield from self.sender.sendMessage(prompt)
        l = self._bot.create_listener()
        l.set_options(timeout=timeout)
        l.capture(chat__id=self.chat_id)
        retmsg = yield from l.wait()

        return retmsg

    @asyncio.coroutine
    def request_sign(self, msg):
        # TODO make a table in the Django way for the signs left to sign
        try:
            short_description = yield from self.request_info(
                "Which sign would you like to video later?")
            short_description = short_description['text']
            description = yield from self.request_info(
                "Description?")
            description = description['text']
            rs = RequestedSign.objects.create(
                short_description=short_description,
                description=description,)
        except telepot.helper.WaitTooLong:
            yield from self.sender.sendMessage("Request cancelled.")

        yield from self.sender.sendMessage("Sign request logged.")
<commit_msg>Functioned out hash of open file function.<commit_after>import asyncio
import random
import telepot
import dropbox
import tempfile
import shutil
import myconf
import os
import hashlib
from moviepy.editor import VideoFileClip
import datetime

import vimeo
from sizefield.utils import parse_size

from tgbot.models import RequestedSign
from dictionary.models import BSLEntry
from videolibrary.models import SourceVideo

@asyncio.coroutine
def upload_to_dropbox(file_object, file_name, directory="/bslparlour/videos/tg_uploads/"):
    """A simple function to upload a thing to dropbox."""
    dbox_client = dropbox.client.DropboxClient(myconf.dbox_master_key)
    print("Uploading to dropbox...")
    retval = dbox_client.put_file(
        os.path.join(directory, file_name), file_object)
    print("Uploaded to dropbox...")
    return retval
    
@asyncio.coroutine
def upload_to_vimeo(file_name):
    print("Uploading to vimeo...")
    v = vimeo.VimeoClient(
        token=myconf.vimeo_access_token,
        key=myconf.vimeo_client_identifier,
        secret=myconf.vimeo_client_secret
        )
    return v.upload(file_name)
    print("Uploaded to vimeo.")
    

@asyncio.coroutine
def sha224_of_file(f):
    """Return the sha224 of a file."""
    file_hash = hashlib.sha224()
    BUFSIZE = 65536
    buf = f.read(BUFSIZE)
    while len(buf) > 0:
        file_hash.update(buf)
        buf = f.read(BUFSIZE)

    return file_hash

 

class Charles(telepot.helper.ChatHandler):
    def __init__(self, seed_tuple, timeout):
        super(Charles, self).__init__(seed_tuple, timeout)

    @asyncio.coroutine
    def on_message(self, msg):
        try:
            if msg["text"] == "/requestsign" or msg['text'] == "/tosign":
                yield from self.request_sign(msg)
            elif msg["text"] == "/listsigns":
                yield from self.list_signs(msg)
            elif msg["text"] == "/signwhat":
                yield from self.sign_what(msg)
            elif msg["text"] == "/uploadsign":
                yield from self.upload_sign(msg)
        except KeyError:
            # If there is no "text", just carry on.
            pass

    @asyncio.coroutine
    def fetch_video_from_user(self, video_json, video_file):
        retval = yield from self._bot.downloadFile(video_json["video"]["file_id"], video_file)
        return retval
        
    @asyncio.coroutine
    def upload_sign(self, msg):
        video_json = yield from self.request_info("Send your video!")

        prefix =   datetime.datetime.strftime(
            datetime.datetime.now(),
            "%Y%m%d%H%M%S",
        )
        with tempfile.NamedTemporaryFile(prefix=prefix, suffix=".mp4") as f:
            try:
                yield from self.fetch_video_from_user(video_json, f.file)
            except KeyError:
                yield from self.sender.sendMessage("Sorry, I must recieve a file.")
                return
            
            yield from self.sender.sendMessage("Processing video...")

            f.seek(0)
            dropbox_json = yield from upload_to_dropbox(f.file, f.name.split("/")[-1])

            vimeo_uri = yield from upload_to_vimeo(f.name)
            
            # TODO add logger stuffs for return things
        
            f.seek(0)
            file_hash = sha224_of_file(f)
                
            source_video = SourceVideo.objects.create(
                sha224=file_hash.hexdigest(),
                filename=f.name.split("/")[-1],
                dropbox_directory="/bslparlour/videos/tg_uploads",
                mime_type=dropbox_json["mime_type"],
                size=parse_size(dropbox_json["size"]),
                vimeo_uri=int(vimeo_uri.split("/")[-1]),
            )

        yield from self.sender.sendMessage(
            "Added to Dropbox, Vimeo, and SourceVideo table")

        # gloss = self.request_info("What's the gloss for this sign?")
        # # Check for additional gloss
        # # Confirm if gloss already present
        # bsl_entry = BSLEntry.objects.create(
        #     # blah blah blah
        # )
        
        # tweet_entry =           # blah blah


 
    @asyncio.coroutine
    def sign_what(self, msg):
        a = yield from self.request_info("What would you like to know the sign for?")
        a = a["text"]
        try:
            bsl_entry = BSLEntry.objects.get(gloss=a.upper())
        except:
            bsl_entries = BSLEntry.objects.filter(gloss__contains=a.upper())
            bsl_entry = bsl_entries[0]
        yield from self.sender.sendMessage(bsl_entry.gloss)
        source_video = bsl_entry.source_videos.all()[0]
        
        yield from self._bot.sendChatAction(self.chat_id, 'upload_video')

        dbox_client = dropbox.client.DropboxClient(myconf.dbox_master_key)
        temp_dir = tempfile.mkdtemp()
        try:
            f, metadata = dbox_client.get_file_and_metadata(
                os.path.join(
                    source_video.dropbox_directory, 
                    source_video.filename))
            print("Got video path")
            temp_video_file_path = os.path.join(temp_dir, source_video.filename)
            print("Writing file")
            with open(temp_video_file_path, "wb") as out:
                out.write(f.read())
            print("Converting file")
            v = VideoFileClip(temp_video_file_path)
            v.write_videofile(temp_video_file_path+".mp4")
            print("Sending file")
            yield from self.sender.sendVideo(
                open(temp_video_file_path+".mp4", "rb"))
            
        except Exception as e:
            # TODO something clever
            raise(e)
        finally:
            shutil.rmtree(temp_dir)

            
   
    @asyncio.coroutine
    def list_signs(self, msg):
        yield from self.sender.sendMessage(
            "\n".join(
                [str(x) for x in RequestedSign.objects.all()]))

    @asyncio.coroutine
    def request_info(self, prompt, timeout=30):
        yield from self.sender.sendMessage(prompt)
        l = self._bot.create_listener()
        l.set_options(timeout=timeout)
        l.capture(chat__id=self.chat_id)
        retmsg = yield from l.wait()

        return retmsg

    @asyncio.coroutine
    def request_sign(self, msg):
        # TODO make a table in the Django way for the signs left to sign
        try:
            short_description = yield from self.request_info(
                "Which sign would you like to video later?")
            short_description = short_description['text']
            description = yield from self.request_info(
                "Description?")
            description = description['text']
            rs = RequestedSign.objects.create(
                short_description=short_description,
                description=description,)
        except telepot.helper.WaitTooLong:
            yield from self.sender.sendMessage("Request cancelled.")

        yield from self.sender.sendMessage("Sign request logged.")

{'fraction_non_alphanumeric': 0.09717823207601498, 'fraction_numerical': 0.004103080909876188, 'mean_word_length': 2.0955882352941178, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '644151', 'n_tokens_mistral': 4147, 'n_tokens_neox': 4011, 'n_words': 885}