<commit_before>from django.db import models
from django.template.defaultfilters import slugify
from django.contrib.auth.models import User

from autoslug import AutoSlugField
from taggit.managers import TaggableManager

import appsettings as settings

from pytranscode.ffmpeg import *
from pytranscode.presets import *
from pytranscode.runner import *

import os
import subprocess
import re
import decimal 

import transmissionrpc
import urllib2
import datetime
import os
import shutil
import re
from threading import Event

from BitTorrent.btmakemetafile import calcsize, make_meta_file, ignore


KIND_CHOICES = (
    (0, 'Video-only'),
    (1, 'Audio-only'),
    (2, 'Audio & Video'),
)

class Video(models.Model):
    ''' The model for our videos. It uses slugs (with DjangoAutoSlug) and tags (with Taggit)
    everything else is quite standard. The sizes fields are used in the feeds to make enclosures
    possible. The videoThumbURL is the URL for Projekktor's "poster" and assemblyid is just a storage
    for the result we get back from transloadit so that we know which video just triggered the "encoding_done"
    view. Why are there URL fields and not file fields? Because you maybe want to use external storage
    (like Amazon S3) to store your files '''
    title = models.CharField(u"Titel",max_length=200)
    slug = AutoSlugField(populate_from='title',unique=True)
    date = models.DateField("Datum")
    description = models.TextField(u"Beschreibung")
    user = models.ForeignKey(User, blank=True, null=True)
    channel = models.ForeignKey('videoportal.Channel',blank=True,null=True)
    linkURL = models.URLField("Link",blank=True,verify_exists=False)
    kind = models.IntegerField("Art",max_length=1, choices=KIND_CHOICES)
    torrentURL = models.URLField("Torrent-URL",blank=True,verify_exists=False)
    mp4URL = models.URLField("MP4-URL",blank=True,verify_exists=False)
    mp4Size = models.BigIntegerField("MP4 Size in Bytes",null=True,blank=True)
    webmURL = models.URLField("WEBM-URL",blank=True,verify_exists=False)
    webmSize = models.BigIntegerField("WEBM Size in Bytes",null=True,blank=True)
    mp3URL = models.URLField("MP3-URL",blank=True,verify_exists=False)
    mp3Size = models.BigIntegerField("MP3 Size in Bytes",null=True,blank=True)
    oggURL = models.URLField("OGG-URL",blank=True,verify_exists=False)
    oggSize = models.BigIntegerField("OGG Size in Bytes",null=True,blank=True)
    videoThumbURL = models.URLField("Thumb-URL",blank=True,verify_exists=False)
    duration = models.DecimalField(null=True, max_digits=10, decimal_places=2,blank=True)
    autoPublish = models.BooleanField(default=True)
    published = models.BooleanField()
    encodingDone = models.BooleanField()
    torrentDone = models.BooleanField()
    assemblyid = models.CharField("Transloadit Result",max_length=100,blank=True)
    tags = TaggableManager("Tags")
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)
    originalFile = models.FileField("Datei",upload_to="raw/%Y/%m/%d/",blank=True, max_length=2048)
    def __unicode__(self):
        return self.title
    def get_absolute_url(self):
        return "/videos/%s/" % self.slug
    def getClassName(self):
        return self.__class__.__name__

    def encode_media(self):
        ''' This is used to tell ffmpeg what to do '''
        kind = self.kind
        path = self.originalFile.path
        name_array = os.path.basename(self.originalFile.path).partition('.')
        name = slugify(name_array[0])
        outputdir = settings.ENCODING_OUTPUT_DIR + slugify(name)
        if not os.path.exists(outputdir):
            os.makedirs(outputdir)
        outputdir = outputdir + '/'
        if ((kind == 0) or (kind == 2)):
            logfile = settings.ENCODING_OUTPUT_DIR + slugify(name) + 'encoding_mp4_log.txt'
            outfile_mp4 = outputdir + slugify(name) + '.mp4'
            # Create the command line
            cl_mp4 = ffmpeg(path, outfile_mp4, logfile, OWNTUBE_MP4_VIDEO, OWNTUBE_MP4_AUDIO).build_command_line()
            
            logfile = settings.ENCODING_OUTPUT_DIR + slugify(name) + 'encoding_webm_log.txt'
            outfile_webm = outputdir + slugify(name) + '.webm'
    
            cl_webm = ffmpeg(path, outfile_webm, logfile, OWNTUBE_WEBM_VIDEO, OWNTUBE_WEBM_AUDIO).build_command_line()
            
            self.mp4URL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) + '/' + slugify(name) + '.mp4'
            self.webmURL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) + '/' + slugify(name) + '.webm' 
            self.videoThumbURL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) + '/' + slugify(name) + '.jpg'
            outcode = subprocess.Popen(cl_mp4, shell=True)
            
            while outcode.poll() == None:
                pass
    
            if outcode.poll() == 0:
                self.mp4Size = os.path.getsize(outfile_mp4)
                self.duration = getLength(outfile_mp4)
            else:
                raise StandardError('Encoding MP4 Failed')
            
            print(cl_mp4)
            print(cl_webm)    
            outcode = subprocess.Popen(cl_webm, shell=True)
            
            while outcode.poll() == None:
                pass
    
            if outcode.poll() == 0:
                self.wembSize = os.path.getsize(outfile_webm)
            else:
                raise StandardError('Encoding WEBM Failed')
    
            outcode = subprocess.Popen(['ffmpeg -i '+ self.originalFile.path + ' -ss 5.0 -vframes 1 -f image2 ' + outputdir + slugify(name) + '.jpg'],shell = True)
            
            while outcode.poll() == None:
                pass
    
            if outcode.poll() == 0:
                pass 
            else:
                raise StandardError('Making Thumb Failed')
            
            
        if((kind == 1) or (kind == 2)):
            logfile = settings.ENCODING_OUTPUT_DIR + slugify(name) + 'encoding_mp3_log.txt'
            outfile_mp3 = outputdir + slugify(name) + '.mp3'
            # Create the command line
            cl_mp3 = ffmpeg(path, outfile_mp3, logfile, OWNTUBE_NULL_VIDEO , OWNTUBE_MP3_AUDIO).build_command_line()
            
            logfile = settings.ENCODING_OUTPUT_DIR + slugify(name) + 'encoding_ogg_log.txt'
            outfile_ogg = outputdir + slugify(name) + '.ogg'

            cl_ogg = ffmpeg(path, outfile_ogg, logfile, OWNTUBE_NULL_VIDEO, OWNTUBE_OGG_AUDIO).build_command_line()
            
            self.mp3URL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) +  '/' + slugify(name) + '.mp3'
            self.oggURL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) +  '/' + slugify(name) + '.ogg'
                        
            outcode = subprocess.Popen(cl_mp3, shell=True)
            
            while outcode.poll() == None:<commit_msg>Logfiles are now correct<commit_after>from django.db import models
from django.template.defaultfilters import slugify
from django.contrib.auth.models import User

from autoslug import AutoSlugField
from taggit.managers import TaggableManager

import appsettings as settings

from pytranscode.ffmpeg import *
from pytranscode.presets import *
from pytranscode.runner import *

import os
import subprocess
import re
import decimal 

import transmissionrpc
import urllib2
import datetime
import os
import shutil
import re
from threading import Event

from BitTorrent.btmakemetafile import calcsize, make_meta_file, ignore


KIND_CHOICES = (
    (0, 'Video-only'),
    (1, 'Audio-only'),
    (2, 'Audio & Video'),
)

class Video(models.Model):
    ''' The model for our videos. It uses slugs (with DjangoAutoSlug) and tags (with Taggit)
    everything else is quite standard. The sizes fields are used in the feeds to make enclosures
    possible. The videoThumbURL is the URL for Projekktor's "poster" and assemblyid is just a storage
    for the result we get back from transloadit so that we know which video just triggered the "encoding_done"
    view. Why are there URL fields and not file fields? Because you maybe want to use external storage
    (like Amazon S3) to store your files '''
    title = models.CharField(u"Titel",max_length=200)
    slug = AutoSlugField(populate_from='title',unique=True)
    date = models.DateField("Datum")
    description = models.TextField(u"Beschreibung")
    user = models.ForeignKey(User, blank=True, null=True)
    channel = models.ForeignKey('videoportal.Channel',blank=True,null=True)
    linkURL = models.URLField("Link",blank=True,verify_exists=False)
    kind = models.IntegerField("Art",max_length=1, choices=KIND_CHOICES)
    torrentURL = models.URLField("Torrent-URL",blank=True,verify_exists=False)
    mp4URL = models.URLField("MP4-URL",blank=True,verify_exists=False)
    mp4Size = models.BigIntegerField("MP4 Size in Bytes",null=True,blank=True)
    webmURL = models.URLField("WEBM-URL",blank=True,verify_exists=False)
    webmSize = models.BigIntegerField("WEBM Size in Bytes",null=True,blank=True)
    mp3URL = models.URLField("MP3-URL",blank=True,verify_exists=False)
    mp3Size = models.BigIntegerField("MP3 Size in Bytes",null=True,blank=True)
    oggURL = models.URLField("OGG-URL",blank=True,verify_exists=False)
    oggSize = models.BigIntegerField("OGG Size in Bytes",null=True,blank=True)
    videoThumbURL = models.URLField("Thumb-URL",blank=True,verify_exists=False)
    duration = models.DecimalField(null=True, max_digits=10, decimal_places=2,blank=True)
    autoPublish = models.BooleanField(default=True)
    published = models.BooleanField()
    encodingDone = models.BooleanField()
    torrentDone = models.BooleanField()
    assemblyid = models.CharField("Transloadit Result",max_length=100,blank=True)
    tags = TaggableManager("Tags")
    created = models.DateTimeField(auto_now_add=True)
    modified = models.DateTimeField(auto_now=True)
    originalFile = models.FileField("Datei",upload_to="raw/%Y/%m/%d/",blank=True, max_length=2048)
    def __unicode__(self):
        return self.title
    def get_absolute_url(self):
        return "/videos/%s/" % self.slug
    def getClassName(self):
        return self.__class__.__name__

    def encode_media(self):
        ''' This is used to tell ffmpeg what to do '''
        kind = self.kind
        path = self.originalFile.path
        name_array = os.path.basename(self.originalFile.path).partition('.')
        name = slugify(name_array[0])
        outputdir = settings.ENCODING_OUTPUT_DIR + slugify(name)
        if not os.path.exists(outputdir):
            os.makedirs(outputdir)
        outputdir = outputdir + '/'
        if ((kind == 0) or (kind == 2)):
            logfile = outputdir + encoding_mp4_log.txt'
            outfile_mp4 = outputdir + slugify(name) + '.mp4'
            # Create the command line
            cl_mp4 = ffmpeg(path, outfile_mp4, logfile, OWNTUBE_MP4_VIDEO, OWNTUBE_MP4_AUDIO).build_command_line()
            
            logfile = outputdir + encoding_webm_log.txt'
            outfile_webm = outputdir + slugify(name) + '.webm'
    
            cl_webm = ffmpeg(path, outfile_webm, logfile, OWNTUBE_WEBM_VIDEO, OWNTUBE_WEBM_AUDIO).build_command_line()
            
            self.mp4URL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) + '/' + slugify(name) + '.mp4'
            self.webmURL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) + '/' + slugify(name) + '.webm' 
            self.videoThumbURL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) + '/' + slugify(name) + '.jpg'
            outcode = subprocess.Popen(cl_mp4, shell=True)
            
            while outcode.poll() == None:
                pass
    
            if outcode.poll() == 0:
                self.mp4Size = os.path.getsize(outfile_mp4)
                self.duration = getLength(outfile_mp4)
            else:
                raise StandardError('Encoding MP4 Failed')
            
            print(cl_mp4)
            print(cl_webm)    
            outcode = subprocess.Popen(cl_webm, shell=True)
            
            while outcode.poll() == None:
                pass
    
            if outcode.poll() == 0:
                self.wembSize = os.path.getsize(outfile_webm)
            else:
                raise StandardError('Encoding WEBM Failed')
    
            outcode = subprocess.Popen(['ffmpeg -i '+ self.originalFile.path + ' -ss 5.0 -vframes 1 -f image2 ' + outputdir + slugify(name) + '.jpg'],shell = True)
            
            while outcode.poll() == None:
                pass
    
            if outcode.poll() == 0:
                pass 
            else:
                raise StandardError('Making Thumb Failed')
            
            
        if((kind == 1) or (kind == 2)):
            logfile = outputdir + encoding_mp3_log.txt'
            outfile_mp3 = outputdir + slugify(name) + '.mp3'
            # Create the command line
            cl_mp3 = ffmpeg(path, outfile_mp3, logfile, OWNTUBE_NULL_VIDEO , OWNTUBE_MP3_AUDIO).build_command_line()
            
            logfile = outputdir + 'encoding_ogg_log.txt'
            outfile_ogg = outputdir + slugify(name) + '.ogg'

            cl_ogg = ffmpeg(path, outfile_ogg, logfile, OWNTUBE_NULL_VIDEO, OWNTUBE_OGG_AUDIO).build_command_line()
            
            self.mp3URL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) +  '/' + slugify(name) + '.mp3'
            self.oggURL = settings.ENCODING_VIDEO_BASE_URL + slugify(name) +  '/' + slugify(name) + '.ogg'
                        
            outcode = subprocess.Popen(cl_mp3, shell=True)
            
            while outcode.poll() == None:
{'fraction_non_alphanumeric': 0.1052979160551805, 'fraction_numerical': 0.00924567067801585, 'mean_word_length': 2.8069832402234636, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2320449', 'n_tokens_mistral': 4390, 'n_tokens_neox': 4227, 'n_words': 894}