<commit_before>#!/usr/bin/python
import os
import sys
import tempfile
import getopt
import re
import socket
import logging
import argparse
import xml.etree.ElementTree as ET

logging.basicConfig(level=logging.DEBUG)
try:
    import MySQLdb
except ImportError:
    print 'This program requires the python MySQLdb module'
    sys.exit(10)


#Todo
# 1. Add code to verify prereqs including:
#   MySQLdb python module
#   mplayer
#   avconv
#   mythtranscode
# 2. Add code to transcode to a second file for example use on ipod/cell-phone/etc


###########################################################
# Set some reasonable defaults
###########################################################
# Default number of lines to crop off the top
TURBO = 213
MPEGQUALITY = 1800
MYTHRECORDINGSPATH = ['/var/lib/mythtv/recordings', '/usr/local/mythtv/recordings', '.']


# WORKDIRS is A list of directories we can use for temporary files, we will check if the directory exists and has
# adequate space and use the first directory from the list with enough room.
WORKDIRS = ['/tmp', '/work', '/var/lib/mythtv/recordings', '/usr/local/mythtv/recordings']


class video(object):
    def __init__(
            self, filename='', workdir='/var/lib/mythtv/recordings', logfile='/tmp/cleanupvideo.out',
            horizcrop=1, horizcroppercent=0, dbhost='localhost', dbuser='mythtv', dbpass='<PASSWORD>'
    ):
        self.filename = filename
        self.width = 0
        self.height = 0
        self.framerate = 0
        self.currentcrop = ''
        self.frames = 0
        self.horizcrop = horizcrop
        self.horizcroppercent = horizcroppercent
        self.croptop = 0
        self.cropleft = 0
        self.cropright = 0
        self.cropbottom = 0
        self.operationnumber = 0
        self.logfile = logfile
        self.workdir = workdir
        self.dbhost = dbhost
        self.dbuser = dbuser
        self.dbpass = dbpass

    def detectcropvalues(self, frames=0, horizcrop=-1, horizcroppercent=-1, turbo=TURBO):
        segmentsecs = 5
        if turbo < segmentsecs:
            turbo = segmentsecs * 2
        if horizcrop != -1:
            # This value is the number of 16 line blocks to crop from the top/bottom, so we need to multiply by 16
            self.horizcrop = int(horizcrop) * 16
        if horizcroppercent != -1:
            self.horizcroppercent = horizcroppercent
        if frames != 0:
            self.frames = frames
        cropsizes = {}
        crop = ''
        WIDTH = 0
        HEIGHT = 0
        edlfilename = '%s/cleanvideo_tmp.edl' % self.workdir
        edifilename = tempfile.mktemp('.edl', 'cleanupvideo_', self.workdir)
        fh = open(edifilename, 'w')
        for sec in range(1, 14000, turbo):
            fh.write('%d %d 0\n' % (sec, sec + (turbo - segmentsecs)))
        fh.close()
        if frames == 0:
            command = 'mplayer -quiet -edl %s -benchmark -nosound -vf cropdetect=24:16 -vo null %s 2> /dev/null' % (
            edifilename, self.filename)
        else:
            command = 'mplayer -quiet -edl %s -benchmark -nosound -vf cropdetect=24:16 -frames %d -vo null %s 2>/dev/null' % (
            edifilename, frames, self.filename)
        logging.debug('Running command: %s' % command)
        for line in os.popen(command).readlines():
            splitline = line.strip().split()
            if len(splitline) > 3 and splitline[0] == 'VIDEO:':
                WIDTH = splitline[2].split('x')[0]
                HEIGHT = splitline[2].split('x')[1]
                self.framerate = splitline[5]
            if len(splitline) > 7 and splitline[0] == '[CROP]':
                crop = splitline[8][5:-2]
            try:
                cropsizes[crop] = cropsizes[crop] + 1
            except KeyError:
                cropsizes[crop] = 1
        currentcropcount = 0
        currentcrop = ''
        for crop in cropsizes.keys():
            if cropsizes[crop] > currentcropcount:
                currentcrop = crop
                currentcropcount = cropsizes[crop]
        if len(currentcrop):
            splitcrop = currentcrop.split(':')
            height = int(splitcrop[1])
            evenheight = (height / 16) * 16
            remainder = height - evenheight
            if horizcroppercent > 0:
                horizcrop = int(float(height) * (float(horizcroppercent) * .01))
            if remainder == horizcrop:
                currentcrop = '%d:%d:%d:%d' % (
                int(splitcrop[0]), int(splitcrop[1]) - (horizcrop / 2), int(splitcrop[2]),
                int(splitcrop[3]) + (horizcrop / 2))
            if remainder > horizcrop:
                currentcrop = '%d:%d:%d:%d' % (
                int(splitcrop[0]), int(splitcrop[1]) - (remainder / 2), int(splitcrop[2]),
                int(splitcrop[3]) + (remainder / 2))
            if remainder < horizcrop:
                currentcrop = '%d:%d:%d:%d' % (
                int(splitcrop[0]), evenheight - 16, int(splitcrop[2]), int(splitcrop[3]) + 8)
        self.width = int(WIDTH)
        self.height = int(HEIGHT)
        self.currentcrop = currentcrop
        if len(currentcrop):
            cropvalues = currentcrop.split(':')
            self.croptop = int(cropvalues[3])
            self.cropleft = int(cropvalues[2])
            self.cropright = self.width - (self.cropleft + int(cropvalues[0]))
            self.cropbottom = self.height - (self.croptop + (int(cropvalues[1])))
            if self.cropbottom < 0:
                self.cropbottom = 0
        logging.debug(
            'Crop borders are: %s %s %s %s %s %s' % (
                self.width, self.height, self.croptop, self.cropleft, self.cropbottom, self.cropright
            )
        )
        os.remove(edifilename)

    def createlockfile(self, completed=False):
        fh = open("%s.cleanupvideoran" % self.filename, 'w')
        fh.write('filename: %s\n' % self.filename)
        fh.write('hostname: %s\n' % socket.gethostname())
        #fh.write('cutcommercials: %d\n' % self.cutcommercials)
        #fh.write('cropvideo: %d\n' % self.cropvideo)
        fh.write('frames: %d\n' % self.frames)
        fh.write('horizcrop: %d\n' % self.horizcrop)
        fh.write('horizcroppercent: %d\n' % self.horizcroppercent)
        fh.write('completed: %d\n' % completed)
        fh.close()

    def deletelockfile(self):
        os.remove('%s.cleanupvideoran' % self.filename)

    def checklockfile(self):
        rc = 9
        try:
            fh = open("%s.cleanupvideoran" % self.filename, 'r')
        except IOError:
            return (2)
        for line in fh.readlines():
            if len(line):
                splitline = line.strip().split()
                if splitline[0] == 'completed:':
                    try:
                        rc = int(splitline[1])
                    except ValueError:
                        if splitline[1] == 'True':
                            rc = 1
                        else:
                            rc = 0
        return (rc)

    def swapfiles(self, keeporiginal=False):
        logging.debug(
            'Swapping files: %s/new.%s <-> %s' % (self.workdir, os.path.basename(self.filename), self.filename))
        if keeporiginal:
            for backupnumber in range(self.operationnumber, 999):
                try:
                    fh = open('%s/%s.%d' % (self.workdir, os.path.basename(self.filename), backupnumber))
                    fh.close()
                except IOError:
                    break
            try:
                fh = open('%s/new.%s' % (self.workdir, os.path.basename(self.filename)))
            except IOError:
                print 'New file did not get created'
                return (1)
            fh.close()
            os.rename(self.filename, '%s.old.%d' % (self.filename, backupnumber))
            # This doesn't work if the files aren't on the same device
            #os.rename('%s/new.%s' % (self.workdir,os.path.basename(self.filename),self.filename))
            os.system('mv %s/new.%s %s' % (self.workdir, os.path.basename(self.filename), self.filename))
        else:
            try:
                fh = open('%s/new.%s' % (self.workdir, os.path.basename(self.filename)))
            except IOError:
                print 'New file did not get created'
                return (1)
            fh.close()
            os.remove(self.filename)
            # This doesn't work if both files aren't on the same filesystem
            #os.rename('%s/new.%s' % (self.workdir,os.path.basename(self.filename)), self.filename)
            os.system('mv %s/new.%s %s' % (self.workdir, os.path.basename(self.filename), self.filename))
        self.operationnumber = self.operationnumber + 1
        return (0)

    def crop(self, keeporiginal=False, format='mp4', resize=False, mpegquality=MPEGQUALITY):
        if self.currentcrop == '':
            logging.debug('No crop boundries for this video object, running cropdetect')
            self.detectcropvalues()
        if self.croptop == 0 and self.cropbottom == 0 and self.cropleft == 0 and self.cropright == 0:
            print 'No crop borders detected'
            return (0)
            # I am overriding the framerate here since it doesn't always get the correct framerate??
        self.framerate = 29.97
        # Try it first just cropping, if it doesn't work try specifying the target format
        logging.debug(
            'Running command: avconv >> %s 2>&1 -y -i "%s" -cropleft %d -cropright %d -croptop %d -cropbottom %d '
            '-aspect 16:9 -f mp4 -b %dkb "%s/new.%s"' % (
                self.logfile, self.filename, self.cropleft, self.cropright, self.croptop, self.cropbottom, mpegquality,
                self.workdir, os.path.basename(self.filename)
            )
        )
        rc = os.system(
            'avconv >> %s 2>&1 -y -i "%s" -cropleft %d -cropright %d -croptop %d -cropbottom %d -aspect 16:9 '
            '-f mp4 -b %dkb "%s/new.%s"' % (
                self.logfile, self.filename, self.cropleft, self.cropright, self.croptop, self.cropbottom, mpegquality,
                self.workdir, os.path.basename(self.filename)
            )
        ) >> 8
        if rc == 1:
            logging.debug(
                'Running command: avconv >> %s 2>&1 -y -i "%s" -cropleft %d -cropright %d -croptop %d '
                '-cropbottom %d -target ntsc-dvd -aspect 16:9 -b %dkb "%s/new.%s"' % (
                self.logfile, self.filename, self.cropleft, self.cropright, self.croptop, self.cropbottom, mpegquality,
                self.workdir, os.path.basename(self.filename)))
            rc = os.system(
                'avconv >> %s 2>&1 -y -i "%s" -cropleft %d -cropright %d -croptop %d -cropbottom %d -target '
                'ntsc-dvd -aspect 16:9 -b %dkb "%s/new.%s"' % (
                self.logfile, self.filename, self.cropleft, self.cropright, self.croptop, self.cropbottom, mpegquality,
                self.workdir, os.path.basename(self.filename))) >> 8
            if rc == 1:
                logging.debug('Crop failed, returning failure code')
                return False
        self.swapfiles(keeporiginal)
        return True

    def cutcommercials(self, keeporiginal=False):
    # Commercial cutting is experimental and may not work as intended
        rc = os.system('mythcommflag > /dev/null 2>&1 -f %s' % self.filename) >> 8
        if rc > 126:
            print 'Commercial flagging failed for filename %s with error %d' % (self.filename, rc)
            return (1)
        rc = os.system('mythcommflag --gencutlist -f %s' % self.filename) >> 8
        if rc != 0:
            print 'Copying cutlist failed for %s with error %d' % (self.filename, rc)
            return (1)
        temppath = ''
        rc = os.system('mythtranscode --honorcutlist -i "%s" -o "%s/new.%s"' % (
        self.filename, self.workdir, os.path.basename(self.filename))) >> 8
        if rc != 0:
            print 'Cut commercials and transcoding failed for %s with error %d' % (self.filename, rc)
            return (2)<commit_msg>switch to avconv<commit_after>#!/usr/bin/python
import os
import sys
import tempfile
import getopt
import re
import socket
import logging
import argparse
import xml.etree.ElementTree as ET

logging.basicConfig(level=logging.DEBUG)
try:
    import MySQLdb
except ImportError:
    print 'This program requires the python MySQLdb module'
    sys.exit(10)


#Todo
# 1. Add code to verify prereqs including:
#   MySQLdb python module
#   mplayer
#   avconv
#   mythtranscode
# 2. Add code to transcode to a second file for example use on ipod/cell-phone/etc


###########################################################
# Set some reasonable defaults
###########################################################
# Default number of lines to crop off the top
TURBO = 213
MPEGQUALITY = 1800
MYTHRECORDINGSPATH = ['/var/lib/mythtv/recordings', '/usr/local/mythtv/recordings', '.']


# WORKDIRS is A list of directories we can use for temporary files, we will check if the directory exists and has
# adequate space and use the first directory from the list with enough room.
WORKDIRS = ['/tmp', '/work', '/var/lib/mythtv/recordings', '/usr/local/mythtv/recordings']


class video(object):
    def __init__(
            self, filename='', workdir='/var/lib/mythtv/recordings', logfile='/tmp/cleanupvideo.out',
            horizcrop=1, horizcroppercent=0, dbhost='localhost', dbuser='mythtv', dbpass='<PASSWORD>'
    ):
        self.filename = filename
        self.width = 0
        self.height = 0
        self.framerate = 0
        self.currentcrop = ''
        self.frames = 0
        self.horizcrop = horizcrop
        self.horizcroppercent = horizcroppercent
        self.croptop = 0
        self.cropleft = 0
        self.cropright = 0
        self.cropbottom = 0
        self.operationnumber = 0
        self.logfile = logfile
        self.workdir = workdir
        self.dbhost = dbhost
        self.dbuser = dbuser
        self.dbpass = dbpass

    def detectcropvalues(self, frames=0, horizcrop=-1, horizcroppercent=-1, turbo=TURBO):
        segmentsecs = 5
        if turbo < segmentsecs:
            turbo = segmentsecs * 2
        if horizcrop != -1:
            # This value is the number of 16 line blocks to crop from the top/bottom, so we need to multiply by 16
            self.horizcrop = int(horizcrop) * 16
        if horizcroppercent != -1:
            self.horizcroppercent = horizcroppercent
        if frames != 0:
            self.frames = frames
        cropsizes = {}
        crop = ''
        WIDTH = 0
        HEIGHT = 0
        edlfilename = '%s/cleanvideo_tmp.edl' % self.workdir
        edifilename = tempfile.mktemp('.edl', 'cleanupvideo_', self.workdir)
        fh = open(edifilename, 'w')
        for sec in range(1, 14000, turbo):
            fh.write('%d %d 0\n' % (sec, sec + (turbo - segmentsecs)))
        fh.close()
        if frames == 0:
            command = 'mplayer -quiet -edl %s -benchmark -nosound -vf cropdetect=24:16 -vo null %s 2> /dev/null' % (
            edifilename, self.filename)
        else:
            command = 'mplayer -quiet -edl %s -benchmark -nosound -vf cropdetect=24:16 -frames %d -vo null %s 2>/dev/null' % (
            edifilename, frames, self.filename)
        logging.debug('Running command: %s' % command)
        for line in os.popen(command).readlines():
            splitline = line.strip().split()
            if len(splitline) > 3 and splitline[0] == 'VIDEO:':
                WIDTH = splitline[2].split('x')[0]
                HEIGHT = splitline[2].split('x')[1]
                self.framerate = splitline[5]
            if len(splitline) > 7 and splitline[0] == '[CROP]':
                crop = splitline[8][5:-2]
            try:
                cropsizes[crop] = cropsizes[crop] + 1
            except KeyError:
                cropsizes[crop] = 1
        currentcropcount = 0
        currentcrop = ''
        for crop in cropsizes.keys():
            if cropsizes[crop] > currentcropcount:
                currentcrop = crop
                currentcropcount = cropsizes[crop]
        if len(currentcrop):
            splitcrop = currentcrop.split(':')
            height = int(splitcrop[1])
            evenheight = (height / 16) * 16
            remainder = height - evenheight
            if horizcroppercent > 0:
                horizcrop = int(float(height) * (float(horizcroppercent) * .01))
            if remainder == horizcrop:
                currentcrop = '%d:%d:%d:%d' % (
                int(splitcrop[0]), int(splitcrop[1]) - (horizcrop / 2), int(splitcrop[2]),
                int(splitcrop[3]) + (horizcrop / 2))
            if remainder > horizcrop:
                currentcrop = '%d:%d:%d:%d' % (
                int(splitcrop[0]), int(splitcrop[1]) - (remainder / 2), int(splitcrop[2]),
                int(splitcrop[3]) + (remainder / 2))
            if remainder < horizcrop:
                currentcrop = '%d:%d:%d:%d' % (
                int(splitcrop[0]), evenheight - 16, int(splitcrop[2]), int(splitcrop[3]) + 8)
        self.width = int(WIDTH)
        self.height = int(HEIGHT)
        self.currentcrop = currentcrop
        if len(currentcrop):
            cropvalues = currentcrop.split(':')
            self.croptop = int(cropvalues[3])
            self.cropleft = int(cropvalues[2])
            self.cropright = self.width - (self.cropleft + int(cropvalues[0]))
            self.cropbottom = self.height - (self.croptop + (int(cropvalues[1])))
            if self.cropbottom < 0:
                self.cropbottom = 0
        logging.debug(
            'Crop borders are: %s %s %s %s %s %s' % (
                self.width, self.height, self.croptop, self.cropleft, self.cropbottom, self.cropright
            )
        )
        os.remove(edifilename)

    def createlockfile(self, completed=False):
        fh = open("%s.cleanupvideoran" % self.filename, 'w')
        fh.write('filename: %s\n' % self.filename)
        fh.write('hostname: %s\n' % socket.gethostname())
        #fh.write('cutcommercials: %d\n' % self.cutcommercials)
        #fh.write('cropvideo: %d\n' % self.cropvideo)
        fh.write('frames: %d\n' % self.frames)
        fh.write('horizcrop: %d\n' % self.horizcrop)
        fh.write('horizcroppercent: %d\n' % self.horizcroppercent)
        fh.write('completed: %d\n' % completed)
        fh.close()

    def deletelockfile(self):
        os.remove('%s.cleanupvideoran' % self.filename)

    def checklockfile(self):
        rc = 9
        try:
            fh = open("%s.cleanupvideoran" % self.filename, 'r')
        except IOError:
            return (2)
        for line in fh.readlines():
            if len(line):
                splitline = line.strip().split()
                if splitline[0] == 'completed:':
                    try:
                        rc = int(splitline[1])
                    except ValueError:
                        if splitline[1] == 'True':
                            rc = 1
                        else:
                            rc = 0
        return (rc)

    def swapfiles(self, keeporiginal=False):
        logging.debug(
            'Swapping files: %s/new.%s <-> %s' % (self.workdir, os.path.basename(self.filename), self.filename))
        if keeporiginal:
            for backupnumber in range(self.operationnumber, 999):
                try:
                    fh = open('%s/%s.%d' % (self.workdir, os.path.basename(self.filename), backupnumber))
                    fh.close()
                except IOError:
                    break
            try:
                fh = open('%s/new.%s' % (self.workdir, os.path.basename(self.filename)))
            except IOError:
                print 'New file did not get created'
                return (1)
            fh.close()
            os.rename(self.filename, '%s.old.%d' % (self.filename, backupnumber))
            # This doesn't work if the files aren't on the same device
            #os.rename('%s/new.%s' % (self.workdir,os.path.basename(self.filename),self.filename))
            os.system('mv %s/new.%s %s' % (self.workdir, os.path.basename(self.filename), self.filename))
        else:
            try:
                fh = open('%s/new.%s' % (self.workdir, os.path.basename(self.filename)))
            except IOError:
                print 'New file did not get created'
                return (1)
            fh.close()
            os.remove(self.filename)
            # This doesn't work if both files aren't on the same filesystem
            #os.rename('%s/new.%s' % (self.workdir,os.path.basename(self.filename)), self.filename)
            os.system('mv %s/new.%s %s' % (self.workdir, os.path.basename(self.filename), self.filename))
        self.operationnumber = self.operationnumber + 1
        return (0)

    def crop(self, keeporiginal=False, format='mp4', resize=False, mpegquality=MPEGQUALITY):
        if self.currentcrop == '':
            logging.debug('No crop boundries for this video object, running cropdetect')
            self.detectcropvalues()
        if self.croptop == 0 and self.cropbottom == 0 and self.cropleft == 0 and self.cropright == 0:
            print 'No crop borders detected'
            return (0)
            # I am overriding the framerate here since it doesn't always get the correct framerate??
        self.framerate = 29.97
        # Try it first just cropping, if it doesn't work try specifying the target format
        logging.debug(
            'Running command: avconv >> %s 2>&1 -y -i "%s" -cropleft %d -cropright %d -croptop %d -cropbottom %d '
            '-aspect 16:9 -f mp4 -b %dkb "%s/new.%s"' % (
                self.logfile, self.filename, self.cropleft, self.cropright, self.croptop, self.cropbottom, mpegquality,
                self.workdir, os.path.basename(self.filename)
            )
        )
        rc = os.system(
            'avconv >> %s 2>&1 -y -i "%s" -cropleft %d -cropright %d -croptop %d -cropbottom %d -aspect 16:9 '
            '-f mp4 -b %dkb "%s/new.%s"' % (
                self.logfile, self.filename, self.cropleft, self.cropright, self.croptop, self.cropbottom, mpegquality,
                self.workdir, os.path.basename(self.filename)
            )
        ) >> 8
        if rc == 1:
            logging.debug(
                'Running command: avconv >> %s 2>&1 -y -i "%s" -cropleft %d -cropright %d -croptop %d '
                '-cropbottom %d -target ntsc-dvd -aspect 16:9 -b %dkb "%s/new.%s"' % (
                self.logfile, self.filename, self.cropleft, self.cropright, self.croptop, self.cropbottom, mpegquality,
                self.workdir, os.path.basename(self.filename)))
            rc = os.system(
                'avconv >> %s 2>>/tmp/log.out -y -i "%s" -cropleft %d -cropright %d -croptop %d -cropbottom %d -target '
                'ntsc-dvd -aspect 16:9 -b %dkb "%s/new.%s"' % (
                self.logfile, self.filename, self.cropleft, self.cropright, self.croptop, self.cropbottom, mpegquality,
                self.workdir, os.path.basename(self.filename))) >> 8
            if rc == 1:
                logging.debug('Crop failed, returning failure code')
                return False
        self.swapfiles(keeporiginal)
        return True

    def cutcommercials(self, keeporiginal=False):
    # Commercial cutting is experimental and may not work as intended
        rc = os.system('mythcommflag > /dev/null 2>&1 -f %s' % self.filename) >> 8
        if rc > 126:
            print 'Commercial flagging failed for filename %s with error %d' % (self.filename, rc)
            return (1)
        rc = os.system('mythcommflag --gencutlist -f %s' % self.filename) >> 8
        if rc != 0:
            print 'Copying cutlist failed for %s with error %d' % (self.filename, rc)
            return (1)
        temppath = ''
        rc = os.system('mythtranscode --honorcutlist -i "%s" -o "%s/new.%s"' % (
        self.filename, self.workdir, os.path.basename(self.filename))) >> 8
        if rc != 0:
            print 'Cut commercials and transcoding failed for %s with error %d' % (self.filename, rc)
            return (2)
{'fraction_non_alphanumeric': 0.13504889772915632, 'fraction_numerical': 0.01371622741587933, 'mean_word_length': 1.9394640682095006, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2059428', 'n_tokens_mistral': 7917, 'n_tokens_neox': 7455, 'n_words': 1873}