<commit_before>from requests import post, codes, Timeout
from json import dumps
from random import sample
import subprocess
import os

from svtplay_dl.log import log
from svtplay_dl.utils import which


class postprocess(object):
    def __init__(self, stream, options, subfixes = []):
        self.stream = stream
        self.merge_subtitle = options.merge_subtitle
        self.external_subtitle = options.subtitle
        self.get_all_subtitles = options.get_all_subtitles
        self.subfixes = subfixes
        self.detect = None
        for i in ["ffmpeg", "avconv"]:
            self.detect = which(i)
            if self.detect:
                break

    def sublanguage(self):
        # parse() function partly borrowed from a guy on github. /thanks!
        # https://github.com/riobard/srt.py/blob/master/srt.py
        def parse(self):
            def parse_block(block):
                lines   = block.strip('-').split('\n')
                txt     = '\r\n'.join(lines[2:])
                return txt
            return list(map(parse_block,
                        open(self).read().strip().replace('\r', '').split('\n\n')))
        
        def query(self):
            random_sentences = ' '.join(sample(parse(self),8)).replace('\r\n', '')
            url = 'https://whatlanguage.herokuapp.com'
            payload = { "query": random_sentences }
            headers = {'content-type': 'application/json'} # Note: requests handles json from version 2.4.2 and onwards so i use json.dumps for now.
            try:
                r = post(url, data=dumps(payload), headers=headers, timeout=30) # Note: reasonable timeout i guess? svtplay-dl is mainly used while multitasking i presume, and it is heroku after all (fast enough)
                if r.status_code == codes.ok:
                    response = r.json()
                    return response['language']
                else:
                    log.error("Server error appeared. Setting language as undetermined.")
                    return 'und'
            except Timeout:<commit_msg>postprocess: return und if we cant get the right language<commit_after>from requests import post, codes, Timeout
from json import dumps
from random import sample
import subprocess
import os

from svtplay_dl.log import log
from svtplay_dl.utils import which


class postprocess(object):
    def __init__(self, stream, options, subfixes = []):
        self.stream = stream
        self.merge_subtitle = options.merge_subtitle
        self.external_subtitle = options.subtitle
        self.get_all_subtitles = options.get_all_subtitles
        self.subfixes = subfixes
        self.detect = None
        for i in ["ffmpeg", "avconv"]:
            self.detect = which(i)
            if self.detect:
                break

    def sublanguage(self):
        # parse() function partly borrowed from a guy on github. /thanks!
        # https://github.com/riobard/srt.py/blob/master/srt.py
        def parse(self):
            def parse_block(block):
                lines   = block.strip('-').split('\n')
                txt     = '\r\n'.join(lines[2:])
                return txt
            return list(map(parse_block,
                        open(self).read().strip().replace('\r', '').split('\n\n')))
        
        def query(self):
            random_sentences = ' '.join(sample(parse(self),8)).replace('\r\n', '')
            url = 'https://whatlanguage.herokuapp.com'
            payload = { "query": random_sentences }
            headers = {'content-type': 'application/json'} # Note: requests handles json from version 2.4.2 and onwards so i use json.dumps for now.
            try:
                r = post(url, data=dumps(payload), headers=headers, timeout=30) # Note: reasonable timeout i guess? svtplay-dl is mainly used while multitasking i presume, and it is heroku after all (fast enough)
                if r.status_code == codes.ok:
                    try:
                        response = r.json()
                        return response['language']
                    except TypeError:
                        return 'und'
                else:
                    log.error("Server error appeared. Setting language as undetermined.")
                    return 'und'
            except Timeout:
{'fraction_non_alphanumeric': 0.10630841121495327, 'fraction_numerical': 0.0032710280373831778, 'mean_word_length': 1.9142273655547992, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '115565', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1145, 'n_words': 323}