<commit_before>class SrtItem(object):
    def __init__(self):
        self.index = -1        
        self.start_time = ""
        self.start_millisecond = ""
        self.end_time = ""
        self.end_millisecond = ""
        self.text = []
    def __str__(self):
        return "{0},{1},{2},{3},{4},{5}".format(self.index, self.start_time, self.start_millisecond, self.end_time, self.end_millisecond, self.text)
    
def parse_next_line(next_line):
    line = next_line()    
    return line
    
def parse_srt_item(next_line, index):    
    try:
        srtItem = SrtItem() 
        srtItem.index = index
        next_index_string = str(index+1)
        timeLines = parse_next_line(next_line).split(" --> ")
        startTimeLine = timeLines[0]            
        endTimeLine = timeLines[1]
        srtItem.start_time = startTimeLine.split(",")[0]
        srtItem.start_millisecond = startTimeLine.split(",")[1]
        srtItem.end_time = endTimeLine.split(",")[0]
        srtItem.end_millisecond = endTimeLine.split(",")[1]
        line = parse_next_line(next_line)
        last_line_empty = False
        while True:
            if (len(line) == 0):
                last_line_empty = True                
            elif(len(line) > 0 and last_line_empty and line == next_index_string):                
                break                
            else:
                last_line_empty = False
                srtItem.text.append(line)                
            line = parse_next_line(next_line)        
        return srtItem        
    except StopIteration:
        print "file end reached."
        return None

def read_file_line(f):
    line = f.readline()
    if line == "":
        raise StopIteration
    return line[0:-2]    

import codecs
import chardet
import os

def detect_encoding(file_name):
    bytes = min(256, os.path.getsize(file_name))
    with open(file_name, 'rb') as f:
        header = f.read(bytes)
        if header.startswith(codecs.BOM_UTF8):
            return ('utf8', 1)
        elif header.startswith(codecs.BOM_UTF16):
            return ('utf16', 0)
        else:
            result = chardet.detect(header)    
            encoding = result['encoding']
            if encoding == "GB2312":
                encoding = "GB18030"
            return (encoding, 0)
        
def parse_srt(path): 
    result = detect_encoding(path)
    encoding = result[0]
    skip_bytes = result[1]
    next_line = lambda:read_file_line(f)
    with codecs.open(path, "r", encoding) as f:
        if skip_bytes != 0:
            f.read(skip_bytes)
        next_line()
        item = parse_srt_item(next_line, 1)        
        while item != None:<commit_msg>Fix starting with empty lines issue.<commit_after>class SrtItem(object):
    def __init__(self):
        self.index = -1        
        self.start_time = ""
        self.start_millisecond = ""
        self.end_time = ""
        self.end_millisecond = ""
        self.text = []
    def __str__(self):
        return "{0},{1},{2},{3},{4},{5}".format(self.index, self.start_time, self.start_millisecond, self.end_time, self.end_millisecond, self.text)
    
def parse_next_line(next_line):
    line = next_line()    
    return line
    
def parse_srt_item(next_line, index):    
    try:
        srtItem = SrtItem() 
        srtItem.index = index
        next_index_string = str(index+1)
        timeLines = parse_next_line(next_line).split(" --> ")
        startTimeLine = timeLines[0]            
        endTimeLine = timeLines[1]
        srtItem.start_time = startTimeLine.split(",")[0]
        srtItem.start_millisecond = startTimeLine.split(",")[1]
        srtItem.end_time = endTimeLine.split(",")[0]
        srtItem.end_millisecond = endTimeLine.split(",")[1]
        line = parse_next_line(next_line)
        last_line_empty = False
        while True:
            if (len(line) == 0):
                last_line_empty = True                
            elif(len(line) > 0 and last_line_empty and line == next_index_string):                
                break                
            else:
                last_line_empty = False
                srtItem.text.append(line)                
            line = parse_next_line(next_line)        
        return srtItem        
    except StopIteration:
        print "file end reached."
        return None

def read_file_line(f):
    line = f.readline()
    if line == "":
        raise StopIteration
    return line[0:-2]    

import codecs
import chardet
import os

def detect_encoding(file_name):
    bytes = min(256, os.path.getsize(file_name))
    with open(file_name, 'rb') as f:
        header = f.read(bytes)
        if header.startswith(codecs.BOM_UTF8):
            return ('utf8', 1)
        elif header.startswith(codecs.BOM_UTF16):
            return ('utf16', 0)
        else:
            result = chardet.detect(header)    
            encoding = result['encoding']
            if encoding == "GB2312":
                encoding = "GB18030"
            return (encoding, 0)
        
def parse_srt(path): 
    result = detect_encoding(path)
    encoding = result[0]
    skip_bytes = result[1]
    next_line = lambda:read_file_line(f)
    with codecs.open(path, "r", encoding) as f:
        if skip_bytes != 0:
            f.read(skip_bytes)
        while len(next_line()) == 0:
            continue
        item = parse_srt_item(next_line, 1)        
        while item != None:
{'fraction_non_alphanumeric': 0.10990626723028855, 'fraction_numerical': 0.015989707774306194, 'mean_word_length': 1.887002652519894, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2302035', 'n_tokens_mistral': 1722, 'n_tokens_neox': 1674, 'n_words': 333}