<commit_before>import sys

import datetime

class Unbuffered:
   def __init__(self, stream):
       self.stream = stream
   def write(self, data):
       self.stream.write(data)
       self.stream.flush()
   def __getattr__(self, attr):
       return getattr(self.stream, attr)

def progress_meter(starting_time, chrm, pos, bp_processed, total_bp_in_dataset):
    
    #sys.stdout=Unbuffered(sys.stdout) # make sure writing to std out isn't buffered

    ct = datetime.datetime.now()
    elapsed = (datetime.datetime.now() - starting_time)

    proportion_processed = bp_processed/float(total_bp_in_dataset)

    if elapsed.seconds % 30 == 0 and elapsed.seconds > 10:

        "INFO  19:29:31,683 ProgressMeter - GL343193.1:820101\t1.79e+09\t2.3 h\t4.6 s\t99.2%\t2.3 h\t62.5 s "
        status = "INFO  {} ProgressMeter - {}:{:,} {:.3} {:.2%}\n".format(ct.time(), chrm, pos, elapsed, proportion_processed)
        sys.stdout.write(status)
        return ct

    else:
        return starting_time<commit_msg>Added hackey function to test of vcf file is gzipped. <commit_after>import sys
import gzip
import datetime

class Unbuffered:
   def __init__(self, stream):
       self.stream = stream
   def write(self, data):
       self.stream.write(data)
       self.stream.flush()
   def __getattr__(self, attr):
       return getattr(self.stream, attr)


def open_vcf(args):
    if args.input.endswith('.gz') == True: # To Do: This is hacky.
        fin = gzip.open(args.input,'rb')
    else:
        fin = open(args.input, 'rU')

    return fin



def progress_meter(starting_time, chrm, pos, bp_processed, total_bp_in_dataset):
    
    #sys.stdout=Unbuffered(sys.stdout) # make sure writing to std out isn't buffered

    ct = datetime.datetime.now()
    elapsed = (datetime.datetime.now() - starting_time)

    proportion_processed = bp_processed/float(total_bp_in_dataset)

    if elapsed.seconds % 30 == 0 and elapsed.seconds > 10:

        "INFO  19:29:31,683 ProgressMeter - GL343193.1:820101\t1.79e+09\t2.3 h\t4.6 s\t99.2%\t2.3 h\t62.5 s "
        status = "INFO  {} ProgressMeter - {}:{:,} {:.3} {:.2%}\n".format(ct.time(), chrm, pos, elapsed, proportion_processed)
        sys.stdout.write(status)
        return ct

    else:
        return starting_time
{'fraction_non_alphanumeric': 0.12859036676977464, 'fraction_numerical': 0.040653999116217414, 'mean_word_length': 3.0500894454382825, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3452364', 'n_tokens_mistral': 861, 'n_tokens_neox': 761, 'n_words': 177}