<jupyter_start><jupyter_text>version 1<jupyter_code># version 1
import os
from itertools import groupby, chain

def flatten(sm):
    return chain.from_iterable(sm)

def sliding(supermer: str, k: int):
    return (supermer[i : i + k] for i in range(len(supermer) - k + 1))

def countKmers(k: int, inputFile: str, bucket: str):
    if not inputFile.endswith('.txt'):
        raise DiscountCustomError("FileFormatError : required .txt file", inputFile.split('/')[-1], "XXX.txt")
    else:
        try:
            path = bucket.rsplit('/', 1)[0]
            print(path)
            os.mkdir(bucket.rsplit('/', 1)[0])
        except OSError:
            pass
    
    with open(inputFile) as f:
        lines = map(lambda _: _.rstrip('\n').split('\t'), f)
        
        print(bucket)
        with open(bucket, 'w+') as f:
            for minimizer, thisBin in groupby(lines, lambda _: _[0]):
                supermersThisBin = map(lambda _: _[1], thisBin)

                slide = map(lambda sm: sliding(sm, k), supermersThisBin)
                flat = flatten(slide)
                kmers = iter(sorted(flat))

                # kmers = sorted(flat)
                # print(minimizer, kmers)

                last = next(kmers)
                count = 1

                for _ in kmers:
                    if _ == last:
                        count += 1
                    else:
                        f.write(f'{last}\t{count}\n')
                        last = _
                        count = 1

                f.write(f'{last}\t{count}\n')
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq/kmers.txt'
k = 28

%lprun -T 100k_freq.txt -f countKmers countKmers(k, file, bucket)<jupyter_output>C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq
C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq/kmers.txt

*** Profile printout saved to text file '100k_freq.txt'. 
Wall time: 3min 45s<jupyter_text>version 2<jupyter_code># version 2

import os
from more_itertools import flatten, peekable, run_length, bucket as buk

def sliding(supermer: str, k: int):
    return (supermer[i : i + k] for i in range(len(supermer) - k + 1))

def takewhile(lines, m):
    while lines and lines.peek()[0] == m:
        yield next(lines)[1]

def countKmers(k: int, inputFile: str, bucket: str):
    if not inputFile.endswith('.txt'):
        raise DiscountCustomError("FileFormatError : required .txt file", inputFile.split('/')[-1], "XXX.txt")
    else:
        try:
            path = bucket.rsplit('/', 1)[0]
            print(path)
            os.mkdir(bucket.rsplit('/', 1)[0])
        except OSError:
            pass
    
    with open(inputFile, 'r') as f:
        lines = peekable(map(lambda _: _.rstrip('\n').split('\t'), f))
#         lines = buk(map(lambda _: _.rstrip('\n').split('\t'), f), lambda _: _[0])
#         for _ in lines:
#             print(list(lines[_]))
#         print(list(lines))

        result = []
        while lines:
            minimizer = lines.peek()[0]
            supermersThisBin = takewhile(lines, minimizer)

            kmers = peekable(sorted(flatten(map(lambda _: sliding(_, k), supermersThisBin))))
            
            count = 0
            lastKmer = kmers.peek()
            while kmers:
                if kmers.peek() == lastKmer:
                    count += 1
                    next(kmers)
                else:
                    result.append('%s\t%s'%(lastKmer, count))
                    count = 1
                    lastKmer = next(kmers)
                    
            result.append('%s\t%s'%(lastKmer, count))

        open(bucket, 'w').write(''.join('%s\n'%(_) for _ in result))
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt' 
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq/kmers2.txt'
k = 28

%lprun -T 100k_freq2.txt -f countKmers countKmers(k, file, bucket)<jupyter_output>C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq

*** Profile printout saved to text file '100k_freq2.txt'. 
Wall time: 4min 49s<jupyter_text>version 3<jupyter_code># version 3
import os
from itertools import groupby, chain

def flatten(sm):
    return chain.from_iterable(sm)

def sliding(supermer: str, k: int):
    return (supermer[i : i + k] for i in range(len(supermer) - k + 1))

def countKmers(k: int, inputFile: str, bucket: str):
    if not inputFile.endswith('.txt'):
        raise DiscountCustomError("FileFormatError : required .txt file", inputFile.split('/')[-1], "XXX.txt")
    else:
        try:
            path = bucket.rsplit('/', 1)[0]
            print(path)
            os.mkdir(bucket.rsplit('/', 1)[0])
        except OSError:
            pass
    
    with open(inputFile) as f:
        lines = map(lambda _: _.rstrip('\n').split('\t'), f)
        
        result = []
        for minimizer, thisBin in groupby(lines, lambda _: _[0]):
            supermersThisBin = map(lambda _: _[1], thisBin)

            slide = map(lambda sm: sliding(sm, k), supermersThisBin)
            flat = flatten(slide)
            kmers = iter(sorted(flat))

            # kmers = sorted(flat)
            # print(minimizer, kmers)

            last = next(kmers)
            count = 1

            for _ in kmers:
                if _ == last:
                    count += 1
                else:
                    result.append('%s\t%s'%(last, count))
                    last = _
                    count = 1

            result.append('%s\t%s'%(last, count))
        
        open(bucket, 'w').write(''.join('%s\n'%(_) for _ in result))
        
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt' 
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq/kmers3.txt'
k = 28

%lprun -T 100k_freq3.txt -f countKmers countKmers(k, file, bucket)<jupyter_output>C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_100k_freq

*** Profile printout saved to text file '100k_freq3.txt'. 
Wall time: 4min<jupyter_text>version 4<jupyter_code># version 4
import os
from more_itertools import peekable
from collections import Counter

def countKmers(k: int, inputFile: str, bucket: str):
    if not inputFile.endswith('.txt'):
        raise DiscountCustomError("FileFormatError : required .txt file", inputFile.split('/')[-1], "XXX.txt")
    else:
        try:
            path = bucket.rsplit('/', 1)[0]
            print(path)
            os.mkdir(bucket.rsplit('/', 1)[0])
        except OSError:
            pass
    
    def takeWhile(lines, m):
        _thisBin = Counter()

        while lines and lines.peek()[0] == m:
            _thisBin.update(next(lines)[1])

        return _thisBin

    def sliding(supermer):
        return [supermer[i : i + k] for i in range(len(supermer) - k + 1)]

    def kmer_fun(x):
        return x[0], sliding(x[1])
        
    with open(inputFile) as f:
        lines = peekable(map(lambda _: kmer_fun(_.rstrip('\n').split('\t')), f))
        
        with open(bucket, 'w+') as demo:
            while lines:
                minimizer = lines.peek()[0]
                supermersThisBin = takeWhile(lines, minimizer)
                demo.writelines(['%s\t%s\n'%(_, supermersThisBin[_]) for _ in supermersThisBin])
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_demo/freq.txt'
k = 28

%lprun -T freq.txt -f countKmers countKmers(k, file, bucket)
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_ufreq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_demo/ufreq.txt'
k = 28

%lprun -T ufreq.txt -f countKmers countKmers(k, file, bucket)
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_freq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_prof/freq.txt'
k = 28

%prun -D freq.prof countKmers(k, file, bucket)
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_100k_ufreq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_prof/ufreq.txt'
k = 28

%prun -D ufreq.prof countKmers(k, file, bucket)
%%time
file = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/output/sort_1M_ufreq.txt'
bucket = 'C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_1M/ufreq.txt'
k = 28

%lprun -T ufreq.txt -f countKmers countKmers(k, file, bucket)<jupyter_output>C:/Users/umesh/Desktop/Discount-In-Python/Discount/bucket_1M

*** Profile printout saved to text file 'ufreq.txt'. 
Wall time: 9min 57s
{'fraction_non_alphanumeric': 0.14127581878352163, 'fraction_numerical': 0.01700330936893758, 'mean_word_length': 2.351434034416826, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '459317', 'license': 'MIT', 'n_tokens_mistral': 3187, 'n_tokens_neox': 2957, 'n_words': 611, 'path': 'notebook/Discount In Python.ipynb', 'repo_name': 'Umesh-JNU/DiscountPy', 'size': '8763'}