<commit_before>#! /usr/bin/env python3
import alignments
import re
import read
import pairedread
import junction
import bisect
import binaryIO
import math

from pympler import asizeof

import huffman

import os

class Compressor:
    aligned = None

    sectionLen = 100000
    exonChunkSize = 100
    junctionChunkSize = 50

    # 0 - zlib
    # 1 - lzma
    # 2 - bz2
    compressMethod = 0

    def __init__(self):   
        if self.compressMethod == 0:
            self.zlib = __import__('zlib')
        elif self.compressMethod == 1:
            self.lzma = __import__('lzma')
        elif self.compressMethod == 2:
            self.bz2 = __import__('bz2')

    def compress(self, samFilename, compressedFilename, binary=False, huffman=False):
        ''' Compresses the alignments to 2 files, one for unspliced and one for spliced

            file_prefix: Prefix for all output file names
        '''

        self.huffman = huffman

        # Read header
        header = ''
        with open(samFilename, 'r') as f:
            for line in f:
                if line[0] == '@':
                    header += line
                else:
                    break
        self.chromosomes = self.parseSAMHeader(header)
        self.aligned = alignments.Alignments(self.chromosomes)

        print('Finished processing alignments')
        print('Alignments size: %f\n' % (asizeof.asizeof(self.aligned)/1000000))

        with open(samFilename, 'r') as f:
            self.parseAlignments(f)
        
        if binary:
            # Write spliced and unspliced information
            with open(compressedFilename, 'wb') as f:                
                s = binaryIO.writeChroms(self.aligned.chromosomes)
                f.write(s)
                s, self.exonBytes = binaryIO.writeExons(self.aligned.exons)
                f.write(s)

                #size = os.stat('compressed/compressed_binary.txt').st_size
                #print('Chroms + Exons size: %d bytes' % size)

                #print('Computing junctions')
                junctions, maxReadLen = self.computeJunctions()

                print('%d junctions' % len(junctions))
                print('Junctions size: %f' % (asizeof.asizeof(junctions)/1000000))

                #print('Compressing unspliced')
                self.compressUnspliced(f, binary)
                #size = os.stat('compressed/compressed_binary.txt').st_size - size
                #print('Unspliced size: %d bytes' % size)

                #print('Compressing spliced')
                self.compressSpliced(junctions, maxReadLen, f, binary)
                #size = os.stat('compressed/compressed_binary.txt').st_size - size
                #print('Spliced size: %d bytes' % size)
            
            # Write exons and index information
            compressed = None
            with open(compressedFilename, 'rb') as f:
                compressed = f.read()
            with open(compressedFilename, 'wb') as f:
                self.writeIndexBinary(f)
                #size = os.stat('compressed/compressed_binary.txt').st_size
                #print('Index size: %d bytes' % size)<commit_msg>Mem profiling<commit_after>#! /usr/bin/env python3
import alignments
import re
import read
import pairedread
import junction
import bisect
import binaryIO
import math

from pympler import asizeof

import huffman

import os

class Compressor:
    aligned = None

    sectionLen = 100000
    exonChunkSize = 100
    junctionChunkSize = 50

    # 0 - zlib
    # 1 - lzma
    # 2 - bz2
    compressMethod = 0

    def __init__(self):   
        if self.compressMethod == 0:
            self.zlib = __import__('zlib')
        elif self.compressMethod == 1:
            self.lzma = __import__('lzma')
        elif self.compressMethod == 2:
            self.bz2 = __import__('bz2')

    def compress(self, samFilename, compressedFilename, binary=False, huffman=False):
        ''' Compresses the alignments to 2 files, one for unspliced and one for spliced

            file_prefix: Prefix for all output file names
        '''

        self.huffman = huffman

        # Read header
        header = ''
        with open(samFilename, 'r') as f:
            for line in f:
                if line[0] == '@':
                    header += line
                else:
                    break
        self.chromosomes = self.parseSAMHeader(header)
        self.aligned = alignments.Alignments(self.chromosomes)

        print('Finished processing alignments')
        print('Alignments size: %f\n' % (asizeof.asizeof(self.aligned)/1000000))

        with open(samFilename, 'r') as f:
            self.parseAlignments(f)
        
        if binary:
            # Write spliced and unspliced information
            with open(compressedFilename, 'wb') as f:                
                s = binaryIO.writeChroms(self.aligned.chromosomes)
                f.write(s)
                s, self.exonBytes = binaryIO.writeExons(self.aligned.exons)
                f.write(s)

                #size = os.stat('compressed/compressed_binary.txt').st_size
                #print('Chroms + Exons size: %d bytes' % size)

                #print('Computing junctions')
                junctions, maxReadLen = self.computeJunctions()

                print('%d junctions' % len(junctions))
                print('Junctions size: %f' % (asizeof.asizeof(junctions)/1000000))

                #print('Compressing unspliced')
                self.compressUnspliced(f, binary)
                #size = os.stat('compressed/compressed_binary.txt').st_size - size
                #print('Unspliced size: %d bytes' % size)

                print('Unspliced alignments size: %f\n' % (asizeof.asizeof(self.aligned.unspliced)/1000000))

                #print('Compressing spliced')
                self.compressSpliced(junctions, maxReadLen, f, binary)
                #size = os.stat('compressed/compressed_binary.txt').st_size - size
                #print('Spliced size: %d bytes' % size)
            
            # Write exons and index information
            compressed = None
            with open(compressedFilename, 'rb') as f:
                compressed = f.read()
            with open(compressedFilename, 'wb') as f:
                self.writeIndexBinary(f)
                #size = os.stat('compressed/compressed_binary.txt').st_size
                #print('Index size: %d bytes' % size)
{'fraction_non_alphanumeric': 0.09850374064837905, 'fraction_numerical': 0.012936408977556109, 'mean_word_length': 1.8545373665480427, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4955180', 'n_tokens_mistral': 1984, 'n_tokens_neox': 1766, 'n_words': 439}