<commit_before>"""Go through BAM file made from simulated reads. Score alignment of each read and create a data structure
consisting of MQ and simple d metric. Plot this data"""
from multiprocessing import Pool
import time
import logging

import pysam
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
# from matplotlib.colors import LogNorm

from mitty.simulation.readgenerate import parse_qname, score_alignment_error


logger = logging.getLogger(__name__)


def process_bam(bam_fname, max_d=200, strict=False, sample_name=None, processes=2, out_csv=None):
  bam_fp = pysam.AlignmentFile(bam_fname)

  p = Pool(processes=processes)
  mq_mat = np.zeros((100, 2 * max_d + 1), dtype=np.uint64)
  t_st = t0 = time.time()
  tot_rd = 0
  for shard in p.imap_unordered(process_bam_part_w,
                                ((bam_fname, s['SN'], max_d, strict, sample_name) for s in bam_fp.header['SQ'])):
    rd_cnt = shard.sum()
    tot_rd += rd_cnt
    mq_mat += shard
    dt = time.time() - t0
    logger.debug('{} templates processed ({:0.2f} templates/sec).'.format(rd_cnt, rd_cnt / dt))
    t0 = time.time()
<commit_msg>Updating to renaming of module<commit_after>"""Go through BAM file made from simulated reads. Score alignment of each read and create a data structure
consisting of MQ and simple d metric. Plot this data"""
from multiprocessing import Pool
import time
import logging

import pysam
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
# from matplotlib.colors import LogNorm

from mitty.benchmarking.alignmentscore import load_qname_sidecar, parse_qname, score_alignment_error


logger = logging.getLogger(__name__)


def process_bam(bam_fname, max_d=200, strict=False, sample_name=None, processes=2, out_csv=None):
  bam_fp = pysam.AlignmentFile(bam_fname)

  p = Pool(processes=processes)
  mq_mat = np.zeros((100, 2 * max_d + 1), dtype=np.uint64)
  t_st = t0 = time.time()
  tot_rd = 0
  for shard in p.imap_unordered(process_bam_part_w,
                                ((bam_fname, s['SN'], max_d, strict, sample_name) for s in bam_fp.header['SQ'])):
    rd_cnt = shard.sum()
    tot_rd += rd_cnt
    mq_mat += shard
    dt = time.time() - t0
    logger.debug('{} templates processed ({:0.2f} templates/sec).'.format(rd_cnt, rd_cnt / dt))
    t0 = time.time()

{'fraction_non_alphanumeric': 0.10546378653113088, 'fraction_numerical': 0.014400677678949597, 'mean_word_length': 4.0905172413793105, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6310520', 'n_tokens_mistral': 857, 'n_tokens_neox': 816, 'n_words': 231}