<commit_before>#! /usr/bin/env python

import sys
import random

from Bio import SeqIO

import numpy as np
import scipy as sp
import scipy.stats

import matplotlib as mpl
mpl.use('agg')
import matplotlib.pyplot as plt

from pbs import wc

input_file = sys.argv[1]
output_file = sys.argv[2]

num_lines = int(wc(input_file, '-l').split()[0])
assert(num_lines % 4 == 0)
num_reads = num_lines / 4

if num_reads > 10000000:
    idxs = set(sorted(random.sample(xrange(num_reads),10000000)))

qualities = []
for (i,record) in enumerate(SeqIO.parse(input_file, 'fastq')):
    if num_reads <= 10000000 or i in idxs:
        qualities.append(record.letter_annotations['phred_quality'])
    
    if i % 10000 == 0:
        sys.stdout.write("%i " % i)
        sys.stdout.flush()

positions = range(1, qualities.shape[1]+1)

p5  = sp.stats.scoreatpercentile(qualities, 5)
p25 = sp.stats.scoreatpercentile(qualities, 25)
p50 = sp.stats.scoreatpercentile(qualities, 50)
p75 = sp.stats.scoreatpercentile(qualities, 75)
p95 = sp.stats.scoreatpercentile(qualities, 95)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(positions,p5,  s=3, c='k', linewidths=0, zorder=2)
ax.scatter(positions,p95, s=3, c='k', linewidths=0, zorder=2)
for (pos, low, high) in zip(positions, p25, p75):
    ax.plot([pos, pos], [low, high], color='#bdbdbd', lw=1, zorder=1)<commit_msg>bugfix<commit_after>#! /usr/bin/env python

import sys
import random

from Bio import SeqIO

import numpy as np
import scipy as sp
import scipy.stats

import matplotlib as mpl
mpl.use('agg')
import matplotlib.pyplot as plt

from pbs import wc

input_file = sys.argv[1]
output_file = sys.argv[2]

num_lines = int(wc(input_file, '-l').split()[0])
assert(num_lines % 4 == 0)
num_reads = num_lines / 4

if num_reads > 10000000:
    idxs = set(sorted(random.sample(xrange(num_reads),10000000)))

qualities = []
for (i,record) in enumerate(SeqIO.parse(input_file, 'fastq')):
    if num_reads <= 10000000 or i in idxs:
        qualities.append(record.letter_annotations['phred_quality'])
    
    if i % 10000 == 0:
        sys.stdout.write("%i " % i)
        sys.stdout.flush()

qualities = np.array(qualities)

positions = range(1, qualities.shape[1]+1)

p5  = sp.stats.scoreatpercentile(qualities, 5)
p25 = sp.stats.scoreatpercentile(qualities, 25)
p50 = sp.stats.scoreatpercentile(qualities, 50)
p75 = sp.stats.scoreatpercentile(qualities, 75)
p95 = sp.stats.scoreatpercentile(qualities, 95)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(positions,p5,  s=3, c='k', linewidths=0, zorder=2)
ax.scatter(positions,p95, s=3, c='k', linewidths=0, zorder=2)
for (pos, low, high) in zip(positions, p25, p75):
    ax.plot([pos, pos], [low, high], color='#bdbdbd', lw=1, zorder=1)
{'fraction_non_alphanumeric': 0.14871794871794872, 'fraction_numerical': 0.054945054945054944, 'mean_word_length': 4.689583333333333, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4397060', 'n_tokens_mistral': 1170, 'n_tokens_neox': 1060, 'n_words': 241}