<commit_before>from __future__ import division, print_function
import sys
import matplotlib.pyplot as plt
from maltpynt.mp_base import mp_detection_level
from maltpynt.mp_io import mp_load_data
import collections
import numpy as np

from scipy.optimize import curve_fit


def baseline_fun(x, a):
    return a


if __name__ == '__main__':
    ax = plt.subplot(1,1,1)
    rainbow = ax._get_lines.color_cycle
    for i, fname in enumerate(sys.argv[1:]):
        pdsdata = mp_load_data(fname)
        try:
            freq = pdsdata['freq']
        except:
            flo = pdsdata['flo']
            fhi = pdsdata['fhi']
            freq = (fhi + flo) / 2
            plt.semilogx()

        pds = pdsdata['pds']
        epds = pdsdata['epds']
        npds = pdsdata['npds']
        norm = pdsdata['norm']
        rebin = pdsdata['rebin']

        nbin = len(pds[1:])

        lev = mp_detection_level(nbin, n_summed_spectra=npds, n_rebin=rebin)

        color = rainbow.next()

        p, pcov = curve_fit(baseline_fun, freq, pds, p0=[2], sigma=epds)
        pds -= p[0]
        if isinstance(lev, collections.Iterable):
            plt.plot(freq, lev - p[0], color=color)
        else:
            plt.axhline(lev - p[0], color=color)<commit_msg>Minor change<commit_after>from __future__ import division, print_function
import sys
import matplotlib.pyplot as plt
from maltpynt.mp_base import mp_detection_level
from maltpynt.mp_io import mp_load_data
import collections
import numpy as np

from scipy.optimize import curve_fit


def baseline_fun(x, a):
    return a


if __name__ == '__main__':
    ax = plt.subplot(1,1,1)
    rainbow = ax._get_lines.color_cycle
    for i, fname in enumerate(sys.argv[1:]):
        pdsdata = mp_load_data(fname)
        try:
            freq = pdsdata['freq']
        except:
            flo = pdsdata['flo']
            fhi = pdsdata['fhi']
            freq = (fhi + flo) / 2
            plt.semilogx()

        pds = pdsdata['pds']
        epds = pdsdata['epds']
        npds = pdsdata['npds']
        norm = pdsdata['norm']
        rebin = pdsdata['rebin']

        nbin = len(pds[1:])

        lev = mp_detection_level(nbin, n_summed_spectra=npds, n_rebin=rebin)

        color = rainbow.next()

        p, pcov = curve_fit(baseline_fun, freq, pds, p0=[2], sigma=epds)
        print ('White noise level is', p[0])
        pds -= p[0]
        if isinstance(lev, collections.Iterable):
            plt.plot(freq, lev - p[0], color=color)
        else:
            plt.axhline(lev - p[0], color=color)
{'fraction_non_alphanumeric': 0.11553627760252366, 'fraction_numerical': 0.009069400630914827, 'mean_word_length': 2.3826666666666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '576258', 'n_tokens_mistral': 922, 'n_tokens_neox': 888, 'n_words': 204}