<commit_before>#!/usr/bin/env python2.7

import logging
import sys
import argparse
import os

def main():
    logging.basicConfig(level=logging.INFO)
    logging.info('Program started.')

    #creazione del parser di input da linea di comando
    parser = argparse.ArgumentParser(prog = "predicted-introns-filtered",
                                      description = "Filtering intron from predicted-introns",
                                      formatter_class = argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-p', '--predicted-introns', help = " Predicted-introns file",
                         required = False, dest = 'pfile')
    parser.add_argument('-o', '--where-save-output', help = ' Name of file where save the output',
                         required = False, dest = 'wfile')
    parser.add_argument('-n', '--filter-value', help = ' Filter value',
                         required = False, dest = 'nValue', type = int, default = 20)
    parser.add_argument('-m', '--minimum-canonic-reads', help = ' Minimum canonic reads and',
                         required = False, dest = 'minimum', type = int)
    args = parser.parse_args()

    # controllo che il predicted-introns e il file di output sia specificato dall'utente
    # sia che il controllo fallisca, sia che il controllo vada a buon fine scrive sul file di LOG
    # In entrambi i casi scrive l'indirizzo assoluto del file sul file di LOG'
    if not args.pfile :
        logging.error('No predicted-introns file given.\nAborting...')
        sys.exit(1)

    abspfile = os.path.abspath(args.pfile)
    logging.info('Predicted-introns file: ' + abspfile)

    if not args.wfile:
        logging.error('ERROR: No output file specified.\nAborting...')
        sys.exit(1)

    abswfile = os.path.abspath(args.wfile)
    logging.info('Output file: ' + abswfile)

    # controllo su valori di taglio inseriti dall'utente (se non specificati sono N=20 e M=N/2=10)'
    if args.minimum > args.nValue :
        logging.error('ERROR: Minimum value is greater than N the filter value .\nAborting...')
        sys.exit(1)

    logging.info('Filter threshold: ' + str(args.nValue))

    # inizializzo le variabili con i valori presi dal parser e apro i file input/output
    N = args.nValue
    M = 0
    if not args.minimum:
	M = N / 2
    else: 
    	M = args.minimum
    logging.info('Minimum reads: ' + str(M))
    
    with open(args.pfile, 'r') as f, open(args.wfile, 'w') as out:
        # legge la linea e la trasforma in un lista di caratteri indicizzati [0,1...N] con l'istruzione .split'
   
        # CHECK corretto inserimento valori di taglio
        logging.debug("N: " + str(N) + " M: " + str(M))
        for linea in f.readlines():
            lista = linea.split()

            # fa il filtraggio dei dati
            num_reads = int(lista[5])
            intron_type = int(lista[13])
            intron_pattern = lista[14]
            if (nReads >= N) or \
               (\
                 (num_reads >= M) and \
                 (intron_type == 0 or intron_type == 1) and \
                 (intron_pattern == "GTAG" or intron_pattern == "GCAG" or intron_pattern == "ATAC") \
            ):
                # CHECK controllo dei parametri filtrati,
                logging.debug(str(num_reads) + " " + str(intron_type) + " " + intron_pattern)
                out.write(linea)
    logging.info('Program completed.')

if __name__ == "__main__":
    main()<commit_msg>Changed filter program to handle JSON files.<commit_after>#!/usr/bin/env python2.7

import logging
import sys
import argparse
import os
import json
import gzip

def main():
    logging.basicConfig(level=logging.INFO)
    logging.info('Program started.')

    #creazione del parser di input da linea di comando
    parser = argparse.ArgumentParser(prog = "predicted-introns-filtered",
                                      description = "Filtering intron from predicted-introns",
                                      formatter_class = argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-p', '--predicted-introns', help = "Predicted-introns file in JSON format.",
                         required = False, dest = 'pfile')
    parser.add_argument('-o', '--output-file', help = "Name of the output file in JSON format.",
                         required = False, dest = 'wfile')
    parser.add_argument('-n', '--filter-value', help = ' Filter value',
                         required = False, dest = 'nValue', type = int, default = 20)
    parser.add_argument('-m', '--minimum-canonic-reads', help = ' Minimum canonic reads and',
                         required = False, dest = 'minimum', type = int)
    args = parser.parse_args()

    # controllo che il predicted-introns e il file di output sia specificato dall'utente
    # sia che il controllo fallisca, sia che il controllo vada a buon fine scrive sul file di LOG
    # In entrambi i casi scrive l'indirizzo assoluto del file sul file di LOG'
    if not args.pfile :
        logging.error('No predicted-introns file given.\nAborting...')
        sys.exit(1)

    if not (args.pfile.endswith(".json") or args.pfile.endswith(".gz")):
        logging.error('Input file name must be a JSON (or GZ).\nAborting...')
        sys.exit(1)

    logging.info('Predicted-introns file: ' + args.pfile)

    if not args.wfile:
        logging.error('No output file specified.\nAborting...')
        sys.exit(1)

    if not (args.wfile.endswith(".json") or args.wfile.endswith(".gz")):
        logging.error('Output file name must be a JSON (or GZ).\nAborting...')
        sys.exit(1)

    logging.info('Output file: ' + args.wfile)

    # controllo su valori di taglio inseriti dall'utente (se non specificati sono N=20 e M=N/2=10)'
    if args.minimum > args.nValue :
        logging.error('Minimum value is greater than N the filter value .\nAborting...')
        sys.exit(1)

    logging.info('Filter threshold: ' + str(args.nValue))

    # inizializzo le variabili con i valori presi dal parser e apro i file input/output
    N = args.nValue
    M = 0
    if not args.minimum:
	M = N / 2
    else: 
    	M = args.minimum
    logging.info('Minimum reads: ' + str(M))

    jdata = {}
    if args.pfile.endswith(".gz"):
       with gzip.open(args.pfile, 'rb') as jfile:
           jdata = json.loads(jfile.read().decode("ascii"))
    else:
        with open(args.pfile, 'r') as jfile:
            jdata = json.load(jfile)
    # print jdata

    if args.wfile.endswith(".gz"):
        out = gzip.open(args.wfile, 'wb')
    else:
        out = open(args.wfile, 'w')

    num_kept_introns = 0
    num_tot_introns = 0
    introns = jdata['introns']
    new_introns = []
    for i in introns:
        num_tot_introns += 1
        num_reads = int(i['number_of_supporting_transcripts'])
        intron_type = int(i['type'])
        intron_pattern = i['pattern']
        if (num_reads >= N) or \
           ( \
             (num_reads >= M) and \
             (intron_type == 0 or intron_type == 1) and \
             (intron_pattern == "GTAG" or intron_pattern == "GCAG" or intron_pattern == "ATAC") \
         ):
            # Controllo dei parametri filtrati
            logging.debug(str(num_reads) + " " + str(intron_type) + " " + intron_pattern)
            num_kept_introns += 1
            new_introns.append(i)

    logging.info("Num. input introns: " + str(num_tot_introns))
    logging.info("Num. kept introns: " + str(num_kept_introns))
    jdata['introns'] = new_introns
    json.dump(jdata, out)
    
    logging.info('Program completed.')

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.12069871289729446, 'fraction_numerical': 0.00591016548463357, 'mean_word_length': 2.2710481099656357, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1447471', 'n_tokens_mistral': 2470, 'n_tokens_neox': 2345, 'n_words': 656}