<commit_before>import os
import json
import argparse
import subprocess

from subprocess import Popen, PIPE

# ------------------------------------------------------------------------------------------


search_dirs = ["/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch1_chunk1_searches",
              "/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch1_chunk2_searches",
              "/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch2/searches"]

MEMORY = 8000
CPU = 4
LSF_GROUP = "/family_srch"


REF_STRING = """RN   [1]
RM   30423142
RT   miRBase: from microRNA sequences to function.
RA   <NAME>, <NAME>, <NAME>;
RL   Nucleic Acids Res. 2019;47:D155."""

# ------------------------------------------------------------------------------------------


def check_desc_reference_is_valid(desc_loc, ref_string):

	process = Popen(["grep", ref_string, desc_loc], stdin=PIPE, stdout=PIPE, stderr=PIPE)
        output = process.communicate()[1]

        if output.find(ref_string) == -1:
                return False

	return True

# ------------------------------------------------------------------------------------------

def parse_arguments():

	parser = argparse.ArgumentParser()
	
	parser.add_argument("--mirna-list", help="A .json file containing all miRNAs to validate", action="store")
	parser.add_argument("--log", help="Creates a log file with all validated DESC files", action="store_true", default=False)

	return parser	

# ------------------------------------------------------------------------------------------

def get_mirna_directory_location(mirna_id):
	
	
        if mirna_id.find("_relabelled")==-1:
		dir_label = mirna_id+"_relabelled"

        for search_dir in search_dirs:
        	family_dir_loc = os.path.join(search_dir, dir_label)
                if os.path.exists(family_dir_loc):
			return family_dir_loc

	return None

# ------------------------------------------------------------------------------------------

if __name__=='__main__':

	parser = parse_arguments()
	args = parser.parse_args()

	fp = open(args.mirna_list, 'r')
	mirnas = json.load(fp)
	fp.close()

	for mirna in mirnas:
		mirna_dir_loc = get_mirna_directory_location(mirna)
		if mirna_dir_loc is not None:
			desc_loc = os.path.join(mirna_dir_loc, "DESC")
			if os.path.exists(desc_loc):
				check = check_desc_reference_is_valid(desc_loc, REF_STRING)
				if check is False:
					print (mirna_dir_loc)
					
		
	

<commit_msg>Re-implements validation function<commit_after>import os
import json
import argparse
import subprocess
import logging

from subprocess import Popen, PIPE

# ------------------------------------------------------------------------------------------


search_dirs = ["/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch1_chunk1_searches",
              "/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch1_chunk2_searches",
              "/hps/nobackup/production/xfam/rfam/RELEASES/14.3/miRNA_relabelled/batch2/searches"]

MEMORY = 8000
CPU = 4
LSF_GROUP = "/family_srch"


REF_STRING = """RN   [1]
RM   30423142
RT   miRBase: from microRNA sequences to function.
RA   <NAME>, <NAME>, <NAME>;
RL   Nucleic Acids Res. 2019;47:D155."""

# ------------------------------------------------------------------------------------------


def check_desc_reference_is_valid(desc_loc, ref_string):

	fp = open(desc_loc, 'r')
	desc_lines = fp.read()
	fp.close()
	
	# check if we can find the reference lines in DESC
        if desc_lines.find(REF_STRING) != -1:
		return True	

	return False
	
# ------------------------------------------------------------------------------------------

def parse_arguments():

	parser = argparse.ArgumentParser()
	
	parser.add_argument("--mirna-list", help="A .json file containing all miRNAs to validate", action="store")
	parser.add_argument("--log", help="Creates a log file with all validated DESC files", action="store_true", default=False)

	return parser	

# ------------------------------------------------------------------------------------------

def get_mirna_directory_location(mirna_id):
	
	
        if mirna_id.find("_relabelled")==-1:
		dir_label = mirna_id+"_relabelled"

        for search_dir in search_dirs:
        	family_dir_loc = os.path.join(search_dir, dir_label)
                if os.path.exists(family_dir_loc):
			return family_dir_loc

	return None

# ------------------------------------------------------------------------------------------

if __name__=='__main__':

		
	parser = parse_arguments()
	args = parser.parse_args()

	fp = open(args.mirna_list, 'r')
	mirnas = json.load(fp)
	fp.close()

	
#	if args.log is True:

	for mirna in mirnas:
		mirna_dir_loc = get_mirna_directory_location(mirna)
		if mirna_dir_loc is not None:
			desc_loc = os.path.join(mirna_dir_loc, "DESC")
			if os.path.exists(desc_loc):
				check = check_desc_reference_is_valid(desc_loc, REF_STRING)
				if check is False:
					print (mirna_dir_loc)
				
		
	

	
{'fraction_non_alphanumeric': 0.2709741550695825, 'fraction_numerical': 0.01570576540755467, 'mean_word_length': 1.831176139561058, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2964515', 'n_tokens_mistral': 1675, 'n_tokens_neox': 1513, 'n_words': 309}