<commit_before>package org.rcsb.mmtf.preupdatetests;

import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.List;

import org.biojava.nbio.structure.StructureException;
import org.biojava.nbio.structure.StructureIO;
import org.biojava.nbio.structure.align.util.AtomCache;
import org.biojava.nbio.structure.io.FileParsingParameters;
import org.biojava.nbio.structure.io.mmcif.ChemCompGroupFactory;
import org.biojava.nbio.structure.io.mmcif.DownloadChemCompProvider;
import org.rcsb.mmtf.biojavaencoder.EncoderUtils;
import org.rcsb.mmtf.update.TestingUtils;
import org.rcsb.mmtf.update.WeeklyUpdateUtils;

/**
 * Is the data to be added on the FTP site available, parseable and consistent when roundtripped.
 * @author <NAME>
 *
 */
public class DataConsistencyCheck {

	public static void main(String[] args) throws IllegalAccessException, InvocationTargetException, IOException, StructureException {


		// Set up the atom cache etc
	  	EncoderUtils encoderUtils = new EncoderUtils();
	  	AtomCache cache = encoderUtils.setUpBioJava();
	  	FileParsingParameters params = cache.getFileParsingParams();

		// Get 
		WeeklyUpdateUtils weeklyUpdate = new WeeklyUpdateUtils();
		weeklyUpdate.getDataFromFtpSite(args[0]);
		List<String> listToAdd = weeklyUpdate.getAddedList();
		String[] urlList = new String[listToAdd.size()];
		for (int i =0; i< listToAdd.size(); i++) {
			urlList[i] = args[1] + listToAdd.get(i);
		}
		TestingUtils testingUtils = new TestingUtils();
		testingUtils.testAll(urlList, params, cache);
	}


}<commit_msg>Updates to the data consistency checker<commit_after>package org.rcsb.mmtf.preupdatetests;

import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.List;

import org.biojava.nbio.structure.StructureException;
import org.biojava.nbio.structure.StructureIO;
import org.biojava.nbio.structure.align.util.AtomCache;
import org.biojava.nbio.structure.io.FileParsingParameters;
import org.biojava.nbio.structure.io.mmcif.ChemCompGroupFactory;
import org.biojava.nbio.structure.io.mmcif.DownloadChemCompProvider;
import org.rcsb.mmtf.biojavaencoder.EncoderUtils;
import org.rcsb.mmtf.update.TestingUtils;
import org.rcsb.mmtf.update.WeeklyUpdateUtils;

/**
 * Is the data to be added on the FTP site available, parseable and consistent when roundtripped.
 * @author <NAME>
 *
 */
public class DataConsistencyCheck {

	/**
	 * 1) Argument one is the FTP server for the update lists
	 * 2) Argumnet two is the server for the mmcif.gz files
	 * @param args
	 * @throws IllegalAccessException
	 * @throws InvocationTargetException
	 * @throws IOException
	 * @throws StructureException
	 */
	public static void main(String[] args) throws IllegalAccessException, InvocationTargetException, IOException, StructureException {


		// Set up the atom cache etc
	  	EncoderUtils encoderUtils = new EncoderUtils();
	  	AtomCache cache = encoderUtils.setUpBioJava();
	  	FileParsingParameters params = cache.getFileParsingParams();

		// Get 
		WeeklyUpdateUtils weeklyUpdate = new WeeklyUpdateUtils();
		System.out.println("getting data");
		weeklyUpdate.getDataFromFtpSite(args[0]);
		List<String> listToAdd = weeklyUpdate.getAddedList();
		String[] urlList = new String[listToAdd.size()];
		for (int i =0; i< listToAdd.size(); i++) {
			urlList[i] = args[1] + listToAdd.get(i);
			System.out.println(urlList[i]);
		}
		TestingUtils testingUtils = new TestingUtils();
		testingUtils.testAll(urlList, params, cache);
	}


}
{'fraction_non_alphanumeric': 0.10615952313369288, 'fraction_numerical': 0.0022707919386886176, 'mean_word_length': 5.24822695035461, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5220595', 'n_tokens_mistral': 1159, 'n_tokens_neox': 1079, 'n_words': 245}