<commit_before>package earthcube.eager.write;

import earthcube.eager.util.FileWrite;
import earthcube.eager.util.GetListOfFiles;
import earthcube.eager.data.NsfData;
import earthcube.eager.read.NsfXmlData;
import earthcube.eager.write.oceanlink.*;

import java.io.File;
import java.util.ArrayList;

public class NsfFundedAwards {
	
	private File file = null;
	private FileWrite fw = new FileWrite();
	private GetListOfFiles getFileList = new GetListOfFiles ();
	private NsfXmlData nsfParser = new NsfXmlData();
	private ArrayList <String> personUris = new ArrayList <String> ();
	//private ArrayList <String> organizationUris = new ArrayList <String> ();
	
	// Counters to ensure unique IDs
	private long idCounter = 0;
	private long personRoleCount = -1;
	public String getPersonRoleCount () { personRoleCount++; return String.valueOf(personRoleCount); }
	public void incrementID() { idCounter++; }
	public String getID() { return String.valueOf(idCounter); }
	
	// Patterns
	private AgentRole agentRole = new AgentRole ();
	private AwardAmount awardAmount = new AwardAmount ();
	private FundingAward fundingAward = new FundingAward ();
	private FundingAwardInfoObject fundingAwardInfoObject = new FundingAwardInfoObject ();
	private InformationObject informationObject = new InformationObject ();
	private Organization organization = new Organization ();
	private Person person = new Person ();
	private PersonalInfoItem personInfoItem = new PersonalInfoItem ();
	private PersonName personName = new PersonName ();
	
	private boolean personUriExists ( String uri ) {
		if ( personUris.contains(uri) ) { return true; } else { return false; }
	}
	
	//private boolean organizationUriExists ( String uri ) {
	//	if ( organizationUris.contains(uri) ) { return true; } else { return false; }
	//}
	
	public void clearXmlFileList () { getFileList.clear(); }
	
	public ArrayList <String> getListOfXmlFiles ( String dir, boolean verbose ) {
		
		ArrayList <String> xmlFiles = getFileList.Process( new File (dir) );
		if ( verbose ) { System.out.println( "There are " + xmlFiles.size() + " XML files"); }
		return xmlFiles;
		
	}
			
	public static void main (String[] args) {
				
		// Inputs
		String inDir = args[0];	
		String outDir = args[1];
		
		String rdf, inputDir;
		String[] years = {"1976", "1977", "1978", "1979", "1980", "1981", "1982", "1983", "1984", "1985",
				"1986", "1987", "1988", "1989", "1990", "1991", "1992", "1993", "1994", "1995", "1996", 
				"1997", "1998", "1999", "2000", "2001", "2002", "2003", "2004", "2005", "2006", "2007",
				"2008", "2009", "2010", "2011", "2012"};
		
		NsfFundedAwards ol = new NsfFundedAwards ();

		// Create NSF as an Organization and NSF Divisions as Organizations
		NsfData nsfData = new NsfData ();
		NsfDivisions nsfDivisions = new NsfDivisions ();
		String[] divisions = nsfDivisions.getNsfDivisions();
		String[] divisionUri = nsfDivisions.getNsfDivsionUris();
		String[] divisionShortName = nsfDivisions.getNSfDivisionsShortName();
		String[] divisionInfoObjectUri = nsfDivisions.getNsfDivsionInfoObjectUris();
		
		//// NSF
		rdf = ol.organization.toRdfXml( "http://www.oceanlink.org/Organization/National_Science_Foundation", 
				nsfData.getFunderRoleTypeUri(), 
				"http://www.oceanlink.org/InformationObject/InformationObject_National_Science_Foundation",
			    divisionUri);
		ol.fw.newFile( outDir + "\\NSF.rdf", rdf );
		rdf = ol.informationObject.toRdfXml( 
				"http://www.oceanlink.org/InformationObject/InformationObject_National_Science_Foundation", 
				"The National Science Foundation", "The National Science Foundation");		
		ol.fw.newFile( outDir + "\\NSF_Information_Object.rdf", rdf );
		
        //// NSF Divisions
		for ( int i=0; i<divisionUri.length; i++ ) {
			rdf = ol.organization.toRdfXml( divisionUri[i], nsfData.getFunderRoleTypeUri(), 
					divisionInfoObjectUri[i], null );
			ol.fw.newFile( outDir + "\\NSF_" + divisionShortName[i].replace(" ","_") + ".rdf", rdf );
			// Create associated Information Object
			rdf = ol.informationObject.toRdfXml( divisionInfoObjectUri[i], divisions[i], divisions[i] );
			ol.fw.newFile( outDir + "\\NSF_Information_Object_" + divisionShortName[i].replace(" ","_") + 
					".rdf", rdf );
		}
					
		// Create variables we'll need
		String personUri, personAgentRoleUri, personalInfoItemUri, personNameUri, fundingAwardUri, 
		awardAmountUri, startDateUri, endDateUri, fundingAwardInformationObjectUri;
				
		// Loop over all the years of data
		String outputDir;
		ArrayList <String> xmlFiles = new ArrayList <String> ();
		for ( int i=0; i<years.length; i++ ) {
					
			// Find all the XML files
			outputDir = outDir + "\\" + years[i];
			inputDir = inDir + "\\" + years[i];
			xmlFiles.clear();
			ol.clearXmlFileList();
			xmlFiles = ol.getListOfXmlFiles( inputDir, true );
			
			// Status statistics
			int twentyPercent = (int) (xmlFiles.size() * 0.2);<commit_msg>Updates for OceanLink pattern writing<commit_after>package earthcube.eager.write;

import earthcube.eager.util.FileWrite;
import earthcube.eager.util.GetListOfFiles;
import earthcube.eager.data.NsfData;
import earthcube.eager.read.NsfXmlData;
import earthcube.eager.write.oceanlink.*;

import java.io.File;
import java.util.ArrayList;

public class NsfFundedAwards {
	
	private File file = null;
	private FileWrite fw = new FileWrite();
	private GetListOfFiles getFileList = new GetListOfFiles ();
	private NsfXmlData nsfParser = new NsfXmlData();
	private ArrayList <String> personUris = new ArrayList <String> ();
	//private ArrayList <String> organizationUris = new ArrayList <String> ();
	
	// Counters to ensure unique IDs
	private long idCounter = 0;
	private long personRoleCount = -1;
	public String getPersonRoleCount () { personRoleCount++; return String.valueOf(personRoleCount); }
	public void incrementID() { idCounter++; }
	public String getID() { return String.valueOf(idCounter); }
	
	// Patterns
	private AgentRole agentRole = new AgentRole ();
	private AwardAmount awardAmount = new AwardAmount ();
	private FundingAward fundingAward = new FundingAward ();
	private FundingAwardInfoObject fundingAwardInfoObject = new FundingAwardInfoObject ();
	private InformationObject informationObject = new InformationObject ();
	private Organization organization = new Organization ();
	private Person person = new Person ();
	private PersonalInfoItem personInfoItem = new PersonalInfoItem ();
	private PersonName personName = new PersonName ();
	
	private boolean personUriExists ( String uri ) {
		if ( personUris.contains(uri) ) { return true; } else { return false; }
	}
	
	//private boolean organizationUriExists ( String uri ) {
	//	if ( organizationUris.contains(uri) ) { return true; } else { return false; }
	//}
	
	public void clearXmlFileList () { getFileList.clear(); }
	
	public ArrayList <String> getListOfXmlFiles ( String dir, boolean verbose ) {
		
		ArrayList <String> xmlFiles = getFileList.Process( new File (dir) );
		if ( verbose ) { System.out.println( "There are " + xmlFiles.size() + " XML files"); }
		return xmlFiles;
		
	}
			
	public static void main (String[] args) {
				
		// Inputs
		String inDir = args[0];	
		String outDir = args[1];
		
		String rdf, inputDir;
		String[] years = {"1976", "1977", "1978", "1979", "1980", "1981", "1982", "1983", "1984", "1985",
				"1986", "1987", "1988", "1989", "1990", "1991", "1992", "1993", "1994", "1995", "1996", 
				"1997", "1998", "1999", "2000", "2001", "2002", "2003", "2004", "2005", "2006", "2007",
				"2008", "2009", "2010", "2011", "2012"};
		
		NsfFundedAwards ol = new NsfFundedAwards ();

		// Create NSF as an Organization and NSF Divisions as Organizations
		NsfData nsfData = new NsfData ();
		NsfDivisions nsfDivisions = new NsfDivisions ();
		String[] divisions = nsfDivisions.getNsfDivisions();
		String[] divisionUri = nsfDivisions.getNsfDivsionUris();
		String[] divisionShortName = nsfDivisions.getNSfDivisionsShortName();
		String[] divisionInfoObjectUri = nsfDivisions.getNsfDivsionInfoObjectUris();
		
		//// NSF
		rdf = ol.organization.toRdfXml( "http://www.oceanlink.org/Organization/National_Science_Foundation", 
				nsfData.getFunderRoleTypeUri(), 
				"http://www.oceanlink.org/InformationObject/InformationObject_National_Science_Foundation",
			    divisionUri);
		ol.fw.newFile( outDir + "\\NSF.rdf", rdf );
		rdf = ol.informationObject.toRdfXml( 
				"http://www.oceanlink.org/InformationObject/InformationObject_National_Science_Foundation", 
				"The National Science Foundation", "The National Science Foundation");		
		ol.fw.newFile( outDir + "\\NSF_Information_Object.rdf", rdf );
		
        //// NSF Divisions
		for ( int i=0; i<divisionUri.length; i++ ) {
			rdf = ol.organization.toRdfXml( divisionUri[i], nsfData.getFunderRoleTypeUri(), 
					divisionInfoObjectUri[i], null );
			ol.fw.newFile( outDir + "\\NSF_" + divisionShortName[i].replace(" ","_") + ".rdf", rdf );
			// Create associated Information Object
			rdf = ol.informationObject.toRdfXml( divisionInfoObjectUri[i], divisions[i], divisions[i] );
			ol.fw.newFile( outDir + "\\NSF_Information_Object_" + divisionShortName[i].replace(" ","_") + 
					".rdf", rdf );
		}
					
		// Create variables we'll need
		String personUri, personAgentRoleUri, personalInfoItemUri, personNameUri, fundingAwardUri, 
		awardAmountUri, startDateUri, endDateUri, fundingAwardInformationObjectUri;
				
		// Loop over all the years of data
		String outputDir;
		ArrayList <String> xmlFiles = new ArrayList <String> ();
		for ( int i=0; i<years.length; i++ ) {
					
			// Find all the XML files
			outputDir = outDir + "\\" + years[i];
			inputDir = inDir + "\\" + years[i];
			xmlFiles.clear();
			ol.clearXmlFileList();
			xmlFiles = ol.getListOfXmlFiles( inputDir, true );
			
			// Create the year subdirectory
			File subdir = new File ( outputDir );
			subdir.mkdir();
			
			// Status statistics
			int twentyPercent = (int) (xmlFiles.size() * 0.2);
{'fraction_non_alphanumeric': 0.132312959419998, 'fraction_numerical': 0.03141677575269359, 'mean_word_length': 4.302722904431393, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6812952', 'n_tokens_mistral': 3635, 'n_tokens_neox': 3026, 'n_words': 785}