<commit_before>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package org.ihtsdo.json;

import com.google.gson.Gson;
import org.ihtsdo.json.model.*;
import org.ihtsdo.json.utils.FileHelper;
import org.mapdb.DBMaker;

import java.io.*;
import java.util.*;

/**
 *
 * @author <NAME>
 */
public class TransformerOnePass {

	private String MODIFIER = "Existential restriction";
	private String sep = System.getProperty("line.separator");

	private Map<Long, ConceptDescriptor> concepts;
	private Map<Long, List<LightDescription>> descriptions;
	private Map<Long, List<LightRelationship>> relationships;
	private Map<Long, List<LightRefsetMembership>> simpleMembers;
	private Map<Long, List<LightRefsetMembership>> simpleMapMembers;
	private Map<Long, List<LightLangMembership>> languageMembers;
	private Map<String, String> langCodes;

	private String defaultLangCode = "en";
	public String fsnType = "900000000000003001";
	public String synType = "900000000000013009";
	private Long inferred = 900000000000011006l;
	private Long stated = 900000000000010007l;
	private Long isaSCTId=116680003l;
	private String defaultTermType = fsnType;
	private Map<Long, List<LightDescription>> tdefMembers;
	private Map<Long, List<LightRefsetMembership>> attrMembers;
	private Map<Long, List<LightRefsetMembership>> assocMembers;
	private ArrayList<Long> listA;
	private Map<String, String> charConv;
	private Map<Long, String> cptFSN;
	private HashSet<Long> notLeafInferred;
	private HashSet<Long> notLeafStated;


	public TransformerOnePass() throws IOException {
        String fileName = "/Volumes/Macintosh HD2/conversiondb/conversiondb";
        deleteDir(new File("/Volumes/Macintosh HD2/conversiondb"));

		concepts = DBMaker.newTempHashMap();
		descriptions = DBMaker.newTempHashMap();
		relationships = DBMaker.newTempHashMap();
		simpleMembers = DBMaker.newTempHashMap();
		assocMembers = DBMaker.newTempHashMap();
		attrMembers = DBMaker.newTempHashMap();
		tdefMembers = DBMaker.newTempHashMap();
		simpleMapMembers = DBMaker.newTempHashMap();
		languageMembers = DBMaker.newTempHashMap();
		notLeafInferred=new HashSet<Long>();
		notLeafStated=new HashSet<Long>();
		cptFSN = DBMaker.newTempHashMap();

		langCodes = new HashMap<String, String>();
		langCodes.put("en", "english");
		langCodes.put("es", "spanish");
		langCodes.put("da", "danish");
		langCodes.put("sv", "swedish");
		langCodes.put("fr", "french");
		langCodes.put("nl", "dutch");
	}

    public static void deleteDir(File dir) {
        if (dir.isDirectory()) {
            String[] children = dir.list();
            for (int i = 0; i < children.length; i++) {
                File child = new File(dir, children[i]);
                child.delete();
            }
        }
    }

	public static void main(String[] args) throws Exception {
		TransformerOnePass tr = new TransformerOnePass();
		tr.setDefaultLangCode("en");
		tr.setDefaultTermType(tr.fsnType);

		HashSet<String> folders=new HashSet<String>();
		folders.add("/Volumes/Macintosh HD2/uk_sct2cl_17/SnomedCT_Release_INT_20140131/RF2Release/Snapshot");
		folders.add("/Volumes/Macintosh HD2/uk_sct2cl_17/SnomedCT2_GB1000000_20140401/RF2Release/Snapshot");
		folders.add("/Users/termmed/Downloads/SnomedCT_Release_US1000124_20140301/RF2Release/Snapshot");
		folders.add("/Users/termmed/Downloads/SnomedCT_Release_AU1000036_20140531/RF2 Release/Snapshot");
		String valConfig= "config/validation-rules.xml";
        HashSet<String> files = tr.getFilesFromFolders(folders);
        tr.processFiles(files, valConfig);
		tr.createConceptsJsonFile("/Volumes/Macintosh HD2/Multi-english-data/concepts.json");
		tr.createTextIndexFile("/Volumes/Macintosh HD2/Multi-english-data/text-index.json");
		//tr.freeStep1();
		//tr.createTClosures(folders, valConfig, "/Volumes/Macintosh HD2/Multi-english-data/tclosure-inferred.json", "/Volumes/Macintosh HD2/tclosure-stated.json");
	}

	public void freeStep1() {
		descriptions =  null;
		simpleMembers =  null;
		assocMembers =  null;
		attrMembers = null;
		tdefMembers =  null;
		simpleMapMembers =  null;
		languageMembers =  null;
		notLeafInferred= null;
		notLeafStated= null;
		cptFSN =  null;<commit_msg>working version, no default term<commit_after>/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package org.ihtsdo.json;

import com.google.gson.Gson;
import org.ihtsdo.json.model.*;
import org.ihtsdo.json.utils.FileHelper;
import org.mapdb.DBMaker;

import java.io.*;
import java.util.*;

/**
 *
 * @author <NAME>
 */
public class TransformerOnePass {

	private String MODIFIER = "Existential restriction";
	private String sep = System.getProperty("line.separator");

	private Map<Long, ConceptDescriptor> concepts;
	private Map<Long, List<LightDescription>> descriptions;
	private Map<Long, List<LightRelationship>> relationships;
	private Map<Long, List<LightRefsetMembership>> simpleMembers;
	private Map<Long, List<LightRefsetMembership>> simpleMapMembers;
	private Map<Long, List<LightLangMembership>> languageMembers;
	private Map<String, String> langCodes;

	private String defaultLangCode = "en";
	public String fsnType = "900000000000003001";
	public String synType = "900000000000013009";
	private Long inferred = 900000000000011006l;
	private Long stated = 900000000000010007l;
	private Long isaSCTId=116680003l;
	private String defaultTermType = fsnType;
	private Map<Long, List<LightDescription>> tdefMembers;
	private Map<Long, List<LightRefsetMembership>> attrMembers;
	private Map<Long, List<LightRefsetMembership>> assocMembers;
	private ArrayList<Long> listA;
	private Map<String, String> charConv;
	private Map<Long, String> cptFSN;
	private HashSet<Long> notLeafInferred;
	private HashSet<Long> notLeafStated;


	public TransformerOnePass() throws IOException {
        String fileName = "/Volumes/Macintosh HD2/conversiondb/conversiondb";
        deleteDir(new File("/Volumes/Macintosh HD2/conversiondb"));

		concepts = DBMaker.newTempHashMap();
		descriptions = DBMaker.newTempHashMap();
		relationships = DBMaker.newTempHashMap();
		simpleMembers = DBMaker.newTempHashMap();
		assocMembers = DBMaker.newTempHashMap();
		attrMembers = DBMaker.newTempHashMap();
		tdefMembers = DBMaker.newTempHashMap();
		simpleMapMembers = DBMaker.newTempHashMap();
		languageMembers = DBMaker.newTempHashMap();
		notLeafInferred=new HashSet<Long>();
		notLeafStated=new HashSet<Long>();
		cptFSN = DBMaker.newTempHashMap();

		langCodes = new HashMap<String, String>();
		langCodes.put("en", "english");
		langCodes.put("es", "spanish");
		langCodes.put("da", "danish");
		langCodes.put("sv", "swedish");
		langCodes.put("fr", "french");
		langCodes.put("nl", "dutch");
	}

    public static void deleteDir(File dir) {
        if (dir.isDirectory()) {
            String[] children = dir.list();
            for (int i = 0; i < children.length; i++) {
                File child = new File(dir, children[i]);
                child.delete();
            }
        }
    }

	public static void main(String[] args) throws Exception {
		TransformerOnePass tr = new TransformerOnePass();
		tr.setDefaultLangCode("en");
		tr.setDefaultTermType(tr.fsnType);

		HashSet<String> folders=new HashSet<String>();
		folders.add("/Volumes/Macintosh HD2/uk_sct2cl_17/SnomedCT_Release_INT_20140131/RF2Release/Snapshot");
		folders.add("/Volumes/Macintosh HD2/uk_sct2cl_17/SnomedCT2_GB1000000_20140401/RF2Release/Snapshot");
		folders.add("/Users/termmed/Downloads/SnomedCT_Release_US1000124_20140301/RF2Release/Snapshot");
		//folders.add("/Users/termmed/Downloads/SnomedCT_Release_AU1000036_20140531/RF2 Release/Snapshot");
		String valConfig= "config/validation-rules.xml";
        HashSet<String> files = tr.getFilesFromFolders(folders);
        tr.processFiles(files, valConfig);
		tr.createConceptsJsonFile("/Volumes/Macintosh HD2/Multi-english-data/concepts.json");
		tr.createTextIndexFile("/Volumes/Macintosh HD2/Multi-english-data/text-index.json");
		//tr.freeStep1();
		//tr.createTClosures(folders, valConfig, "/Volumes/Macintosh HD2/Multi-english-data/tclosure-inferred.json", "/Volumes/Macintosh HD2/tclosure-stated.json");
	}

	public void freeStep1() {
		descriptions =  null;
		simpleMembers =  null;
		assocMembers =  null;
		attrMembers = null;
		tdefMembers =  null;
		simpleMapMembers =  null;
		languageMembers =  null;
		notLeafInferred= null;
		notLeafStated= null;
		cptFSN =  null;
{'fraction_non_alphanumeric': 0.12128569776852816, 'fraction_numerical': 0.036073534512660425, 'mean_word_length': 4.953200275292498, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '368392', 'n_tokens_mistral': 3107, 'n_tokens_neox': 2825, 'n_words': 559}