<commit_before>package org.sitenv.vocabularies.loader.code.snomed;

import com.orientechnologies.common.io.OIOUtils;
import com.orientechnologies.orient.core.intent.OIntentMassiveInsert;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.text.StrBuilder;
import org.apache.log4j.Logger;
import org.sitenv.vocabularies.constants.VocabularyConstants;
import org.sitenv.vocabularies.loader.code.CodeLoader;
import org.sitenv.vocabularies.loader.code.CodeLoaderManager;
import org.sitenv.vocabularies.model.VocabularyModelDefinition;
import org.sitenv.vocabularies.model.impl.Icd9CmDxModel;
import org.sitenv.vocabularies.model.impl.SnomedModel;
import org.sitenv.vocabularies.model.impl.SnomedModel;
import org.sitenv.vocabularies.repository.VocabularyRepository;

import com.orientechnologies.orient.core.sql.OCommandSQL;
import com.orientechnologies.orient.object.db.OObjectDatabaseTx;

public class SnomedLoader implements CodeLoader {

	private static Logger logger = Logger.getLogger(SnomedLoader.class);
	

	static {
		CodeLoaderManager.getInstance()
				.registerLoader(VocabularyConstants.SNOMEDCT_CODE_NAME, SnomedLoader.class);
		logger.info("Loaded: " + VocabularyConstants.SNOMEDCT_CODE_NAME + "(" + VocabularyConstants.SNOMEDCT_CODE_SYSTEM + ")");
		
		VocabularyModelDefinition snomed = new VocabularyModelDefinition(SnomedModel.class, VocabularyConstants.SNOMEDCT_CODE_SYSTEM);
			

		VocabularyRepository.getInstance().getVocabularyMap().put(VocabularyConstants.SNOMEDCT_CODE_SYSTEM, snomed);
		
	}
	
	
public void load(List<File> filesToLoad) {
		

		OObjectDatabaseTx dbConnection = VocabularyRepository.getInstance().getInactiveDbConnection();
		BufferedReader br = null;
		
		try {
			
			logger.info("Truncating SnomedModel Datastore...");
			VocabularyRepository.truncateModel(dbConnection, SnomedModel.class);
			logger.info(dbConnection.getName() + ".SnomedModel Datastore Truncated... records remaining: " + VocabularyRepository.getRecordCount(dbConnection, SnomedModel.class));

			VocabularyRepository.updateIndexProperties(dbConnection, SnomedModel.class, true);
		
			String insertQueryPrefix = "insert into " + SnomedModel.class.getSimpleName() + " (codeIndex, displayNameIndex, code, displayName) values ";
			
			StrBuilder insertQueryBuilder = new StrBuilder(insertQueryPrefix);
			insertQueryBuilder.ensureCapacity(1000);
			
			int totalCount = 0, pendingCount = 0;
			
			dbConnection.declareIntent(new OIntentMassiveInsert());
			
			for (File file : filesToLoad)
			{
				if (file.isFile() && !file.isHidden())
				{
					
					logger.debug("Loading SNOMED File: " + file.getName());
	
					int count = 0;

					br = new BufferedReader(new FileReader(file));
					String available;
					while ((available = br.readLine()) != null) {
						if (count++ == 0) {
							continue; // skip header row
						} else {

							String[] line = StringUtils.splitPreserveAllTokens(available, "\t", 6);
							
							if (pendingCount++ > 0) {
								insertQueryBuilder.append(",");
							}
							
							insertQueryBuilder.append("(\"");
							insertQueryBuilder.append(OIOUtils.encode(line[4].toUpperCase()));
							insertQueryBuilder.append("\",\"");
							insertQueryBuilder.append(OIOUtils.encode(line[2].toUpperCase()));
							insertQueryBuilder.append("\",\"");
							insertQueryBuilder.append(OIOUtils.encode(line[4]));
							insertQueryBuilder.append("\",\"");
							insertQueryBuilder.append(OIOUtils.encode(line[2]));
							insertQueryBuilder.append("\")");
							
							if ((totalCount % 5000) == 0) {
								dbConnection.command(new OCommandSQL(insertQueryBuilder.toString())).execute();
								dbConnection.commit();
								
								insertQueryBuilder.clear();<commit_msg>Fixed SNOMED loader code value extraction (the first delimited field/column is the code value).<commit_after>package org.sitenv.vocabularies.loader.code.snomed;

import com.orientechnologies.common.io.OIOUtils;
import com.orientechnologies.orient.core.intent.OIntentMassiveInsert;
import com.orientechnologies.orient.core.sql.OCommandSQL;
import com.orientechnologies.orient.object.db.OObjectDatabaseTx;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.text.StrBuilder;
import org.apache.log4j.Logger;
import org.sitenv.vocabularies.constants.VocabularyConstants;
import org.sitenv.vocabularies.loader.code.CodeLoader;
import org.sitenv.vocabularies.loader.code.CodeLoaderManager;
import org.sitenv.vocabularies.model.VocabularyModelDefinition;
import org.sitenv.vocabularies.model.impl.SnomedModel;
import org.sitenv.vocabularies.repository.VocabularyRepository;

public class SnomedLoader implements CodeLoader {

	private static Logger logger = Logger.getLogger(SnomedLoader.class);
	

	static {
		CodeLoaderManager.getInstance()
				.registerLoader(VocabularyConstants.SNOMEDCT_CODE_NAME, SnomedLoader.class);
		logger.info("Loaded: " + VocabularyConstants.SNOMEDCT_CODE_NAME + "(" + VocabularyConstants.SNOMEDCT_CODE_SYSTEM + ")");
		
		VocabularyModelDefinition snomed = new VocabularyModelDefinition(SnomedModel.class, VocabularyConstants.SNOMEDCT_CODE_SYSTEM);
			

		VocabularyRepository.getInstance().getVocabularyMap().put(VocabularyConstants.SNOMEDCT_CODE_SYSTEM, snomed);
		
	}
	
	
public void load(List<File> filesToLoad) {
		

		OObjectDatabaseTx dbConnection = VocabularyRepository.getInstance().getInactiveDbConnection();
		BufferedReader br = null;
		
		try {
			
			logger.info("Truncating SnomedModel Datastore...");
			VocabularyRepository.truncateModel(dbConnection, SnomedModel.class);
			logger.info(dbConnection.getName() + ".SnomedModel Datastore Truncated... records remaining: " + VocabularyRepository.getRecordCount(dbConnection, SnomedModel.class));

			VocabularyRepository.updateIndexProperties(dbConnection, SnomedModel.class, true);
		
			String insertQueryPrefix = "insert into " + SnomedModel.class.getSimpleName() + " (codeIndex, displayNameIndex, code, displayName) values ";
			
			StrBuilder insertQueryBuilder = new StrBuilder(insertQueryPrefix);
			insertQueryBuilder.ensureCapacity(1000);
			
			int totalCount = 0, pendingCount = 0;
			
			dbConnection.declareIntent(new OIntentMassiveInsert());
			
			for (File file : filesToLoad)
			{
				if (file.isFile() && !file.isHidden())
				{
					
					logger.debug("Loading SNOMED File: " + file.getName());
	
					int count = 0;

					br = new BufferedReader(new FileReader(file));
					String available;
					while ((available = br.readLine()) != null) {
						if (count++ == 0) {
							continue; // skip header row
						} else {

							String[] line = StringUtils.splitPreserveAllTokens(available, "\t", 4);
							
							if (pendingCount++ > 0) {
								insertQueryBuilder.append(",");
							}
							
							insertQueryBuilder.append("(\"");
							insertQueryBuilder.append(OIOUtils.encode(line[0].toUpperCase()));
							insertQueryBuilder.append("\",\"");
							insertQueryBuilder.append(OIOUtils.encode(line[2].toUpperCase()));
							insertQueryBuilder.append("\",\"");
							insertQueryBuilder.append(OIOUtils.encode(line[0]));
							insertQueryBuilder.append("\",\"");
							insertQueryBuilder.append(OIOUtils.encode(line[2]));
							insertQueryBuilder.append("\")");
							
							if ((totalCount % 5000) == 0) {
								dbConnection.command(new OCommandSQL(insertQueryBuilder.toString())).execute();
								dbConnection.commit();
								
								insertQueryBuilder.clear();
{'fraction_non_alphanumeric': 0.11861593462717059, 'fraction_numerical': 0.005745658835546476, 'mean_word_length': 4.4929873772791025, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7064270', 'n_tokens_mistral': 2872, 'n_tokens_neox': 2340, 'n_words': 360}