<commit_before>/* 
 * Licensed under the Apache License, Version 2.0 (the "License");
 *
 * You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Contributions from 2013-2017 where performed either by US government 
 * employees, or under US Veterans Health Administration contracts. 
 *
 * US Veterans Health Administration contributions by government employees
 * are work of the U.S. Government and are not subject to copyright
 * protection in the United States. Portions contributed by government 
 * employees are USGovWork (17USC §105). Not subject to copyright. 
 * 
 * Contribution by contractors to the US Veterans Health Administration
 * during this period are contractually contributed under the
 * Apache License, Version 2.0.
 *
 * See: https://www.usa.gov/government-works
 * 
 * Contributions prior to 2013:
 *
 * Copyright (C) International Health Terminology Standards Development Organisation.
 * Licensed under the Apache License, Version 2.0.
 *
 */



package sh.isaac.pombuilder.converter;

import java.io.IOException;

import java.util.Arrays;
import java.util.List;

import org.apache.commons.lang3.StringUtils;
import javafx.util.Pair;

import sh.isaac.pombuilder.FileUtil;
import sh.isaac.pombuilder.upload.SrcUploadCreator;

/**
 * {@link SupportedConverterTypes}.
 *
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 */
public enum SupportedConverterTypes
{
	//(?i) and (?-i) constructs are not supported in JavaScript (they are in Ruby)
	LOINC("loinc-src-data", ".*$", 
			"A typical LOINC version number is '2.59'.  The version numbers should be used directly from LOINC.  There are no enforced restrictions on the format.",
			new String[] {}, new String[] {}, new UploadFileInfo[] {
			new UploadFileInfo("", "https://loinc.org/downloads/loinc", 
					"LOINC_2.54_Text.zip",
					"The primary LOINC file is the 'LOINC Table File' in the csv format'.  This should be a zip file that contains a file named 'loinc.csv'."
					+ "  Additionally, the zip file may (optionally) contain 'map_to.csv' and 'source_organization.csv'."
					+ "  The zip file must contain 'text' within its name.", ".*text.*\\.zip$", true),
			new UploadFileInfo("", "https://loinc.org/downloads/files/loinc-multiaxial-hierarchy", 
					"LOINC_2.54_MULTI-AXIAL_HIERARCHY.zip",
					"The Multiaxial Hierarchy file is a zip file that contains a file named *multi-axial_hierarchy.csv.  The zip file containing the multiaxial hierarchy"
					+ " must contain 'multi-axial_hierarchy' within its name", ".*multi\\-axial_hierarchy.*\\.zip$", true),
			new UploadFileInfo("", "https://loinc.org/downloads/loinc",
					"LOINC_ReleaseNotes.txt",
					"The LOINC Release Notes file must be included for recent versions of LOINC.", ".*releasenotes\\.txt$", true)
	}, "loinc-ibdf", "convert-loinc-to-ibdf", "LOINC", 
			new String[] {"shared/licenses/loinc.xml"}, 
			new String[] {"shared/noticeAdditions/loinc-NOTICE-addition.txt"}),
	
	//This one is deprecated, it was still the old form of the data.
	LOINC_TECH_PREVIEW("loinc-src-data-tech-preview", ".*$", 
			"A typical LOINC tech preview version number is '2015.08.01'.  The version numbers should be used directly from LOINC.  There are no enforced restrictions "
			+ "on the format.", 
			new String[] {"loinc-src-data"}, new String[] {"rf2-ibdf-sct"}, new UploadFileInfo[] {
			new UploadFileInfo("", "https://www.nlm.nih.gov/healthit/snomedct/international.html",
					"SnomedCT_LOINC_AlphaPhase3_INT_20160401.zip",
					"  The expected file is the RF2 release (NOT the Human Readable release nor the OWL release). "
					+ "The file must be a zip file, which ends with .zip", ".*\\.zip$", true)
	}, "loinc-ibdf-tech-preview", "convert-loinc-tech-preview-to-ibdf", "LOINC Tech Preview", 
			new String[] {"shared/licenses/loinc.xml", "shared/licenses/sct.xml"}, 
			new String[] {"shared/noticeAdditions/loinc-tech-preview-NOTICE-addition.txt", "shared/noticeAdditions/loinc-NOTICE-addition.txt", "shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	
	SCT("rf2-src-data-sct", "\\d{8}.*$", 
			"A typical Snomed version number is '20170131' or '20170131T120000'.  The value here should be the same as the version number in the name of the uploaded "
			+ "zip file.  This requires a 4 digit year, 2 digit month, 2 digit day.  Any values can be appended after the 8 digits.",
			new String[] {}, new String[] {}, new UploadFileInfo[] {
			new UploadFileInfo("", "https://www.nlm.nih.gov/healthit/snomedct/international.html",
					"SnomedCT_RF2Release_INT_20160131.zip",
					"The expected file is the RF2 release zip file.  The filename must end with .zip, and must contain the release date in the Snomed standard"
					+ " naming convention (4 digit year, 2 digit month, 2 digit day).",
					".*_\\d{8}.*\\.zip$", true)
	}, "rf2-ibdf-sct", "convert-RF2-direct-to-ibdf", "SnomedCT", 
			new String[] {"shared/licenses/sct.xml"},
			new String[] {"shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	
	SCT_EXTENSION("rf2-src-data-*-extension", "\\d{8}.*$", 
			"A typical Snomed extension version number is '20170131' or '20170131T120000'.  The value here should be the same as the version number in the name of the uploaded "
			+ "zip file.  This requires a 4 digit year, 2 digit month, 2 digit day.  Any values can be appended after the 8 digits.",
			new String[] {}, new String[] {"rf2-ibdf-sct"}, new UploadFileInfo[] {
			new UploadFileInfo("Snomed Extensions come from a variety of sources.  Note that the NLM has choosen to stop advertising the download links to the "
					+ " US Extension, but still publishes it.  The current download pattern is now: "
					+ "https://download.nlm.nih.gov/mlb/utsauth/USExt/SnomedCT_USExtensionRF2_Production_YYYYMMDDTHHMMSS.zip",
					"",
					"SnomedCT_USExtensionRF2_Production_20170301T120000.zip",
					"The expected file is the RF2 release zip file.  The filename must end with .zip, and must contain the release date in the Snomed standard"
					+ " naming convention (4 digit year, 2 digit month, 2 digit day) - it also now also accepts the new naming convention with T and 2 digits each "
					+ "of hour, minute and second.",
					".*_\\d{8}.*\\.zip$", true)
	}, "rf2-ibdf-", "convert-RF2-direct-to-ibdf", "SnomedCT Extension", 
			new String[] {"shared/licenses/sct.xml"},
			new String[] {"shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	
	SCT_EXTENSION_LOINC("rf2-src-data-loinc-extension", "\\d{8}.*$", 
			"A typical Snomed extension version number is '20170131' or '20170131T120000'.  The value here should be the same as the version number in the name of the uploaded "
			+ "zip file.  This requires a 4 digit year, 2 digit month, 2 digit day.  Any values can be appended after the 8 digits.  This should be used for the "
					+ "LOINC snomed collaboration RF2 release.  This requires a loinc file - and should be the 'solor' converted loinc file.",
			new String[] {}, new String[] {"rf2-ibdf-sct", "loinc-ibdf"}, new UploadFileInfo[] {
			new UploadFileInfo("The LOINC coop file comes from IHTSDO - https://www.nlm.nih.gov/healthit/snomedct/international.html",
					"https://download.nlm.nih.gov/umls/kss/IHTSDO20170731/SnomedCT_LOINCRF2_PRODUCTION_20170831T120000Z.zip",
					"SnomedCT_LOINCRF2_PRODUCTION_YYYYMMDDTHHMMSSZ.zip",
					"The expected file is the RF2 release zip file.  The filename must end with .zip, and must contain the release date in the Snomed standard"
					+ " naming convention (4 digit year, 2 digit month, 2 digit day) - it also now also accepts the new naming convention with T and 2 digits each "
					+ "of hour, minute and second.",
					".*_\\d{8}.*\\.zip$", true)
	}, "rf2-ibdf-loinc", "convert-RF2-direct-to-ibdf", "SnomedCT Loinc Extension", 
			new String[] {"shared/licenses/sct.xml"},
			new String[] {"shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	<commit_msg>another fix for a loinc naming change<commit_after>/* 
 * Licensed under the Apache License, Version 2.0 (the "License");
 *
 * You may not use this file except in compliance with the License.
 *
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Contributions from 2013-2017 where performed either by US government 
 * employees, or under US Veterans Health Administration contracts. 
 *
 * US Veterans Health Administration contributions by government employees
 * are work of the U.S. Government and are not subject to copyright
 * protection in the United States. Portions contributed by government 
 * employees are USGovWork (17USC §105). Not subject to copyright. 
 * 
 * Contribution by contractors to the US Veterans Health Administration
 * during this period are contractually contributed under the
 * Apache License, Version 2.0.
 *
 * See: https://www.usa.gov/government-works
 * 
 * Contributions prior to 2013:
 *
 * Copyright (C) International Health Terminology Standards Development Organisation.
 * Licensed under the Apache License, Version 2.0.
 *
 */



package sh.isaac.pombuilder.converter;

import java.io.IOException;

import java.util.Arrays;
import java.util.List;

import org.apache.commons.lang3.StringUtils;
import javafx.util.Pair;

import sh.isaac.pombuilder.FileUtil;
import sh.isaac.pombuilder.upload.SrcUploadCreator;

/**
 * {@link SupportedConverterTypes}.
 *
 * @author <a href="mailto:<EMAIL>"><NAME></a>
 */
public enum SupportedConverterTypes
{
	//(?i) and (?-i) constructs are not supported in JavaScript (they are in Ruby)
	LOINC("loinc-src-data", ".*$", 
			"A typical LOINC version number is '2.59'.  The version numbers should be used directly from LOINC.  There are no enforced restrictions on the format.",
			new String[] {}, new String[] {}, new UploadFileInfo[] {
			new UploadFileInfo("", "https://loinc.org/downloads/loinc", 
					"LOINC_2.54_Text.zip",
					"The primary LOINC file is the 'LOINC Table File' in the csv format'.  This should be a zip file that contains a file named 'loinc.csv'."
					+ "  Additionally, the zip file may (optionally) contain 'map_to.csv' and 'source_organization.csv'."
					+ "  The zip file must contain 'text' within its name.", ".*text.*\\.zip$", true),
			new UploadFileInfo("", "https://loinc.org/downloads/files/loinc-multiaxial-hierarchy", 
					"LOINC_2.54_MULTI-AXIAL_HIERARCHY.zip",
					"The Multiaxial Hierarchy file is a zip file that contains a file named *multi-axial_hierarchy.csv.  The zip file containing the multiaxial hierarchy"
					+ " must contain 'MultiAxialHierarchy' within its name, with or without hypens or underscores at word breaks", ".*multi[\\-]?axial[_]?hierarchy.*\\.zip$", true),
			new UploadFileInfo("", "https://loinc.org/downloads/loinc",
					"LOINC_ReleaseNotes.txt",
					"The LOINC Release Notes file must be included for recent versions of LOINC.", ".*releasenotes\\.txt$", true)
	}, "loinc-ibdf", "convert-loinc-to-ibdf", "LOINC", 
			new String[] {"shared/licenses/loinc.xml"}, 
			new String[] {"shared/noticeAdditions/loinc-NOTICE-addition.txt"}),
	
	//This one is deprecated, it was still the old form of the data.
	LOINC_TECH_PREVIEW("loinc-src-data-tech-preview", ".*$", 
			"A typical LOINC tech preview version number is '2015.08.01'.  The version numbers should be used directly from LOINC.  There are no enforced restrictions "
			+ "on the format.", 
			new String[] {"loinc-src-data"}, new String[] {"rf2-ibdf-sct"}, new UploadFileInfo[] {
			new UploadFileInfo("", "https://www.nlm.nih.gov/healthit/snomedct/international.html",
					"SnomedCT_LOINC_AlphaPhase3_INT_20160401.zip",
					"  The expected file is the RF2 release (NOT the Human Readable release nor the OWL release). "
					+ "The file must be a zip file, which ends with .zip", ".*\\.zip$", true)
	}, "loinc-ibdf-tech-preview", "convert-loinc-tech-preview-to-ibdf", "LOINC Tech Preview", 
			new String[] {"shared/licenses/loinc.xml", "shared/licenses/sct.xml"}, 
			new String[] {"shared/noticeAdditions/loinc-tech-preview-NOTICE-addition.txt", "shared/noticeAdditions/loinc-NOTICE-addition.txt", "shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	
	SCT("rf2-src-data-sct", "\\d{8}.*$", 
			"A typical Snomed version number is '20170131' or '20170131T120000'.  The value here should be the same as the version number in the name of the uploaded "
			+ "zip file.  This requires a 4 digit year, 2 digit month, 2 digit day.  Any values can be appended after the 8 digits.",
			new String[] {}, new String[] {}, new UploadFileInfo[] {
			new UploadFileInfo("", "https://www.nlm.nih.gov/healthit/snomedct/international.html",
					"SnomedCT_RF2Release_INT_20160131.zip",
					"The expected file is the RF2 release zip file.  The filename must end with .zip, and must contain the release date in the Snomed standard"
					+ " naming convention (4 digit year, 2 digit month, 2 digit day).",
					".*_\\d{8}.*\\.zip$", true)
	}, "rf2-ibdf-sct", "convert-RF2-direct-to-ibdf", "SnomedCT", 
			new String[] {"shared/licenses/sct.xml"},
			new String[] {"shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	
	SCT_EXTENSION("rf2-src-data-*-extension", "\\d{8}.*$", 
			"A typical Snomed extension version number is '20170131' or '20170131T120000'.  The value here should be the same as the version number in the name of the uploaded "
			+ "zip file.  This requires a 4 digit year, 2 digit month, 2 digit day.  Any values can be appended after the 8 digits.",
			new String[] {}, new String[] {"rf2-ibdf-sct"}, new UploadFileInfo[] {
			new UploadFileInfo("Snomed Extensions come from a variety of sources.  Note that the NLM has choosen to stop advertising the download links to the "
					+ " US Extension, but still publishes it.  The current download pattern is now: "
					+ "https://download.nlm.nih.gov/mlb/utsauth/USExt/SnomedCT_USExtensionRF2_Production_YYYYMMDDTHHMMSS.zip",
					"",
					"SnomedCT_USExtensionRF2_Production_20170301T120000.zip",
					"The expected file is the RF2 release zip file.  The filename must end with .zip, and must contain the release date in the Snomed standard"
					+ " naming convention (4 digit year, 2 digit month, 2 digit day) - it also now also accepts the new naming convention with T and 2 digits each "
					+ "of hour, minute and second.",
					".*_\\d{8}.*\\.zip$", true)
	}, "rf2-ibdf-", "convert-RF2-direct-to-ibdf", "SnomedCT Extension", 
			new String[] {"shared/licenses/sct.xml"},
			new String[] {"shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	
	SCT_EDITION("rf2-src-data-*-edition", "\\d{8}.*$", 
			"A typical Snomed edition version number is '20170131' or '20170131T120000'.  The value here should be the same as the version number in the name of the uploaded "
			+ "zip file.  This requires a 4 digit year, 2 digit month, 2 digit day.  Any values can be appended after the 8 digits.",
			new String[] {}, new String[] {}, new UploadFileInfo[] {
			new UploadFileInfo("Snomed Editions are a combination of the international release, plus an extension.",
					"",
					"SnomedCT_USEditionRF2_PRODUCTION_20200901T120000Z",
					"The expected file is the RF2 release zip file.  The filename must end with .zip, and must contain the release date in the Snomed standard"
					+ " naming convention (4 digit year, 2 digit month, 2 digit day) - it also now also accepts the new naming convention with T and 2 digits each "
					+ "of hour, minute and second.",
					".*_\\d{8}.*\\.zip$", true)
	}, "rf2-ibdf-", "convert-RF2-direct-to-ibdf", "SnomedCT Edition", 
			new String[] {"shared/licenses/sct.xml"},
			new String[] {"shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	
	SCT_EXTENSION_LOINC("rf2-src-data-loinc-extension", "\\d{8}.*$", 
			"A typical Snomed extension version number is '20170131' or '20170131T120000'.  The value here should be the same as the version number in the name of the uploaded "
			+ "zip file.  This requires a 4 digit year, 2 digit month, 2 digit day.  Any values can be appended after the 8 digits.  This should be used for the "
					+ "LOINC snomed collaboration RF2 release.  This requires a loinc file - and should be the 'solor' converted loinc file.",
			new String[] {}, new String[] {"rf2-ibdf-sct", "loinc-ibdf"}, new UploadFileInfo[] {
			new UploadFileInfo("The LOINC coop file comes from IHTSDO - https://www.nlm.nih.gov/healthit/snomedct/international.html",
					"https://download.nlm.nih.gov/umls/kss/IHTSDO20170731/SnomedCT_LOINCRF2_PRODUCTION_20170831T120000Z.zip",
					"SnomedCT_LOINCRF2_PRODUCTION_YYYYMMDDTHHMMSSZ.zip",
					"The expected file is the RF2 release zip file.  The filename must end with .zip, and must contain the release date in the Snomed standard"
					+ " naming convention (4 digit year, 2 digit month, 2 digit day) - it also now also accepts the new naming convention with T and 2 digits each "
					+ "of hour, minute and second.",
					".*_\\d{8}.*\\.zip$", true)
	}, "rf2-ibdf-loinc", "convert-RF2-direct-to-ibdf", "SnomedCT Loinc Extension", 
			new String[] {"shared/licenses/sct.xml"},
			new String[] {"shared/noticeAdditions/rf2-sct-NOTICE-addition.txt"}),
	
{'fraction_non_alphanumeric': 0.12129046762589928, 'fraction_numerical': 0.02731564748201439, 'mean_word_length': 3.8284938941655358, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 18, 'lorem ipsum': 0, 'www.': 10, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2021529', 'n_tokens_mistral': 6391, 'n_tokens_neox': 5495, 'n_words': 1927}