<commit_before>*** Settings ***
Documentation 	This test goes through every file in the corpus and sees if a conversion process occurred and if only the accepted errors were passed through and if the XML files matches JATS format.   

Library          	OperatingSystem
Library 		String
Library			XML
Library 		Collections
Library			Process

*** User Keywords ***
			

	
Titles in Sample Body Found at Some Place in Convert
	[arguments]	${filePath}
	[Timeout]	30 sec 	
	${sXMLRawText}= 	Get File 	/var/local/meTypesetTests/testingDocuments/OpenMedXML/${filePath}.nxml
	${sXMLRefinedText}= 	Replace String Using Regexp 	${sXMLRawText}		(<xref)(.*?)(/xref>)	${EMPTY}
	${sXMLText}= 		Parse XML 	${sXMLRefinedText}
	${sXMLPTexts}= 		Get Elements Texts 	${sXMLText}	body/sec/title
	${cXMLRawText}=		Get File 	/var/local/meTypesetTests/tests/testOutput/${filePath}.docx/nlm/out.xml
	${cXMLParsedText}= 	Parse XML 	${cXMLRawText}
	${cXMLPTexts}=		Get Elements Texts 	${cXMLParsedText}	body/sec/title
	:FOR	${text}		IN 	@{sXMLPTexts}
	# The character '—' is a special character that can cause issues depending on the version of python the machine is implementing. Because of this, the removal of this
	# character is done in the context of the test.  
#	\	${noDashesText}=	Replace String 		${text}		—	\ 		 
#	\ 	Create File 	spaced 		${noDashesText}
#	\	Run Process		python spaceString.py spaced 		shell=True
#	\	${spacedString}=	Get File	spaced
	\ 	Log Variables
	\	List Should Contain Value		${cXMLPTexts}		${text}	

# Runs only when error file from xmllint test of meTypeset'd xml returns an error when checked against JATS Green DTD.
# Checks to make sure the only error is "Document ./testOutput/OpenMed-###-e###./nlm/out.xml: ### element xref: validity error IDREFS attribute 
# references an unknown ID "TO_LINK". 

Acceptable Errors Present
	[arguments]		${errors}
	${errorFile}= 	Get File 	${errors}
	${ret}= 		Should Match Regexp		${errorFile} 	(OpenMed-\\d*-e*\\d*?..docx/nlm/out.xml:\\d*: element xref: validity error : IDREFS attribute rid references an unknown ID "TO_LINK")?(Document OpenMed-\\d*-e*\\d*?..docx/nlm/out.xml does not validate against http://jats.nlm.nih.gov/archiving/1.1d3/JATS-archivearticle1.dtd)

# Checks to make sure that a converted file is present in the expected place. If there is an error present, a check is done to make sure that only
# expected errors are present. 

File is Converted and Errors Highlighted
	[arguments] 	${filePath}
	File Should Exist 	testOutput/${filePath}.docx/nlm/out.xml
	${status}	Run Keyword and Ignore Error 		File Should Be Empty         error/${filePath}.docx 
	Run Keyword If 		${status} != ('PASS', None)		Acceptable Errors Present 		  error/${filePath}.docx   

*** Test Cases ***

OpenMed-01-e1   
	File is Converted and Errors Highlighted	OpenMed-01-e1  
<commit_msg>Testing done for Title <article><title> Author (first, last) <contrib-group><contrib><name><surname><given-name> Affiliation <aff> Foot Notes <fn> <fn-group> Caption <fig / table-wrap><caption> Header/Footer Paragraph <sec><p> Reference (inline, section) inline = <xref> section = <ref> Section Header <sec><title> Image <fig><graphic> Table <fig><table-wrap><table> Abstract <abstract><p><commit_after>*** Settings ***
Documentation 	This test goes through every file in the corpus and sees if a conversion process occurred and if only the accepted errors were passed through and if the XML files matches JATS format.   

Library          	OperatingSystem
Library 		String
Library			XML
Library 		Collections
Library			Process

*** User Keywords ***
			
	
Element in Sample Body Found in Convert
	[arguments]	${filePath}	${xpath}
	[Timeout]	30 sec 	
	${sXMLRawText}= 	Get File 	/var/local/meTypesetTests/testingDocuments/OpenMedXML/${filePath}.nxml
	${sXMLRefinedText}= 	Replace String Using Regexp 	${sXMLRawText}		(<xref[^>]>)([^<]*)(</xref>)	${EMPTY}
	${sXMLText}= 		Parse XML 	${sXMLRefinedText}
	${sXMLPTexts}= 		Get Elements Texts 	${sXMLText}		${xpath}
	${cXMLRawText}=		Get File 	/var/local/meTypesetTests/tests/testOutput/${filePath}.docx/nlm/out.xml
	${cXMLParsedText}= 	Parse XML 	${cXMLRawText}
	${cXMLPTexts}=		Get Elements Texts 	${cXMLParsedText}	${xpath}
	${status}=	Run Keyword And Return Status 		Should Not Be Empty 	${cXMLPTexts}
	Run Keyword If 		'${status}' == 'False'			Return From Keyword
	Should Not Be Empty 	${sXMLPTexts}		Sample XML has no tags matching ${xpath} while the converted file does. 	
	:FOR	${text}		IN 	@{sXMLPTexts}
	\ 	Log Variables
	\	List Should Contain Value		${cXMLPTexts}		${text}		The text '${text}' is present in the sample but  not found in the meTypeset'd xml. 

# Runs only when error file from xmllint test of meTypeset'd xml returns an error when checked against JATS Green DTD.
# Checks to make sure the only error is "Document ./testOutput/OpenMed-###-e###./nlm/out.xml: ### element xref: validity error IDREFS attribute 
# references an unknown ID "TO_LINK". 

Acceptable Errors Present
	[arguments]		${errors}
	${errorFile}= 	Get File 	${errors}
	${ret}= 		Should Match Regexp		${errorFile} 	(OpenMed-\\d*-e*\\d*?..docx/nlm/out.xml:\\d*: element xref: validity error : IDREFS attribute rid references an unknown ID "TO_LINK")?(Document OpenMed-\\d*-e*\\d*?..docx/nlm/out.xml does not validate against http://jats.nlm.nih.gov/archiving/1.1d3/JATS-archivearticle1.dtd)

# Checks to make sure that a converted file is present in the expected place. If there is an error present, a check is done to make sure that only
# expected errors are present. 

File is Converted and Errors Highlighted
	[arguments] 	${filePath}
	File Should Exist 	testOutput/${filePath}.docx/nlm/out.xml
	${status}	Run Keyword and Ignore Error 		File Should Be Empty         error/${filePath}.docx 
	Run Keyword If 		${status} != ('PASS', None)		Acceptable Errors Present 		  error/${filePath}.docx   

File Test Cases
	[arguments] 	${filePath}
	File is Converted and Errors Highlighted 	${filePath}
	Run Keyword and Continue on Failure	 	Element in Sample Body Found in Convert 	${filePath} 	./article/title
	Run Keyword and Continue on Failure		Element in Sample Body Found in Convert 	${filePath}	.//aff
	Run Keyword and Continue on Failure	 	Element in Sample Body Found in Convert 	${filePath}	.//p
	Run Keyword and Continue on Failure 		Element in Sample Body Found in Convert 	${filePath}	.//abstract/p
	Run Keyword and Continue on Failure             Element in Sample Body Found in Convert         ${filePath}	.//fig/graphic
	Run Keyword and Continue on Failure             Element in Sample Body Found in Convert         ${filePath} 	.//fig/table-wrap/table
	Run Keyword and Continue on Failure             Element in Sample Body Found in Convert         ${filePath} 	.//sec/title
	Run Keyword and Continue on Failure             Element in Sample Body Found in Convert         ${filePath} 	.//fig/table-wrap/caption
	Run Keyword and Continue on Failure             Element in Sample Body Found in Convert         ${filePath} 	.//fn/fn-group
	

*** Test Cases ***

OpenMed-01-e1   
	File Test Cases 	OpenMed-01-e1
{'fraction_non_alphanumeric': 0.10877539503386005, 'fraction_numerical': 0.003386004514672686, 'mean_word_length': 3.7866306549628628, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 17}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2740208', 'n_tokens_mistral': 2417, 'n_tokens_neox': 2304, 'n_words': 807}