<commit_before>	*! version 3.0 26DEC2016  <NAME> <EMAIL>

	cap program drop 	iematch
	    program define	iematch
				
		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			]

		*****

	qui {
		
		* Set version
		version 11.0
		
		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************
		
		tempvar originalSort ifinvar
		
		**Generate a variable that is used 
		* to restore original sort.
		gen `originalSort' = _n

		preserve
			
			* String used to store why and how 
			*  many observations are excluded.
			local obsexcludstring ""
	
			*Deal with if and in and display info. 
			if "`if'`in'" != "" {
				
				* Gen dummy that is 1 when if/in is true. Then
				* count when this dummy is not 1. Ther is no way 
				* to negate `if'`in'
				gen 	 `ifinvar'  = 1 `if'`in'
				count if `ifinvar' != 1
				if `r(N)' > 0 {
					
					*This is not an error just outputting the number
					local obsexcludstring "`obsexcludstring'`r(N)' observation(s) were excluded in {inp:if}/{inp:in}: (`if'`in').{break}"
					
				}
				
				*Drop variables excluded by `if'`in'
				keep `if'`in'
			}
			
			
			********************************
			*
			*	Keep only relevant vars
			*
			********************************
			
			*Keep only input vars
			keep  `grpdummy' `idvar' `matchvar'	`originalSort'			
			
			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "{pstd}The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command.{p_end}"
				error _rc
			}
			
			**********
			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {
			
				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in  grpdummy(`grpdummy').{break}"
			}
			
			*Drop obs with missing value in groupvar
			drop if missing(`grpdummy')


			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "{pstd}The variable to match on specified in matchvar(`matchvar') is not a numeric variable.{p_end}"
				error 109
			}

			**All variables with value 1 or 0 in the group dummy and 
			* not excluded by if/in must have a value in matchvar 
			count if missing(`matchvar')
			if `r(N)' > 0 {
				
				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in matchvar(`matchvar').{break}"	
			}
			
			*Drop obs with missing values in match var
			drop if missing(`matchvar')
		
			
			********************************
			*
			*	Checking match var and group dummy are unique
			*
			********************************			
			
			if "`seedok'" == "" {
				
				**test that there are no duplicates in match 
				* var within each type of observation
				cap isid `matchvar' `grpdummy'
				
				if _rc != 0 {
					
					*Output the error message
					noi di as error "{pstd}There are base observations or target observations {...}"
					noi di as error "with duplicate values in matchvar(`matchvar'). To guarantee {...}"
					noi di as error "a replicable match you must {help seed:set a seed}. To supress {...}"
					noi di as error "this error message after you have set a the seed, or if a {...}"
					noi di as error "replicable match is not important to you, use option {inp:seedok}{p_end}"
					error 198
				
				}
			}
			

			********************************
			*
			*	Checking that there are more target 
			*	obs than base obs in a 1-to-1 match
			*
			********************************			
			
			if "`m1'" == "" {
				
				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'
				
				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'
				
				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs' 
				
				if _rc != 0 {
					
					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternaitve matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}			
			
			********************************
			*
			*	Checking duplicates in variable names in options
			*
			********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresult'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				di as error "{pstd}The same new variable name was used twice or more in the options generating a new variable. Go back and check syntax.{p_end}"
				error 198
			}


			********************************
			*
			*	Checking ID var
			*
			********************************


			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "{pstd}A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set.{p_end}"
					error 110
				}

				*Generate the ID var from row number
				gen `idvar' = _n
				
				*Store local indicating ID var generated has type numeric
				local IDtypeNumeric 1

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "{pstd}Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}.{p_end}"
					error 450
				}
				
				*Test if ID var is string or numeric
				cap confirm numeric variable `idvar'
				if _rc == 0 {
					
					*Store local indicating ID var type is numeric
					local IDtypeNumeric 1
				}
				else {
				
					*Store local indicating ID var type is numeric
					local IDtypeNumeric 0			
				}
			}

			********************************
			*
			*	Checking user defined names
			*	and then create the result
			*	varaibles
			*
			********************************
			
			*******
			* Result var
			
			iematchMatchVarCheck _matchResult `matchresultname'
			local 	 		 matchResultName "`r(validVarName)'"
			gen 			`matchResultName' = .
			label variable	`matchResultName' "Matched obs = 1, not mathched = 0, all other different missing values"
			
			*Create a label for missing values that explains no match
			label define matchLabel 0 "Not Matched" 1 "Matched" .i "Obs excluded due to if/in" .g "Missing value in `grpdummy'" .m "Missing value in `matchvar'" .d "No match within maxdiff"

			*Apply the label
			label value `matchResultName' matchLabel
			
			*******
			*ID var
			
			iematchMatchVarCheck _matchID `matchidname'
			local matchIDname "`r(validVarName)'"
	
			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `matchIDname' = .
			}
			else {
				gen  `matchIDname' = ""
			}
			
			*This variable is slightly differnt between a many-one and one-one match
			if "`m1'" != "" {
				label variable	`matchIDname' "The ID of the target var in each matched group"
			}
			else {
				label variable	`matchIDname' "The ID of the target var in each matched pair"
			}
			
			*******
			*Diff var
			
			iematchMatchVarCheck _matchDiff `matchdiffname'
			local 	 matchDiffName "`r(validVarName)'"
			gen 	`matchDiffName' = .
			
			*This variable is slightly differnt between a many-one and one-one match
			if "`m1'" != "" {
				label variable	`matchDiffName' "The difference in matchvar() between base obs and the target obs it matched with"
			}
			else {
				label variable	`matchDiffName' "The difference in matchvar() between base and target obs in each pair"
			}
			
			*******
			*Count var
			
			*If many-to-one match used
			if "`m1'" != "" {
				iematchMatchVarCheck _matchCount `matchcountname'
				local 			 matchCountName "`r(validVarName)'"
				gen 			`matchCountName' = .
				label variable	`matchDiffName' "The number of base obs this target obs matched with"
				
			}
			else {
				
				*Option match count name is not allowed if it is a one-to-one match
				if "`matchcountname'" != "" {
				
					di as error "{pstd}Option {inp:matchcountname()} is only allowed in combination with option {inp:m1}.{p_end}"
					error 198
				}
			}

		********************************
		*
		*	Output exclude string
		*
		********************************				
			
			*The exclude string is created above but only displayed 
			*after all testing of input is done. 
			
			if "`obsexcludstring'" != "" {
				
				noi di ""
				noi di "{hline}"
				noi di ""
				noi di "{pstd}{ul:Observations are excluded from the matching for the following reasons:}{break}"
				noi di "`obsexcludstring'{p_end}"

			}

	********************************
	*	Start matching
	********************************
		
		
		********************************
		*
		*	Creating tempvar used in matching
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar prefID prefDiff matched

		gen `prefDiff' 		= .
		gen byte `matched' 	= 0

		*Allow the ID var used to be string
		if `IDtypeNumeric' == 1 {
			gen  `prefID' = .
		}
		else {
			gen  `prefID' = ""
		}			
		

		** Generate the inverse of the matchvar to sort descending (gsort is too slow),
		*  a random var to seperate two values with the same match var, and the inverse
		*  of the random var for when sorting descending.
		tempvar rand invsort invrand

		sort `matchvar'
		gen  `invsort' = -1 * `matchvar'

		gen    `rand' = uniform()
		gen `invrand' = -1 * `rand'

		***********
		*Tempvars for matching

		*Diffvars, they are always numeric
		tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1   
		local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

		foreach tempVar of local updownTempVars {

			gen `tempVar' = .
		}
		
		*ID vars, allowed to be both numeric and string
		tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
		local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

		foreach tempVar of local updownIDTempVars {
			
			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `tempVar' = .
			}
			else {
				gen  `tempVar' = ""
			}					
		}			

		***************************
		*
		*	One to one match
		*
		***************************

		if "`m1'" == "" {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching one-to-one. Base observations left to match:}{p_end}"
			count if `grpdummy' == 1 & `matched' == 0
			
			*Create local to display "obs left to match" and to use in while loop
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c
			
			*Match until no more observations to match.
			while (`left2Match' > 0) {
				
				**For all observations still to be matched, assign the preferred
				* match among the other unmatched observations
				qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'
				
				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {
					
					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1
					
					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1
					
					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}	
				}
				
				*If two observations mutually prefer each other, then indicate both of them as matched.
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}
			
			*End formatting for the "base obs left to match"
			noi di "{p_end}" _c
		}

		***************************
		*
		*	Many to one match
		*
		***************************
		else {

			**For all observations to be matched, assign the preferred
			* match among the other unmatched observations
			updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

			*Restrict matches to within maxdiff() if that option is used.
			if "`maxdiff'" != "" {
				
				*Indicate in result var that this obs did not have valid match within maxdiff()
				replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1
				
				*Removed preferred match
				if `IDtypeNumeric' == 1 {
					*IDvar is numeric
					replace `prefID'		= .		if `prefDiff' > `maxdiff'
				}
				else {
					*IDvar is string
					replace `prefID'		= ""	if `prefDiff' > `maxdiff'
				}	
			}
			
			*Assign it's own ID as pref ID for all target vars
			replace `prefID'			=  `idvar' if `grpdummy' == 0
			
			* Replace the _matchCount var with number of base observations in each 
			* match group. Each group is all base observation plus the target 
			* observation, therefore (_N - 1)
			bys 	`prefID' 			:   replace `matchCountName' = _N - 1
		

			**Replace prefID to missing for target obs that had no base 
			* obs matched to it. T
			if `IDtypeNumeric' == 1 { 
				*IDvar is numeric
				replace `prefID'		= .		if `matchCountName' == 0
			}
			else {
				*IDvar is string
				replace `prefID'		= ""	if `matchCountName' == 0
			}					
			
			*Only target obs with base obs prefering it are matched
			replace	 `matched' 			= 1		if `matchCountName' != 0
		
			*Remove values for target obs that were not matched
			replace  `matchCountName'	= . 	if `matchCountName' == 0
		}


		*Update all return vars
		replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
		replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
		replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1
		
		*Matched observations are give value 1 in result var
		replace `matchResultName' = 1 				if `matched' == 1 & `matchResultName' != .d
		
		*Target obs not used
		replace `matchResultName' = 0 if `matched' == 0 & `grpdummy' == 0

		*only keep output vars
		keep `matchIDname' `matchDiffName' `originalSort' `matchResultName' `matchCountName'

		*Merge the results back to the original data set
		tempfile mergefile
		save 	`mergefile'

	restore

	***************************
	*
	*	Merge match results to 
	*	original data and assign
	*	remaining missing values
	*	to the result var
	*
	***************************		

	*Merge the results with the original data
	tempvar mergevar 
	merge 1:1  `originalSort' using `mergefile', gen(`mergevar')	
	<commit_msg>iematch : remove match info if maxdiff()<commit_after>	*! version 3.0 26DEC2016  <NAME> <EMAIL>

	cap program drop 	iematch
	    program define	iematch
				
		syntax  [if] [in]  , 					///
			GRPdummy(varname) 					///
			MATCHvar(varname) 					///
			[									///
			IDvar(varname)						///
			m1									///
			maxdiff(numlist max = 1 min = 1) 	///
			seedok						 		///
			MATCHIDname(string)					///
			MATCHDIffname(string)				///
			MATCHREsultname(string)				///
			MATCHCOuntname(string)				///
			]

		*****

	qui {
		
		* Set version
		version 11.0
		
		********************************
		*
		*	Gen temp sort and merge var
		*
		********************************
		
		tempvar originalSort ifinvar
		
		**Generate a variable that is used 
		* to restore original sort.
		gen `originalSort' = _n

		preserve
			
			* String used to store why and how 
			*  many observations are excluded.
			local obsexcludstring ""
	
			*Deal with if and in and display info. 
			if "`if'`in'" != "" {
				
				* Gen dummy that is 1 when if/in is true. Then
				* count when this dummy is not 1. Ther is no way 
				* to negate `if'`in'
				gen 	 `ifinvar'  = 1 `if'`in'
				count if `ifinvar' != 1
				if `r(N)' > 0 {
					
					*This is not an error just outputting the number
					local obsexcludstring "`obsexcludstring'`r(N)' observation(s) were excluded in {inp:if}/{inp:in}: (`if'`in').{break}"
					
				}
				
				*Drop variables excluded by `if'`in'
				keep `if'`in'
			}
			
			
			********************************
			*
			*	Keep only relevant vars
			*
			********************************
			
			*Keep only input vars
			keep  `grpdummy' `idvar' `matchvar'	`originalSort'			
			
			********************************
			*
			*	Checking group dummy
			*
			********************************

			*Test that the group dummy is 1, 0 or missing
			cap assert inlist(`grpdummy', 1, 0, .) | `grpdummy' > .
			if _rc != 0 {

				di as error "{pstd}The variable in grpdummy(`grpdummy') is not a dummy variable. The variable is only allowed to have the values 1, 0 or missing. Observations with missing varaibles in the grpdummy are ignored by this command.{p_end}"
				error _rc
			}
			
			**********
			**Exclude obs with missing value in groupdummy

			*Count number of obs to be dropped and and output that number if more than zero
			count if missing(`grpdummy')
			if `r(N)' > 0 {
			
				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in  grpdummy(`grpdummy').{break}"
			}
			
			*Drop obs with missing value in groupvar
			drop if missing(`grpdummy')


			********************************
			*
			*	Checking match continous var
			*
			********************************

			*Test that the continous is numeric
			cap confirm numeric variable `matchvar'
			if _rc != 0 {

				noi di as error "{pstd}The variable to match on specified in matchvar(`matchvar') is not a numeric variable.{p_end}"
				error 109
			}

			**All variables with value 1 or 0 in the group dummy and 
			* not excluded by if/in must have a value in matchvar 
			count if missing(`matchvar')
			if `r(N)' > 0 {
				
				*Prepare the local to be outputted with info on observations excluded
				local obsexcludstring  "`obsexcludstring'`r(N)' observation(s) were excluded due to missing value in matchvar(`matchvar').{break}"	
			}
			
			*Drop obs with missing values in match var
			drop if missing(`matchvar')
		
			
			********************************
			*
			*	Checking match var and group dummy are unique
			*
			********************************			
			
			if "`seedok'" == "" {
				
				**test that there are no duplicates in match 
				* var within each type of observation
				cap isid `matchvar' `grpdummy'
				
				if _rc != 0 {
					
					*Output the error message
					noi di as error "{pstd}There are base observations or target observations {...}"
					noi di as error "with duplicate values in matchvar(`matchvar'). To guarantee {...}"
					noi di as error "a replicable match you must {help seed:set a seed}. To supress {...}"
					noi di as error "this error message after you have set a the seed, or if a {...}"
					noi di as error "replicable match is not important to you, use option {inp:seedok}{p_end}"
					error 198
				
				}
			}
			

			********************************
			*
			*	Checking that there are more target 
			*	obs than base obs in a 1-to-1 match
			*
			********************************			
			
			if "`m1'" == "" {
				
				*Count number of base vars
				count if `grpdummy' == 1
				local numBaseObs `r(N)'
				
				*Count number of target vars
				count if `grpdummy' == 0
				local numTrgtObs `r(N)'
				
				*Test that there are more target
				cap assert `numTrgtObs' >= `numBaseObs' 
				
				if _rc != 0 {
					
					noi di as error "{pstd}There are more base observations than target observations. This is not allowed in a one-to-one match. See option {inp:m1} for an alternaitve matching where it is allowed to have more base observations than target ovbservations.{p_end}"
					error _rc
				}
			}			
			
			********************************
			*
			*	Checking duplicates in variable names in options
			*
			********************************

			*Create a local of all varnames used in any option
			local allnewvars `idvar' `matchidname' `matchdiffname' `matchresult'

			*Check if there are any duplicates in the local of all varnames in options
			local dupnewvars : list dups allnewvars

			*Throw error if there were any duplicates
			if "`dupnewvars'" != "" {

				di as error "{pstd}The same new variable name was used twice or more in the options generating a new variable. Go back and check syntax.{p_end}"
				error 198
			}


			********************************
			*
			*	Checking ID var
			*
			********************************


			*Checking set-up for the ID var used for _matchID
			if "`idvar'" == "" {

				*Copy this name
				local idvar _ID

				*Make sure that the default name _ID is not already used
				cap confirm variable `idvar'
				if _rc == 0 {

					di as error "{pstd}A variable with name `idvar' is already defined. Either drop this variable or specify a variable using idvar() that fully and uniquely identifies the data set.{p_end}"
					error 110
				}

				*Generate the ID var from row number
				gen `idvar' = _n
				
				*Store local indicating ID var generated has type numeric
				local IDtypeNumeric 1

			}
			else {

				*Make sure that user specified ID is uniquely and fully identifying the observations
				cap isid `idvar'
				if _rc != 0 {

					di as error "{pstd}Variable `idvar' specified in idvar() does not fully and uniquely identify the observations in the data set. Meaning that `idvar' is not allowed to have any duplicates or missing values. Make `idvar' uniquly and fully idnetifying or exclude the varaibleas using {inp:if} or {inp:in}.{p_end}"
					error 450
				}
				
				*Test if ID var is string or numeric
				cap confirm numeric variable `idvar'
				if _rc == 0 {
					
					*Store local indicating ID var type is numeric
					local IDtypeNumeric 1
				}
				else {
				
					*Store local indicating ID var type is numeric
					local IDtypeNumeric 0			
				}
			}

			********************************
			*
			*	Checking user defined names
			*	and then create the result
			*	varaibles
			*
			********************************
			
			*******
			* Result var
			
			iematchMatchVarCheck _matchResult `matchresultname'
			local 	 		 matchResultName "`r(validVarName)'"
			gen 			`matchResultName' = .
			label variable	`matchResultName' "Matched obs = 1, not mathched = 0, all other different missing values"
			
			*Create a label for missing values that explains no match
			label define matchLabel 0 "Not Matched" 1 "Matched" .i "Obs excluded due to if/in" .g "Missing value in `grpdummy'" .m "Missing value in `matchvar'" .d "No match within maxdiff"

			*Apply the label
			label value `matchResultName' matchLabel
			
			*******
			*ID var
			
			iematchMatchVarCheck _matchID `matchidname'
			local matchIDname "`r(validVarName)'"
	
			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `matchIDname' = .
			}
			else {
				gen  `matchIDname' = ""
			}
			
			*This variable is slightly differnt between a many-one and one-one match
			if "`m1'" != "" {
				label variable	`matchIDname' "The ID of the target var in each matched group"
			}
			else {
				label variable	`matchIDname' "The ID of the target var in each matched pair"
			}
			
			*******
			*Diff var
			
			iematchMatchVarCheck _matchDiff `matchdiffname'
			local 	 matchDiffName "`r(validVarName)'"
			gen 	`matchDiffName' = .
			
			*This variable is slightly differnt between a many-one and one-one match
			if "`m1'" != "" {
				label variable	`matchDiffName' "The difference in matchvar() between base obs and the target obs it matched with"
			}
			else {
				label variable	`matchDiffName' "The difference in matchvar() between base and target obs in each pair"
			}
			
			*******
			*Count var
			
			*If many-to-one match used
			if "`m1'" != "" {
				iematchMatchVarCheck _matchCount `matchcountname'
				local 			 matchCountName "`r(validVarName)'"
				gen 			`matchCountName' = .
				label variable	`matchDiffName' "The number of base obs this target obs matched with"
				
			}
			else {
				
				*Option match count name is not allowed if it is a one-to-one match
				if "`matchcountname'" != "" {
				
					di as error "{pstd}Option {inp:matchcountname()} is only allowed in combination with option {inp:m1}.{p_end}"
					error 198
				}
			}

		********************************
		*
		*	Output exclude string
		*
		********************************				
			
			*The exclude string is created above but only displayed 
			*after all testing of input is done. 
			
			if "`obsexcludstring'" != "" {
				
				noi di ""
				noi di "{hline}"
				noi di ""
				noi di "{pstd}{ul:Observations are excluded from the matching for the following reasons:}{break}"
				noi di "`obsexcludstring'{p_end}"

			}

	********************************
	*	Start matching
	********************************
		
		
		********************************
		*
		*	Creating tempvar used in matching
		*
		********************************

		*Initiate the temporary variables used by this command
		tempvar prefID prefDiff matched

		gen `prefDiff' 		= .
		gen byte `matched' 	= 0

		*Allow the ID var used to be string
		if `IDtypeNumeric' == 1 {
			gen  `prefID' = .
		}
		else {
			gen  `prefID' = ""
		}			
		

		** Generate the inverse of the matchvar to sort descending (gsort is too slow),
		*  a random var to seperate two values with the same match var, and the inverse
		*  of the random var for when sorting descending.
		tempvar rand invsort invrand

		sort `matchvar'
		gen  `invsort' = -1 * `matchvar'

		gen    `rand' = uniform()
		gen `invrand' = -1 * `rand'

		***********
		*Tempvars for matching

		*Diffvars, they are always numeric
		tempvar					 diffup   diffdo   valUp_0   valDo_0   valUp_1   valDo_1   
		local 	 updownTempVars `diffup' `diffdo' `valUp_0' `valDo_0' `valUp_1' `valDo_1'

		foreach tempVar of local updownTempVars {

			gen `tempVar' = .
		}
		
		*ID vars, allowed to be both numeric and string
		tempvar					  	IDup   IDdo   IDup_0   IDdo_0   IDup_1   IDdo_1
		local 	 updownIDTempVars  `IDup' `IDdo' `IDup_0' `IDdo_0' `IDup_1' `IDdo_1'

		foreach tempVar of local updownIDTempVars {
			
			*Allow the ID var used to be string
			if `IDtypeNumeric' == 1 {
				gen  `tempVar' = .
			}
			else {
				gen  `tempVar' = ""
			}					
		}			

		***************************
		*
		*	One to one match
		*
		***************************

		if "`m1'" == "" {

			*Start outputting the countdown
			noi di ""
			noi di "{hline}{break}"
			noi di "{pstd}{ul:Matching one-to-one. Base observations left to match:}{p_end}"
			count if `grpdummy' == 1 & `matched' == 0
			
			*Create local to display "obs left to match" and to use in while loop
			local left2Match = `r(N)'
			noi di "{pstd}`left2Match' " _c
			
			*Match until no more observations to match.
			while (`left2Match' > 0) {
				
				**For all observations still to be matched, assign the preferred
				* match among the other unmatched observations
				qui updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'
				
				*Restrict matches to within maxdiff() if that option is used.
				if "`maxdiff'" != "" {
					
					*Omit base observation from matching if diff is to big
					replace `matched' 			= 1		if `prefDiff' > `maxdiff' & `grpdummy' == 1
					
					*Indicate in result var that this obs did not have valid match within maxdiff()
					replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1
					
					*Removed preferred match
					if `IDtypeNumeric' == 1 {
						*IDvar is numeric
						replace `prefID'		= .		if `prefDiff' > `maxdiff'
					}
					else {
						*IDvar is string
						replace `prefID'		= ""	if `prefDiff' > `maxdiff'
					}	
				}
				
				*If two observations mutually prefer each other, then indicate both of them as matched.
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n-1] & `prefID'[_n-1] == `idvar'
				replace `matched' = 1 if `matched' == 0 & `prefID' == `idvar'[_n+1] & `prefID'[_n+1] == `idvar'

				*Update local to display "obs left to match" and to use in while loop
				count if `grpdummy' == 1 & `matched' == 0
				local left2Match = `r(N)'
				noi di "`left2Match' " _c

			}
			
			*End formatting for the "base obs left to match"
			noi di "{p_end}" _c
		}

		***************************
		*
		*	Many to one match
		*
		***************************
		else {

			**For all observations to be matched, assign the preferred
			* match among the other unmatched observations
			updatePrefDiffPreffID `prefID' `prefDiff' `matchvar' `invsort' `idvar' `grpdummy' `matched' `rand' `invrand' `updownTempVars' `updownIDTempVars'

			*Restrict matches to within maxdiff() if that option is used.
			if "`maxdiff'" != "" {
				
				*Indicate in result var that this obs did not have valid match within maxdiff()
				replace `matchResultName' 	= .d 	if `prefDiff' > `maxdiff' & `grpdummy' == 1
				
				*Removed preferred match
				if `IDtypeNumeric' == 1 {
					*IDvar is numeric
					replace `prefID'		= .		if `prefDiff' > `maxdiff'
				}
				else {
					*IDvar is string
					replace `prefID'		= ""	if `prefDiff' > `maxdiff'
				}	
			}
			
			*Assign it's own ID as pref ID for all target vars
			replace `prefID'			=  `idvar' if `grpdummy' == 0
			
			* Replace the _matchCount var with number of base observations in each 
			* match group. Each group is all base observation plus the target 
			* observation, therefore (_N - 1)
			bys 	`prefID' 			:   replace `matchCountName' = _N - 1 if !missing(`prefID')
		

			**Replace prefID to missing for target obs that had no base 
			* obs matched to it. T
			if `IDtypeNumeric' == 1 { 
				*IDvar is numeric
				replace `prefID'		= .		if `matchCountName' == 0
			}
			else {
				*IDvar is string
				replace `prefID'		= ""	if `matchCountName' == 0
			}					
			
			*Only target obs with base obs prefering it are matched
			replace	 `matched' 			= 1		if `matchCountName' != 0
		
			*Remove values for target obs that were not matched
			replace  `matchCountName'	= . 	if `matchCountName' == 0
		}


		*Update all return vars
		replace `matchDiffName' 	= `prefDiff'	if `matched' == 1
		replace `matchIDname' 		= `idvar' 		if `matched' == 1 & `grpdummy' == 0
		replace `matchIDname' 		= `prefID' 		if `matched' == 1 & `grpdummy' == 1
		
		*Remove the best match value in obs that did not have a match within maxdiff()
		replace `matchDiffName'  == . 				if `matchResultName' == .d
		
		*Matched observations are give value 1 in result var
		replace `matchResultName' = 1 				if `matched' == 1 & `matchResultName' != .d
		
		*Target obs not used
		replace `matchResultName' = 0 if `matched' == 0 & `grpdummy' == 0

		*only keep output vars
		keep `matchIDname' `matchDiffName' `originalSort' `matchResultName' `matchCountName'

		*Merge the results back to the original data set
		tempfile mergefile
		save 	`mergefile'

	restore

	***************************
	*
	*	Merge match results to 
	*	original data and assign
	*	remaining missing values
	*	to the result var
	*
	***************************		

	*Merge the results with the original data
	tempvar mergevar 
	merge 1:1  `originalSort' using `mergefile', gen(`mergevar')	
	
{'fraction_non_alphanumeric': 0.1467687280648675, 'fraction_numerical': 0.008471499455403606, 'mean_word_length': 2.7517593643586835, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '285340', 'n_tokens_mistral': 13211, 'n_tokens_neox': 10116, 'n_words': 3779}