<commit_before>#
# Module 'Dynamite.PowerShell.Toolkit'
# Generated by: GSoft, Team Dynamite.
# Generated on: 10/24/2013
# > GSoft & Dynamite : http://www.gsoft.com
# > Dynamite Github : https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit
# > Documentation : https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit/wiki
#

<#
.SYNOPSIS
	Replaces tokens in all child items with the file extention .template recursively.
	
.DESCRIPTION
	Replaces tokens in all child items with the file extention .template recursively.
	The template item is copied with without the .template extention before the tokens are replaced.
	The tokens are defined in a 'Tokens.Domain.ps1' file.
	Please define tokens as valiables with the prefix 'DSP_'. 
	EX.: $DSP_token1 = "Value 1" will replace [[DSP_token1]] in any .template file
	
    --------------------------------------------------------------------------------------
    Module 'Dynamite.PowerShell.Toolkit'
    by: GSoft, Team Dynamite.
    > GSoft & Dynamite : http://www.gsoft.com
    > Dynamite Github : https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit
    > Documentation : https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit/wiki
    --------------------------------------------------------------------------------------
    
.PARAMETER Path
	The path where all the files are located. By default the value is the current working location.
	
.PARAMETER Domain
	The prefix for the token file 'Tokens.Domain.ps1'. By default the value is the current NetBIOS name.
        
  .LINK
    GSoft, Team Dynamite on Github
    > https://github.com/GSoft-SharePoint
    
    Dynamite PowerShell Toolkit on Github
    > https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit
    
    Documentation
    > https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit/wiki
    
#>
function Update-DSPTokens {
	Param (
		[Parameter(Mandatory=$false, ValueFromPipeline=$true)]
		[ValidateScript({Test-Path $_})]
		[string]$Path = (Get-Location),
		
		[Parameter(Mandatory=$false)]
		[string]$Domain = [System.Net.Dns]::GetHostName(),
		
		[Parameter(Mandatory=$false)]
		[switch]$UseHostName,
		
		[Parameter(Mandatory=$false)]
		[switch]$UseDomain
	)
	
	if ($UseHostName -eq $true) {
		$Domain = [System.Net.Dns]::GetHostName()
	}
	
	if ($UseDomain -eq $true) {
		$Domain = (Get-CurrentDomain)
	}

	$tokenPath = ""
		
	Get-ChildItem -Path $Path -Include "Tokens.$Domain.ps1" -Recurse | foreach {
		$tokenPath = $_.FullName
	}
	
	if (Test-Path $tokenPath) {
		Write-Host "Found token file at : $tokenPath"
		Execute-TokenFile $Path $tokenPath
	}
	else {
		Write-Host "Didn't found the token file named : Tokens.$Domain.ps1"
	}
}

<#
.SYNOPSIS
	Returns the current domain name using wmi. If wmi is not installed, 
	then use the USERDOMAIN environment variable.
#>
function script:Get-CurrentDomain {
	try {
		# Return this version if wmi is installed
		return [string](gwmi Win32_NTDomain).DomainName.Trim()
	} catch {
		# Fall back on this version in case of error
		return $env:USERDOMAIN
	}
}

function script:Execute-TokenFile {
	param (
		$Path,
		$TokenPath
	)
	Write-Host "$TokenPath"
	# Load tokens
	. $TokenPath
	$tokens = Get-Variable -Include "DSP_*"
	
	# Replace tokens in all .template files.
	Get-ChildItem -Path $Path -Include "*.template" -Recurse | foreach {
		Write-Host "Replacing tokens in file '$_'... " -NoNewline
		
		try {
			# Get the contents of the template file.
			$contents  = Get-Content $_ -Encoding UTF8 -ErrorAction Stop
			
			# for each token in our token file, we replace the token in the contents of the file.
			$tokens | ForEach {
				$contents = $contents -replace "\[\[$($_.Name)\]\]", $_.Value
			}
			
			# Write the contents with the replaces tokens to a new file overiding any current file.
			Set-Content -Encoding UTF8 -Value $contents -path $_.FullName.Substring(0, $_.FullName.IndexOf(".template")) -Force -ErrorAction Stop
		} catch {
			Write-Host "Failed - $_" -ForegroundColor Red
		}
		
		Write-Host "Success!" -ForegroundColor Green
	}
}<commit_msg>[Tokenizer] Added an encoding option (default is UTF8 as always).  Needed this to encode batch files in ASCII.  Also added the TemplatePath parameter so you can update tokens in a specific folder.  Also, updated the lacking documentation.<commit_after>#
# Module 'Dynamite.PowerShell.Toolkit'
# Generated by: GSoft, Team Dynamite.
# Generated on: 10/24/2013
# > GSoft & Dynamite : http://www.gsoft.com
# > Dynamite Github : https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit
# > Documentation : https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit/wiki
#

<#
.SYNOPSIS
	Replaces tokens in all child items with the file extention .template recursively.
	
.DESCRIPTION
	Replaces tokens in all child items with the file extention .template recursively.
	The template item is copied with without the .template extention before the tokens are replaced.
	The tokens are defined in a 'Tokens.Domain.ps1' file.
	Please define tokens as valiables with the prefix 'DSP_'. 
	EX.: $DSP_token1 = "Value 1" will replace [[DSP_token1]] in any .template file
	
    --------------------------------------------------------------------------------------
    Module 'Dynamite.PowerShell.Toolkit'
    by: GSoft, Team Dynamite.
    > GSoft & Dynamite : http://www.gsoft.com
    > Dynamite Github : https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit
    > Documentation : https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit/wiki
    --------------------------------------------------------------------------------------
    
.PARAMETER Path
	The path where all the files are located. By default the value is the current working location.
	
.PARAMETER Domain
	The prefix for the token file 'Tokens.Domain.ps1'. By default the value is the current NetBIOS name.
        
  .LINK
    GSoft, Team Dynamite on Github
    > https://github.com/GSoft-SharePoint
    
    Dynamite PowerShell Toolkit on Github
    > https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit
    
    Documentation
    > https://github.com/GSoft-SharePoint/Dynamite-PowerShell-Toolkit/wiki
    
#>
function Update-DSPTokens {
	Param (
		[Parameter(Mandatory=$false, ValueFromPipeline=$true)]
		[ValidateScript({Test-Path $_})]
		[string]$Path = (Get-Location),
		
		[Parameter(Mandatory=$false)]
		[string]$Domain = [System.Net.Dns]::GetHostName(),
		
		[Parameter(Mandatory=$false)]
		[Microsoft.PowerShell.Commands.FileSystemCmdletProviderEncoding]
		$Encoding = [Microsoft.PowerShell.Commands.FileSystemCmdletProviderEncoding]::UTF8,
		
		[Parameter(Mandatory=$false)]
		[switch]$UseHostName,
		
		[Parameter(Mandatory=$false)]
		[switch]$UseDomain
	)
	
	if ($UseHostName -eq $true) {
		$Domain = [System.Net.Dns]::GetHostName()
	}
	
	if ($UseDomain -eq $true) {
		$Domain = (Get-CurrentDomain)
	}

	$tokenPath = ""
	
	$Path = Resolve-Path $Path
	$currentLocation = Get-Location
	Get-ChildItem -Path $currentLocation -Include "Tokens.$Domain.ps1" -Recurse | foreach {
		$tokenPath = $_.FullName
	}
	
	if (Test-Path $tokenPath) {
		Write-Host "Found token file at : $tokenPath"
		Execute-TokenFile $Path $tokenPath $Encoding
	}
	else {
		Write-Host "Didn't found the token file named : Tokens.$Domain.ps1"
	}
}

<#
.SYNOPSIS
	Returns the current domain name using wmi. If wmi is not installed, 
	then use the USERDOMAIN environment variable.
#>
function script:Get-CurrentDomain {
	try {
		# Return this version if wmi is installed
		return [string](gwmi Win32_NTDomain).DomainName.Trim()
	} catch {
		# Fall back on this version in case of error
		return $env:USERDOMAIN
	}
}

function script:Execute-TokenFile {
	param (
		$Path,
		$TokenPath,
		$Encoding
	)
	Write-Host "$TokenPath"
	# Load tokens
	. $TokenPath
	$tokens = Get-Variable -Include "DSP_*"
	
	# Replace tokens in all .template files.
	Get-ChildItem -Path $Path -Include "*.template" -Recurse | foreach {
		Write-Host "Replacing tokens in file '$_'... " -NoNewline
		
		try {
			# Get the contents of the template file.
			$contents  = Get-Content $_ -Encoding $Encoding -ErrorAction Stop
			
			# for each token in our token file, we replace the token in the contents of the file.
			$tokens | ForEach {
				$contents = $contents -replace "\[\[$($_.Name)\]\]", $_.Value
			}
			
			# Write the contents with the replaces tokens to a new file overiding any current file.
			Set-Content -Encoding $Encoding -Value $contents -path $_.FullName.Substring(0, $_.FullName.IndexOf(".template")) -Force -ErrorAction Stop
		} catch {
			Write-Host "Failed - $_" -ForegroundColor Red
		}
		
		Write-Host "Success!" -ForegroundColor Green
	}
}
{'fraction_non_alphanumeric': 0.14885540089727367, 'fraction_numerical': 0.004601403428045554, 'mean_word_length': 3.0287303058387396, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 25, 'https://': 14, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7244203', 'n_tokens_mistral': 2883, 'n_tokens_neox': 2667, 'n_words': 771}