<commit_before><?xml version="1.0" encoding="utf-8" ?>
<Project ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <Target Name="FixMe" BeforeTargets="BeforeBuild">
    <ItemGroup Condition=" '@(FixMeTokens)' == '' ">
      <FixMeTokens Include="BUG;FIXME;HACK;UNDONE;NOTE;OPTIMIZE;TODO;WORKAROUND;XXX;UnresolvedMergeConflict" />
    </ItemGroup>
    <ItemGroup>
      <FixMeItemNames Include="@(AvailableItemName)" />
      <FixMeItemNames Include="ApplicationDefinition;AdditionalFiles;Compile;Content;DesignData;DesignDataWithDesignTimeCreatableTypes;EmbeddedResource;EntityDeploy;None;Page;Resource;SplashScreen" />
    </ItemGroup>
    <ItemGroup>
      <FixMeFiles Include="@(%(FixMeItemNames.Identity))" Exclude="$(FixMeExclude)" />
    </ItemGroup>
    <TokenSearch Files="@(FixMeFiles)" Tokens="@(FixMeTokens)" />
  </Target>
  <UsingTask TaskName="TokenSearch" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v12.0.dll">
    <ParameterGroup>
      <Files ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
      <Tokens ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
    </ParameterGroup>
    <Task>
      <Using Namespace="System.Linq" />
      <Using Namespace="System.Text.RegularExpressions" />
      <Code Type="Fragment" Language="cs">
<![CDATA[
var regexBuilder = new StringBuilder();
foreach (var token in Tokens)
{
    if (regexBuilder.Length > 0)
    {
        regexBuilder.Append("|");
    }

    regexBuilder.Append(Regex.Escape(token.ItemSpec));
}

var tokensRegex = new Regex(regexBuilder.ToString(), RegexOptions.Compiled);
var singleLineCommentRegex = new Regex(@"(?://|--|#|'|REM|::)\s*$", RegexOptions.Compiled);
var multiLineCommentStartRegex = new Regex(@"(/\*+|<#|<!--+|@\*+)\s*$", RegexOptions.Compiled);
var stringStartRegex = new Regex(@"(@?"")\s*$", RegexOptions.Compiled);
var extractCharsVerbatimRegex = new Regex(@"((?<char>[^""])|""(?<char>""))*", RegexOptions.Compiled);
var extractCharsEscapedRegex = new Regex(@"((?<char>[^\\""])|\\(?<char>[""'])|(?<char>\\.))*", RegexOptions.Compiled);

foreach (var file in Files)
{
    var fileName = file.ItemSpec;

    if (!File.Exists(fileName))
    {
        Log.LogMessage(MessageImportance.Normal, "Skipping '{0}' because it was not found.", fileName);<commit_msg>Slight improvement to the regexes.<commit_after><?xml version="1.0" encoding="utf-8" ?>
<Project ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
  <Target Name="FixMe" BeforeTargets="BeforeBuild">
    <ItemGroup Condition=" '@(FixMeTokens)' == '' ">
      <FixMeTokens Include="BUG;FIXME;HACK;UNDONE;NOTE;OPTIMIZE;TODO;WORKAROUND;XXX;UnresolvedMergeConflict" />
    </ItemGroup>
    <ItemGroup>
      <FixMeItemNames Include="@(AvailableItemName)" />
      <FixMeItemNames Include="ApplicationDefinition;AdditionalFiles;Compile;Content;DesignData;DesignDataWithDesignTimeCreatableTypes;EmbeddedResource;EntityDeploy;None;Page;Resource;SplashScreen" />
    </ItemGroup>
    <ItemGroup>
      <FixMeFiles Include="@(%(FixMeItemNames.Identity))" Exclude="$(FixMeExclude)" />
    </ItemGroup>
    <TokenSearch Files="@(FixMeFiles)" Tokens="@(FixMeTokens)" />
  </Target>
  <UsingTask TaskName="TokenSearch" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v12.0.dll">
    <ParameterGroup>
      <Files ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
      <Tokens ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
    </ParameterGroup>
    <Task>
      <Using Namespace="System.Linq" />
      <Using Namespace="System.Text.RegularExpressions" />
      <Code Type="Fragment" Language="cs">
<![CDATA[
var regexBuilder = new StringBuilder();
foreach (var token in Tokens)
{
    if (regexBuilder.Length > 0)
    {
        regexBuilder.Append("|");
    }

    regexBuilder.Append(Regex.Escape(token.ItemSpec));
}

var tokensRegex = new Regex(regexBuilder.ToString(), RegexOptions.Compiled);
var singleLineCommentRegex = new Regex(@"(?://+|--+|#|'|REM|::)\s*$", RegexOptions.Compiled);
var multiLineCommentStartRegex = new Regex(@"(/\*+|<#|<!--+|@\*+)\s*$", RegexOptions.Compiled);
var stringStartRegex = new Regex(@"(@?"")\s*$", RegexOptions.Compiled);
var extractCharsVerbatimRegex = new Regex(@"((?<char>[^""])|""(?<char>""))*", RegexOptions.Compiled);
var extractCharsEscapedRegex = new Regex(@"((?<char>[^\\""])|\\(?<char>[""'])|(?<char>\\.))*", RegexOptions.Compiled);

foreach (var file in Files)
{
    var fileName = file.ItemSpec;

    if (!File.Exists(fileName))
    {
        Log.LogMessage(MessageImportance.Normal, "Skipping '{0}' because it was not found.", fileName);
{'fraction_non_alphanumeric': 0.18940508723985705, 'fraction_numerical': 0.006306495690561278, 'mean_word_length': 5.4646739130434785, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 2, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1744585', 'n_tokens_mistral': 1603, 'n_tokens_neox': 1494, 'n_words': 231}