<commit_before><?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="GenerateHeaders">

  <Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>

  <!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
  <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
  </Target>

  <PropertyGroup>
    <DestDir>$(PyBuildDir)genhdr\</DestDir>
    <PySrcDir>$(PyBaseDir)py\</PySrcDir>
    <QstrDefs>$(PyBaseDir)ports\unix\qstrdefsport.h</QstrDefs>
    <PyQstrDefs>$(PySrcDir)qstrdefs.h</PyQstrDefs>
    <QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected>
    <QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen>
    <PyPython Condition="'$(PyPython)' == ''">python</PyPython>
    <CLToolExe Condition="'$(CLToolExe)' == ''">cl.exe</CLToolExe>
    <PyClTool>$([System.IO.Path]::Combine(`$(CLToolPath)`, `$(CLToolExe)`))</PyClTool>
  </PropertyGroup>

  <Target Name="MakeDestDir">
    <MakeDir Directories="$(DestDir)"/>
  </Target>

  <!-- Concatenate preprocessed files for use with makeqstrdefs.py.
       Filters out any lines which aren't used by makeqstrdefs.py so the resulting file is
       hundreds of times smaller and processing in python takes substantially less time-->
  <UsingTask TaskName="ConcatPreProcFiles" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll" >
    <ParameterGroup>
      <OutputFile Required="true" ParameterType="System.String"/>
      <InputFiles Required="true" ParameterType="System.String[]"/>
    </ParameterGroup>
    <Task>
      <Code Type="Fragment" Language="cs">
        <![CDATA[
using(var outFile = System.IO.File.CreateText(OutputFile)) {
  foreach(var inFile in InputFiles)
    foreach(var line in System.IO.File.ReadAllLines(inFile))
      if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR"))
        outFile.WriteLine( line );
}
        ]]>
      </Code>
    </Task>
  </UsingTask>

  <!-- If any of these changed we'll force all qstrs to be generated again-->
  <ItemGroup>
    <QstrDependencies Include="$(PyWinDir)mpconfigport.h;$(PySrcDir)mpconfig.h;$(PySrcDir)makeqstrdata.py"/>
  </ItemGroup>

  <!-- Preprocess changed files, concatenate and feed into makeqstrdefs.py split/cat-->
  <Target Name="MakeQstrDefs" DependsOnTargets="MakeDestDir" Inputs="@(ClCompile);@(QstrDependencies)" Outputs="$(QstrDefsCollected)">
    <ItemGroup>
      <PyIncDirs Include="$(PyIncDirs)"/>
      <PreProcDefs Include="%(ClCompile.PreProcessorDefinitions);NO_QSTR"/>
      <PyQstrSourceFiles Include="@(ClCompile)">
        <OutFile>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)qstr\'))</OutFile>
      </PyQstrSourceFiles>
      <PyQstrSourceFiles>
        <OutFile>$([System.IO.Path]::ChangeExtension('%(OutFile)', '.pp'))</OutFile>
        <OutDir>$([System.IO.Path]::GetDirectoryName('%(OutFile)'))</OutDir>
      </PyQstrSourceFiles>
      <PyQstrSourceFiles>
        <Changed Condition="$([System.DateTime]::Compare($([System.IO.File]::GetLastWriteTime('%(FullPath)')), $([System.IO.File]::GetLastWriteTime('%(OutFile)')))) &gt; 0">True</Changed>
      </PyQstrSourceFiles>
      <QstrDependencies>
        <Changed Condition="$([System.DateTime]::Compare($([System.IO.File]::GetLastWriteTime('%(FullPath)')), $([System.IO.File]::GetLastWriteTime('$(DestDir)qstr.i.last')))) &gt; 0">True</Changed>
      </QstrDependencies>
    </ItemGroup>
    <PropertyGroup>
      <ForceQstrRebuild>@(QstrDependencies->AnyHaveMetadataValue('Changed', 'True'))</ForceQstrRebuild>
      <RunPreProcConcat>@(PyQstrSourceFiles->AnyHaveMetadataValue('Changed', 'True'))</RunPreProcConcat>
    </PropertyGroup>

    <MakeDir Directories="@(PyQstrSourceFiles->'%(OutDir)')"/>
    <Exec Command="$(PyClTool) /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /Fi%(PyQstrSourceFiles.OutFile) /P %(PyQstrSourceFiles.Identity)"
          Condition="'%(PyQstrSourceFiles.Changed)' == 'True' Or '$(ForceQstrRebuild)' == 'True'"/>
    <ConcatPreProcFiles InputFiles="@(PyQstrSourceFiles->'%(OutFile)')" OutputFile="$(DestDir)qstr.i.last"
                        Condition="'$(RunPreProcConcat)' == 'True' Or '$(ForceQstrRebuild)' == 'True'"/>
    <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py split $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
    <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat $(DestDir)qstr.i.last $(DestDir)qstr $(QstrDefsCollected)"/>
  </Target>

  <Target Name="MakeModuleDefs" DependsOnTargets="MakeDestDir">
    <PropertyGroup>
      <DestFile>$(DestDir)moduledefs.h</DestFile>
      <TmpFile>$(DestFile).tmp</TmpFile>
    </PropertyGroup>
    <ItemGroup>
      <PyUserModuleFiles Include="@(ClCompile)">
        <Path>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', ''))</Path>
      </PyUserModuleFiles>
    </ItemGroup>
    <Exec Command="$(PyPython) $(PySrcDir)makemoduledefs.py --vpath=&quot;., $(PyBaseDir), $(PyUserCModules)&quot; @(PyUserModuleFiles->'%(Path)', ' ') > $(TmpFile)"/>
    <MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
  </Target>
<commit_msg>windows: Update genhdr.targets to match makeqstrdefs.py args.<commit_after><?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" DefaultTargets="GenerateHeaders">

  <Import Project="paths.props" Condition="'$(PyPathsIncluded)' != 'True'"/>

  <!--Generate qstrdefs.generated.h and mpversion.h similar to what is done in py/mkrules.mk and py/py.mk-->
  <Target Name="GenerateHeaders" DependsOnTargets="MakeVersionHdr;MakeModuleDefs;MakeQstrData">
  </Target>

  <PropertyGroup>
    <DestDir>$(PyBuildDir)genhdr\</DestDir>
    <PySrcDir>$(PyBaseDir)py\</PySrcDir>
    <QstrDefs>$(PyBaseDir)ports\unix\qstrdefsport.h</QstrDefs>
    <PyQstrDefs>$(PySrcDir)qstrdefs.h</PyQstrDefs>
    <QstrDefsCollected>$(DestDir)qstrdefscollected.h</QstrDefsCollected>
    <QstrGen>$(DestDir)qstrdefs.generated.h</QstrGen>
    <PyPython Condition="'$(PyPython)' == ''">python</PyPython>
    <CLToolExe Condition="'$(CLToolExe)' == ''">cl.exe</CLToolExe>
    <PyClTool>$([System.IO.Path]::Combine(`$(CLToolPath)`, `$(CLToolExe)`))</PyClTool>
  </PropertyGroup>

  <Target Name="MakeDestDir">
    <MakeDir Directories="$(DestDir)"/>
  </Target>

  <!-- Concatenate preprocessed files for use with makeqstrdefs.py.
       Filters out any lines which aren't used by makeqstrdefs.py so the resulting file is
       hundreds of times smaller and processing in python takes substantially less time-->
  <UsingTask TaskName="ConcatPreProcFiles" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll" >
    <ParameterGroup>
      <OutputFile Required="true" ParameterType="System.String"/>
      <InputFiles Required="true" ParameterType="System.String[]"/>
    </ParameterGroup>
    <Task>
      <Code Type="Fragment" Language="cs">
        <![CDATA[
using(var outFile = System.IO.File.CreateText(OutputFile)) {
  foreach(var inFile in InputFiles)
    foreach(var line in System.IO.File.ReadAllLines(inFile))
      if((line.Contains(".c") && line.StartsWith("#line")) || line.Contains("MP_QSTR"))
        outFile.WriteLine( line );
}
        ]]>
      </Code>
    </Task>
  </UsingTask>

  <!-- If any of these changed we'll force all qstrs to be generated again-->
  <ItemGroup>
    <QstrDependencies Include="$(PyWinDir)mpconfigport.h;$(PySrcDir)mpconfig.h;$(PySrcDir)makeqstrdata.py"/>
  </ItemGroup>

  <!-- Preprocess changed files, concatenate and feed into makeqstrdefs.py split/cat-->
  <Target Name="MakeQstrDefs" DependsOnTargets="MakeDestDir" Inputs="@(ClCompile);@(QstrDependencies)" Outputs="$(QstrDefsCollected)">
    <ItemGroup>
      <PyIncDirs Include="$(PyIncDirs)"/>
      <PreProcDefs Include="%(ClCompile.PreProcessorDefinitions);NO_QSTR"/>
      <PyQstrSourceFiles Include="@(ClCompile)">
        <OutFile>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', '$(DestDir)qstr\'))</OutFile>
      </PyQstrSourceFiles>
      <PyQstrSourceFiles>
        <OutFile>$([System.IO.Path]::ChangeExtension('%(OutFile)', '.pp'))</OutFile>
        <OutDir>$([System.IO.Path]::GetDirectoryName('%(OutFile)'))</OutDir>
      </PyQstrSourceFiles>
      <PyQstrSourceFiles>
        <Changed Condition="$([System.DateTime]::Compare($([System.IO.File]::GetLastWriteTime('%(FullPath)')), $([System.IO.File]::GetLastWriteTime('%(OutFile)')))) &gt; 0">True</Changed>
      </PyQstrSourceFiles>
      <QstrDependencies>
        <Changed Condition="$([System.DateTime]::Compare($([System.IO.File]::GetLastWriteTime('%(FullPath)')), $([System.IO.File]::GetLastWriteTime('$(DestDir)qstr.i.last')))) &gt; 0">True</Changed>
      </QstrDependencies>
    </ItemGroup>
    <PropertyGroup>
      <ForceQstrRebuild>@(QstrDependencies->AnyHaveMetadataValue('Changed', 'True'))</ForceQstrRebuild>
      <RunPreProcConcat>@(PyQstrSourceFiles->AnyHaveMetadataValue('Changed', 'True'))</RunPreProcConcat>
    </PropertyGroup>

    <MakeDir Directories="@(PyQstrSourceFiles->'%(OutDir)')"/>
    <Exec Command="$(PyClTool) /nologo /I@(PyIncDirs, ' /I') /D@(PreProcDefs, ' /D') /Fi%(PyQstrSourceFiles.OutFile) /P %(PyQstrSourceFiles.Identity)"
          Condition="'%(PyQstrSourceFiles.Changed)' == 'True' Or '$(ForceQstrRebuild)' == 'True'"/>
    <ConcatPreProcFiles InputFiles="@(PyQstrSourceFiles->'%(OutFile)')" OutputFile="$(DestDir)qstr.i.last"
                        Condition="'$(RunPreProcConcat)' == 'True' Or '$(ForceQstrRebuild)' == 'True'"/>
    <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py split qstr $(DestDir)qstr.i.last $(DestDir)qstr _"/>
    <Exec Command="$(PyPython) $(PySrcDir)makeqstrdefs.py cat qstr _ $(DestDir)qstr $(QstrDefsCollected)"/>
  </Target>

  <Target Name="MakeModuleDefs" DependsOnTargets="MakeDestDir">
    <PropertyGroup>
      <DestFile>$(DestDir)moduledefs.h</DestFile>
      <TmpFile>$(DestFile).tmp</TmpFile>
    </PropertyGroup>
    <ItemGroup>
      <PyUserModuleFiles Include="@(ClCompile)">
        <Path>$([System.String]::new('%(FullPath)').Replace('$(PyBaseDir)', ''))</Path>
      </PyUserModuleFiles>
    </ItemGroup>
    <Exec Command="$(PyPython) $(PySrcDir)makemoduledefs.py --vpath=&quot;., $(PyBaseDir), $(PyUserCModules)&quot; @(PyUserModuleFiles->'%(Path)', ' ') > $(TmpFile)"/>
    <MSBuild Projects="$(MSBuildThisFileFullPath)" Targets="CopyFileIfDifferent" Properties="SourceFile=$(TmpFile);DestFile=$(DestFile)"/>
  </Target>

{'fraction_non_alphanumeric': 0.18981093415292913, 'fraction_numerical': 0.0024215330166713236, 'mean_word_length': 5.128995433789954, 'pattern_counts': {'":': 0, '<': 197, '<?xml version=': 2, '>': 209, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4990888', 'n_tokens_mistral': 3784, 'n_tokens_neox': 3552, 'n_words': 510}