<commit_before>namespace Paket

open Paket
open Paket.Domain
open Paket.Logging
open System
open System.Collections.Generic
open System.IO
open System.Text
open System.Text.RegularExpressions
open System.Xml
open Paket.Xml
open Paket.Requirements
open Paket.PackageSources
open System.Globalization

[<RequireQualifiedAccess>]
type BuildAction =
    | Compile | Content | Reference
    | Resource | Page // These two are WPF only - https://msdn.microsoft.com/library/aa970494%28v=vs.100%29.aspx?f=255&MSPPError=-2147217396
    // There are some other build actions - http://stackoverflow.com/questions/145752/what-are-the-various-build-action-settings-in-visual-studio-project-properties/145769#145769

    override this.ToString() = 
        match this with
        | Compile -> "Compile"
        | Content -> "Content"
        | Reference -> "Reference"
        | Resource -> "Resource"
        | Page -> "Page"

    static member PaketFileNodeNames =
        [Compile; Content; Resource; Page] |> List.map (fun x->x.ToString())

/// File item inside of project files.
type FileItem = 
    { BuildAction : BuildAction
      Include : string
      WithPaketSubNode: bool
      CopyToOutputDirectory: CopyToOutputDirectorySettings option
      Link : string option }

/// Project references inside of project files.
type ProjectReference = 
    { Path : string
      RelativePath : string
      Name : string option
      ReferenceOutputAssembly : bool
      GUID : Guid option }

/// Compile items inside of project files.
type CompileItem =
    { SourceFile : string
      DestinationPath : string
      BaseDir : string }

/// Project output type.
[<RequireQualifiedAccess>]
type ProjectOutputType =
| Exe 
| Library

type ProjectLanguage = Unknown | CSharp | FSharp | VisualBasic | WiX | Nemerle | CPP

module LanguageEvaluation =
    let private extractProjectTypeGuids (projectDocument:XmlDocument) =
        projectDocument
        |> getDescendants "PropertyGroup"
        |> List.filter(fun g -> g.Attributes.Count = 0)
        |> List.collect(fun g -> g |> getDescendants "ProjectTypeGuids") 
        |> List.filter(fun pt -> pt.Attributes.Count = 0)
        |> List.collect(fun pt -> pt.InnerText.Split ';' |> List.ofArray)
        |> List.distinct
        |> List.choose(fun guid -> match Guid.TryParse guid with | (true, g) -> Some g | _ -> None)

    let private csharpGuids =
        [
            "{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}" // C#
            "{BF6F8E12-879D-49E7-ADF0-5503146B24B8}" // Dynamics 2012 AX C# in AOT
            "{20D4826A-C6FA-45DB-90F4-C717570B9F32}" // Legacy (2003) Smart Device (C#)
            "{593B0543-81F6-4436-BA1E-4747859CAAE2}" // SharePoint (C#)
            "{4D628B5B-2FBC-4AA6-8C16-197242AEB884}" // Smart Device (C#)
            "{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}" // Windows (C#)
            "{C089C8C0-30E0-4E22-80C0-CE093F111A43}" // Windows Phone 8/8.1 App (C#)
            "{14822709-B5A1-4724-98CA-57A101D1B079}" // Workflow (C#)
        ] |> List.map Guid.Parse |> Set.ofList

    let private vbGuids =
        [
            "{CB4CE8C6-1BDB-4DC7-A4D3-65A1999772F8}" // Legacy (2003) Smart Device (VB.NET)
            "{EC05E597-79D4-47f3-ADA0-324C4F7C7484}" // SharePoint (VB.NET)
            "{68B1623D-7FB9-47D8-8664-7ECEA3297D4F}" // Smart Device (VB.NET)
            "{F184B08F-C81C-45F6-A57F-5ABD9991F28F}" // VB.NET
            "{F184B08F-C81C-45F6-A57F-5ABD9991F28F}" // Windows (VB.NET)
            "{DB03555F-0C8B-43BE-9FF9-57896B3C5E56}" // Windows Phone 8/8.1 App (VB.NET)
            "{D59BE175-2ED0-4C54-BE3D-CDAA9F3214C8}" // Workflow (VB.NET)
        ] |> List.map Guid.Parse |> Set.ofList

    let private fsharpGuids =
        [
            "{F2A71F9B-5D33-465A-A702-920D77279786}" // F#
        ] |> List.map Guid.Parse |> Set.ofList

    let private nemerleGuids =
        [
            "{EDCC3B85-0BAD-11DB-BC1A-00112FDE8B61}" // Nemerle
        ] |> List.map Guid.Parse |> Set.ofList

    let private getGuidLanguage (guid:Guid) = 
        let isCsharp = csharpGuids.Contains(guid)
        let isVb = vbGuids.Contains(guid)
        let isFsharp = fsharpGuids.Contains(guid)
        let isNemerle = nemerleGuids.Contains(guid)

        match (isCsharp, isVb, isFsharp, isNemerle) with
        | (true, false, false, false) -> Some CSharp
        | (false, true, false, false) -> Some VisualBasic
        | (false, false, true, false) -> Some FSharp
        | (false, false, false, true) -> Some Nemerle
        | _ -> None

    let private getLanguageFromExtension = function
        | ".csproj" -> Some CSharp
        | ".vbproj" -> Some VisualBasic
        | ".fsproj" -> Some FSharp
        | ".vcxproj" -> Some CPP
        | ".wixproj" -> Some WiX
        | ".nproj"  -> Some Nemerle
        | _ -> None

    let private getLanguageFromFileName (fileName : string) =
        let ext = fileName |> Path.GetExtension
        getLanguageFromExtension (ext.ToLowerInvariant())

    /// Get the programming language for a project file using the "ProjectTypeGuids"
    let getProjectLanguage (projectDocument:XmlDocument) (fileName: string) = 
        let cons x y = x :: y

        let languageGroups =
            projectDocument
            |> extractProjectTypeGuids
            |> List.map getGuidLanguage
            |> cons (getLanguageFromFileName fileName)
            |> List.choose id
            |> List.groupBy id
            |> List.map fst

        match languageGroups with
        | [language] -> language
        | _ -> Unknown

/// Contains methods to read and manipulate project files.
type ProjectFile = 
    { FileName: string
      OriginalText : string
      Document : XmlDocument
      ProjectNode : XmlNode
      Language : ProjectLanguage }


[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
module ProjectFile =

    let isSupportedFile (fi:FileInfo) =
        fi.Extension = ".csproj" || fi.Extension = ".fsproj" || fi.Extension = ".vbproj" || fi.Extension = ".wixproj" || fi.Extension = ".nproj" || fi.Extension = ".vcxproj"

    let name (projectFile:ProjectFile) = FileInfo(projectFile.FileName).Name

    let nameWithoutExtension (projectFile:ProjectFile) = Path.GetFileNameWithoutExtension (name projectFile)


    let loadFromStream (fullName:string) (stream:Stream) =
        let doc = XmlDocument()
        doc.Load stream

        let manager = XmlNamespaceManager doc.NameTable
        manager.AddNamespace ("ns", Constants.ProjectDefaultNameSpace)
        let projectNode = 
            match doc |> getNode "Project" with
            | Some node -> node
            | _ -> failwithf "unable to find Project node in file %s" fullName
        {   FileName = fullName
            Document = doc
            ProjectNode = projectNode
            OriginalText = Utils.normalizeXml doc
            Language = LanguageEvaluation.getProjectLanguage doc (Path.GetFileName fullName)

        }

    let loadFromFile(fileName:string) =
        let fileInfo = FileInfo (normalizePath fileName)
        use stream = fileInfo.OpenRead()
        loadFromStream fileInfo.FullName stream
<commit_msg>fix test errors<commit_after>namespace Paket

open Paket
open Paket.Domain
open Paket.Logging
open System
open System.Collections.Generic
open System.IO
open System.Text
open System.Text.RegularExpressions
open System.Xml
open Paket.Xml
open Paket.Requirements
open Paket.PackageSources
open System.Globalization

[<RequireQualifiedAccess>]
type BuildAction =
    | Compile | Content | Reference
    | Resource | Page // These two are WPF only - https://msdn.microsoft.com/library/aa970494%28v=vs.100%29.aspx?f=255&MSPPError=-2147217396
    // There are some other build actions - http://stackoverflow.com/questions/145752/what-are-the-various-build-action-settings-in-visual-studio-project-properties/145769#145769

    override this.ToString() = 
        match this with
        | Compile -> "Compile"
        | Content -> "Content"
        | Reference -> "Reference"
        | Resource -> "Resource"
        | Page -> "Page"

    static member PaketFileNodeNames =
        [Compile; Content; Resource; Page] |> List.map (fun x->x.ToString())

/// File item inside of project files.
type FileItem = 
    { BuildAction : BuildAction
      Include : string
      WithPaketSubNode: bool
      CopyToOutputDirectory: CopyToOutputDirectorySettings option
      Link : string option }

/// Project references inside of project files.
type ProjectReference = 
    { Path : string
      RelativePath : string
      Name : string option
      ReferenceOutputAssembly : bool
      GUID : Guid option }

/// Compile items inside of project files.
type CompileItem =
    { SourceFile : string
      DestinationPath : string
      BaseDir : string }

/// Project output type.
[<RequireQualifiedAccess>]
type ProjectOutputType =
| Exe 
| Library

type ProjectLanguage = Unknown | CSharp | FSharp | VisualBasic | WiX | Nemerle | CPP

module LanguageEvaluation =
    let private extractProjectTypeGuids (projectDocument:XmlDocument) =
        projectDocument
        |> getDescendants "PropertyGroup"
        |> List.filter(fun g -> g.Attributes.Count = 0)
        |> List.collect(fun g -> g |> getDescendants "ProjectTypeGuids") 
        |> List.filter(fun pt -> pt.Attributes.Count = 0)
        |> List.collect(fun pt -> pt.InnerText.Split ';' |> List.ofArray)
        |> List.distinct
        |> List.choose(fun guid -> match Guid.TryParse guid with | (true, g) -> Some g | _ -> None)

    let private csharpGuids =
        [
            "{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}" // C#
            "{BF6F8E12-879D-49E7-ADF0-5503146B24B8}" // Dynamics 2012 AX C# in AOT
            "{20D4826A-C6FA-45DB-90F4-C717570B9F32}" // Legacy (2003) Smart Device (C#)
            "{593B0543-81F6-4436-BA1E-4747859CAAE2}" // SharePoint (C#)
            "{4D628B5B-2FBC-4AA6-8C16-197242AEB884}" // Smart Device (C#)
            "{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}" // Windows (C#)
            "{C089C8C0-30E0-4E22-80C0-CE093F111A43}" // Windows Phone 8/8.1 App (C#)
            "{14822709-B5A1-4724-98CA-57A101D1B079}" // Workflow (C#)
        ] |> List.map Guid.Parse |> Set.ofList

    let private vbGuids =
        [
            "{CB4CE8C6-1BDB-4DC7-A4D3-65A1999772F8}" // Legacy (2003) Smart Device (VB.NET)
            "{EC05E597-79D4-47f3-ADA0-324C4F7C7484}" // SharePoint (VB.NET)
            "{68B1623D-7FB9-47D8-8664-7ECEA3297D4F}" // Smart Device (VB.NET)
            "{F184B08F-C81C-45F6-A57F-5ABD9991F28F}" // VB.NET
            "{F184B08F-C81C-45F6-A57F-5ABD9991F28F}" // Windows (VB.NET)
            "{DB03555F-0C8B-43BE-9FF9-57896B3C5E56}" // Windows Phone 8/8.1 App (VB.NET)
            "{D59BE175-2ED0-4C54-BE3D-CDAA9F3214C8}" // Workflow (VB.NET)
        ] |> List.map Guid.Parse |> Set.ofList

    let private fsharpGuids =
        [
            "{F2A71F9B-5D33-465A-A702-920D77279786}" // F#
        ] |> List.map Guid.Parse |> Set.ofList

    let private nemerleGuids =
        [
            "{EDCC3B85-0BAD-11DB-BC1A-00112FDE8B61}" // Nemerle
        ] |> List.map Guid.Parse |> Set.ofList

    let private getGuidLanguage (guid:Guid) = 
        let isCsharp = csharpGuids.Contains(guid)
        let isVb = vbGuids.Contains(guid)
        let isFsharp = fsharpGuids.Contains(guid)
        let isNemerle = nemerleGuids.Contains(guid)

        match (isCsharp, isVb, isFsharp, isNemerle) with
        | (true, false, false, false) -> Some CSharp
        | (false, true, false, false) -> Some VisualBasic
        | (false, false, true, false) -> Some FSharp
        | (false, false, false, true) -> Some Nemerle
        | _ -> None

    let private getLanguageFromExtension = function
        | ".csproj" -> Some CSharp
        | ".vbproj" -> Some VisualBasic
        | ".fsproj" -> Some FSharp
        | ".vcxproj" -> Some CPP
        | ".wixproj" -> Some WiX
        | ".nproj"  -> Some Nemerle
        | _ -> None

    let private getLanguageFromFileName (fileName : string) =
        let ext = fileName |> Path.GetExtension
        getLanguageFromExtension (ext.ToLowerInvariant())

    /// Get the programming language for a project file using the "ProjectTypeGuids"
    let getProjectLanguage (projectDocument:XmlDocument) (fileName: string) = 
        let cons x y = x :: y

        let languageGroups =
            projectDocument
            |> extractProjectTypeGuids
            |> List.map getGuidLanguage
            |> cons (getLanguageFromFileName fileName)
            |> List.choose id
            |> List.groupBy id
            |> List.map fst

        match languageGroups with
        | [language] -> language
        | _ -> Unknown

/// Contains methods to read and manipulate project files.
type ProjectFile = 
    { FileName: string
      OriginalText : string
      Document : XmlDocument
      ProjectNode : XmlNode
      Language : ProjectLanguage }


[<CompilationRepresentation(CompilationRepresentationFlags.ModuleSuffix)>]
module ProjectFile =
    let supportedEndings = [ ".csproj"; ".fsproj"; ".vbproj"; ".wixproj"; ".nproj"; ".vcxproj"]

    let isSupportedFile (fi:FileInfo) =
        supportedEndings
        |> List.exists (fun e -> fi.Extension.Contains e)

    let name (projectFile:ProjectFile) = FileInfo(projectFile.FileName).Name

    let nameWithoutExtension (projectFile:ProjectFile) = Path.GetFileNameWithoutExtension (name projectFile)


    let loadFromStream (fullName:string) (stream:Stream) =
        let doc = XmlDocument()
        doc.Load stream

        let manager = XmlNamespaceManager doc.NameTable
        manager.AddNamespace ("ns", Constants.ProjectDefaultNameSpace)
        let projectNode = 
            match doc |> getNode "Project" with
            | Some node -> node
            | _ -> failwithf "unable to find Project node in file %s" fullName
        {   FileName = fullName
            Document = doc
            ProjectNode = projectNode
            OriginalText = Utils.normalizeXml doc
            Language = LanguageEvaluation.getProjectLanguage doc (Path.GetFileName fullName)

        }

    let loadFromFile(fileName:string) =
        let fileInfo = FileInfo (normalizePath fileName)
        use stream = fileInfo.OpenRead()
        loadFromStream fileInfo.FullName stream

{'fraction_non_alphanumeric': 0.1103646161382882, 'fraction_numerical': 0.0562670585765274, 'mean_word_length': 2.460048426150121, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 121, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4546820', 'n_tokens_mistral': 5045, 'n_tokens_neox': 4501, 'n_words': 1018}