<commit_before>// $begin{copyright}
//
// This file is part of WebSharper
//
// Copyright (c) 2008-2013 IntelliFactory
//
// GNU Affero General Public License Usage
// WebSharper is free software: you can redistribute it and/or modify it under
// the terms of the GNU Affero General Public License, version 3, as published
// by the Free Software Foundation.
//
// WebSharper is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
// for more details at <http://www.gnu.org/licenses/>.
//
// If you are unsure which license is appropriate for your use, please contact
// IntelliFactory at http://intellifactory.com/contact.
//
// $end{copyright}

module IntelliFactory.WebSharper.Compiler.FrontEnd

open System
open System.Collections.Generic
open System.IO
open System.Reflection
open System.Text
open System.Web.UI
open Mono.Cecil
open IntelliFactory.Core
open IntelliFactory.WebSharper
module M = IntelliFactory.WebSharper.Core.Metadata
module P = IntelliFactory.JavaScript.Packager
module PC = IntelliFactory.WebSharper.PathConventions
module R = IntelliFactory.WebSharper.Compiler.ReflectionLayer
module Re = IntelliFactory.WebSharper.Core.Reflection
module Res = IntelliFactory.WebSharper.Core.Resources
module TSE = IntelliFactory.WebSharper.TypeScriptExporter
module W = IntelliFactory.JavaScript.Writer
type Path = string
type Pref = IntelliFactory.JavaScript.Preferences

[<Literal>]
let EMBEDDED_METADATA = "WebSharper.dep"

[<Literal>]
let EMBEDDED_JS = "WebSharper.js"

[<Literal>]
let EMBEDDED_MINJS = "WebSharper.min.js"

[<Literal>]
let EMBEDDED_DTS = "WebSharper.d.ts"

let readStream (s: Stream) =
    use m = new MemoryStream()
    s.CopyTo(m)
    m.ToArray()

let readResource name (def: AssemblyDefinition) =
    def.MainModule.Resources
    |> Seq.tryPick (function
        | :? EmbeddedResource as r when r.Name = name ->
            use reader = new StreamReader(r.GetResourceStream())
            reader.ReadToEnd() |> Some
        | _ -> None)

let readResourceBytes name (def: AssemblyDefinition) =
    def.MainModule.Resources
    |> Seq.tryPick (function
        | :? EmbeddedResource as r when r.Name = name ->
            use r = r.GetResourceStream()
            Some (readStream r)
        | _ -> None)

type Symbols =
    | Mdb of byte []
    | Pdb of byte []

type AssemblyContent =
    {
        ACContent : string
        ACFileName : string
    }

    member ac.FileName = ac.ACFileName
    member ac.Content = ac.ACContent

    static member Create(name, content) =
        {
            ACContent = content
            ACFileName = name
        }

let (|StringArg|_|) (attr: CustomAttributeArgument) =
    if attr.Type.FullName = "System.String" then
        Some (attr.Value :?> string)
    else
        None

type EmbeddedFile =
    {
        ResAssembly : string
        mutable ResContent : string
        ResContentBytes : byte []
        ResContentType : string
        ResName : string
    }

    member ri.GetContentData() =
        Array.copy ri.ResContentBytes

    member ri.Content =
        match ri.ResContent with
        | null ->
            try
                let s = UTF8Encoding(false, true).GetString(ri.ResContentBytes)
                ri.ResContent <- s
                s
            with e ->
                failwithf "Encoding problem in resource [%s] in assembly [%s]: %O"
                    ri.ResAssembly ri.ResName e
        | s -> s

    member ri.ContentType = ri.ResContentType
    member ri.FileName = ri.ResName

    member ri.IsScript =
        match ri.ResContentType with
        | "text/javascript" -> true
        | _ -> false

let parseWebResources (def: AssemblyDefinition) =
    def.CustomAttributes
    |> Seq.choose (fun attr ->
        let wra = "System.Web.UI.WebResourceAttribute"
        if attr.AttributeType.FullName = wra then
            match Seq.toList attr.ConstructorArguments with
            | [StringArg resourceName; StringArg contentType] ->
                readResourceBytes resourceName def
                |> Option.map (fun c ->
                    {
                        ResAssembly = string def.FullName
                        ResContent = null
                        ResContentBytes = c
                        ResContentType = contentType
                        ResName = resourceName
                    })
            | _ -> None
        else None)

type Assembly =
    {
        Debug : option<Symbols>
        Definition : AssemblyDefinition
        FullLoadPath : option<string>
    }

    member this.FullName =
        this.Definition.FullName

    member this.GetScripts() =
        parseWebResources this.Definition
        |> Seq.filter (fun r -> r.IsScript)

    member this.GetContents() =
        parseWebResources this.Definition
        |> Seq.filter (fun r -> not r.IsScript)

    member this.OutputParameters(keyPair) =
        let par = WriterParameters()
        match keyPair with
        | Some kp -> par.StrongNameKeyPair <- kp
        | None -> ()
        par

    member this.RawBytes(kP: option<StrongNameKeyPair>) =
        use s = new System.IO.MemoryStream(16 * 1024)<commit_msg>Fix for ignoring non-websharper assembly resources.<commit_after>// $begin{copyright}
//
// This file is part of WebSharper
//
// Copyright (c) 2008-2013 IntelliFactory
//
// GNU Affero General Public License Usage
// WebSharper is free software: you can redistribute it and/or modify it under
// the terms of the GNU Affero General Public License, version 3, as published
// by the Free Software Foundation.
//
// WebSharper is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
// for more details at <http://www.gnu.org/licenses/>.
//
// If you are unsure which license is appropriate for your use, please contact
// IntelliFactory at http://intellifactory.com/contact.
//
// $end{copyright}

module IntelliFactory.WebSharper.Compiler.FrontEnd

open System
open System.Collections.Generic
open System.IO
open System.Reflection
open System.Text
open System.Web.UI
open Mono.Cecil
open IntelliFactory.Core
open IntelliFactory.WebSharper
module M = IntelliFactory.WebSharper.Core.Metadata
module P = IntelliFactory.JavaScript.Packager
module PC = IntelliFactory.WebSharper.PathConventions
module R = IntelliFactory.WebSharper.Compiler.ReflectionLayer
module Re = IntelliFactory.WebSharper.Core.Reflection
module Res = IntelliFactory.WebSharper.Core.Resources
module TSE = IntelliFactory.WebSharper.TypeScriptExporter
module W = IntelliFactory.JavaScript.Writer
type Path = string
type Pref = IntelliFactory.JavaScript.Preferences

[<Literal>]
let EMBEDDED_METADATA = "WebSharper.dep"

[<Literal>]
let EMBEDDED_JS = "WebSharper.js"

[<Literal>]
let EMBEDDED_MINJS = "WebSharper.min.js"

[<Literal>]
let EMBEDDED_DTS = "WebSharper.d.ts"

let readStream (s: Stream) =
    use m = new MemoryStream()
    s.CopyTo(m)
    m.ToArray()

let readResource name (def: AssemblyDefinition) =
    def.MainModule.Resources
    |> Seq.tryPick (function
        | :? EmbeddedResource as r when r.Name = name ->
            use reader = new StreamReader(r.GetResourceStream())
            reader.ReadToEnd() |> Some
        | _ -> None)

let readResourceBytes name (def: AssemblyDefinition) =
    def.MainModule.Resources
    |> Seq.tryPick (function
        | :? EmbeddedResource as r when r.Name = name ->
            use r = r.GetResourceStream()
            Some (readStream r)
        | _ -> None)

type Symbols =
    | Mdb of byte []
    | Pdb of byte []

type AssemblyContent =
    {
        ACContent : string
        ACFileName : string
    }

    member ac.FileName = ac.ACFileName
    member ac.Content = ac.ACContent

    static member Create(name, content) =
        {
            ACContent = content
            ACFileName = name
        }

let (|StringArg|_|) (attr: CustomAttributeArgument) =
    if attr.Type.FullName = "System.String" then
        Some (attr.Value :?> string)
    else
        None

type EmbeddedFile =
    {
        ResAssembly : string
        mutable ResContent : string
        ResContentBytes : byte []
        ResContentType : string
        ResName : string
    }

    member ri.GetContentData() =
        Array.copy ri.ResContentBytes

    member ri.Content =
        match ri.ResContent with
        | null ->
            try
                let s = UTF8Encoding(false, true).GetString(ri.ResContentBytes)
                ri.ResContent <- s
                s
            with e ->
                failwithf "Encoding problem in resource [%s] in assembly [%s]: %O"
                    ri.ResAssembly ri.ResName e
        | s -> s

    member ri.ContentType = ri.ResContentType
    member ri.FileName = ri.ResName

    member ri.IsScript =
        match ri.ResContentType with
        | "text/javascript" -> true
        | _ -> false

let isWebSharperAssembly (a: AssemblyDefinition) =
    let key = EMBEDDED_METADATA
    a.MainModule.Resources
    |> Seq.exists (function
        | :? EmbeddedResource as r when r.Name = key -> true
        | _ -> false)

let parseWebResources (def: AssemblyDefinition) =
    if isWebSharperAssembly def then
        def.CustomAttributes
        |> Seq.choose (fun attr ->
            let wra = "System.Web.UI.WebResourceAttribute"
            if attr.AttributeType.FullName = wra then
                match Seq.toList attr.ConstructorArguments with
                | [StringArg resourceName; StringArg contentType] ->
                    readResourceBytes resourceName def
                    |> Option.map (fun c ->
                        {
                            ResAssembly = string def.FullName
                            ResContent = null
                            ResContentBytes = c
                            ResContentType = contentType
                            ResName = resourceName
                        })
                | _ -> None
            else None)
    else Seq.empty

type Assembly =
    {
        Debug : option<Symbols>
        Definition : AssemblyDefinition
        FullLoadPath : option<string>
    }

    member this.FullName =
        this.Definition.FullName

    member this.GetScripts() =
        parseWebResources this.Definition
        |> Seq.filter (fun r -> r.IsScript)

    member this.GetContents() =
        parseWebResources this.Definition
        |> Seq.filter (fun r -> not r.IsScript)

    member this.OutputParameters(keyPair) =
        let par = WriterParameters()
        match keyPair with
        | Some kp -> par.StrongNameKeyPair <- kp
        | None -> ()
        par

    member this.RawBytes(kP: option<StrongNameKeyPair>) =
        use s = new System.IO.MemoryStream(16 * 1024)
{'fraction_non_alphanumeric': 0.08645243746576593, 'fraction_numerical': 0.002921307285010042, 'mean_word_length': 2.2315634218289087, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3512022', 'n_tokens_mistral': 3142, 'n_tokens_neox': 3025, 'n_words': 866}