<commit_before>﻿open System.Net
open System.IO
open System.Text.RegularExpressions

let getURLInfo(url : string) =
    let fetchAsync(url : string) =
        async {
            let request = WebRequest.Create(url)
            let! response = request.AsyncGetResponse()
            let stream = response.GetResponseStream()
            let reader = new StreamReader(stream)
            let html = reader.ReadToEndAsync()
            let! htmlTask = Async.AwaitTask(html)
            do printfn "%s --- %d" url htmlTask.Length
        }

    let readHtml(url: string) =
        let request = WebRequest.Create(url)
        let response = request.GetResponse()
        let stream = response.GetResponseStream()
        let reader = new StreamReader(stream)
        let html = reader.ReadToEnd()
        html

    let regexExpression = new System.Text.RegularExpressions.Regex("<a.*href=\"http.*\">")
    let webPages = regexExpression.Matches(readHtml(url))
    let proc = [for url in webPages -> 
                     let value = url.Value
                     //printfn "%A" value
                     fetchAsync(value.Substring(value.IndexOf("f=") + 3 , value.IndexOf("\">") - value.IndexOf("=\"") - 2))
                     //fetchAsync(value)
                     ]    
    Async.Parallel proc |> Async.RunSynchronously |> ignore

getURLInfo("http://hwproj.me/courses/20")<commit_msg>adding to hw8<commit_after>﻿open System.Net
open System.IO
open System.Text.RegularExpressions

let getURLInfo(url : string) =
    let fetchAsync(url : string) =
        async {
            let request = WebRequest.Create(url)
            use! response = request.AsyncGetResponse()
            use stream = response.GetResponseStream()
            use reader = new StreamReader(stream)
            let str = reader.ReadToEnd()
            printfn "%s --- %d" url str.Length
            return str
        }

    let readHtml(url:string) =
        async {
            let! html = fetchAsync url
            return html
            }

    let regexExpression = new Regex("<a.*href=\"http.*\">")
    let html = Async.RunSynchronously <| readHtml url
    let webPages = regexExpression.Matches(html)
    let proc = [for url in webPages -> 
                     let value = url.Value
                     fetchAsync(value.Substring(value.IndexOf("f=") + 3 , value.IndexOf("\">") - value.IndexOf("=\"") - 2))
                     ]    
    Async.Parallel proc |> Async.RunSynchronously |> ignore

getURLInfo "http://hwproj.me/courses/20"
{'fraction_non_alphanumeric': 0.11596505162827642, 'fraction_numerical': 0.003574265289912629, 'mean_word_length': 2.0059665871121717, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6251338', 'n_tokens_mistral': 710, 'n_tokens_neox': 688, 'n_words': 166}