<commit_before>﻿Imports System
Imports System.IO
Imports System.Net
Imports System.Collections.Generic
Imports System.Collections.ObjectModel
Imports OpenQA.Selenium
Imports OpenQA.Selenium.PhantomJS
Imports HtmlAgilityPack
Imports System.Runtime.Serialization.Formatters.Binary

Public Class clsWeb
    Implements IDisposable

    Private _driver As IWebDriver
    Private _options As PhantomJSOptions
    Private _service As PhantomJSDriverService
    Private _cookie As Boolean = False
    'Private UserAgent As String = "Chrome/34.0.1847.131"

    Sub New()

        _service = PhantomJSDriverService. _
                   CreateDefaultService(Environment.CurrentDirectory)
        _service.HideCommandPromptWindow = True

        _options = New PhantomJSOptions
        '_options.AddAdditionalCapability("phantomjs.page.settings.userAgent", UserAgent)

        _driver = New PhantomJSDriver(_service, _options)
        _driver.Manage().Timeouts().ImplicitlyWait(New TimeSpan(0, 0, 30))

    End Sub

    Sub Dispose() Implements IDisposable.Dispose

        _driver.Quit()
        _driver = Nothing
        _options = Nothing
        _service = Nothing

    End Sub

    Private Function InnerElements(ctrl As IWebElement) As Boolean

        Dim ret As Boolean = False
        Dim html As String
        Dim doc As HtmlDocument = Nothing

        html = ctrl.GetAttribute("outerHTML")

        doc = CleanHTML(html)

        If doc.DocumentNode.FirstChild.HasChildNodes Then
            If doc.DocumentNode.FirstChild.FirstChild.Name = "#text" Then
                ret = False
            Else
                ret = True
            End If
        End If

        doc = Nothing

        Return ret

    End Function

    Private Function ProcessElement( _
                                     ByRef item As IWebElement, _
                                     ByRef kvp As KeyValuePair(Of String, String) _
                                   ) As Boolean

        Dim ret As Boolean = False
        Dim check As String

        check = item.GetAttribute("name")
        If check = kvp.Key Then
            Select Case item.GetAttribute("type")
                Case "text", "password"
                    item.SendKeys(kvp.Value)
                Case "checkbox"
                    item.Click()
            End Select
            ret = True
        End If

        Return ret

    End Function

    Private Sub GetInnerElements( _
                                  ByRef ctrl As IWebElement, _
                                  ByRef Fields As List(Of KeyValuePair(Of String, String)) _
                                )

        Dim idx As Integer
        Dim ret As Boolean

        Dim kvp As KeyValuePair(Of String, String)
        Dim ChildElements As ReadOnlyCollection(Of IWebElement)

        ChildElements = ctrl.FindElements(By.XPath("*"))

        For Each item As IWebElement In ChildElements

            If InnerElements(item) Then
                GetInnerElements(item, Fields)
            End If

            For idx = 0 To (Fields.Count - 1)

                kvp = Fields(idx)
                ret = ProcessElement(item, kvp)

                If ret = True Then
                    Exit For
                End If

            Next

        Next

    End Sub

    Public Sub FollowLink(ByVal URL As String, ByVal LinkText As String, cookieName As String)

        Dim cookies As ReadOnlyCollection(Of OpenQA.Selenium.Cookie)
        Dim cookies2 As New List(Of OpenQA.Selenium.Cookie)
        Dim link As IWebElement
        Dim ret As String

        _driver.Navigate.GoToUrl(URL)

        link = _driver.FindElement(By.LinkText(LinkText))

        link.Click()

        ret = _driver.PageSource

        cookies = _driver.Manage.Cookies.AllCookies

        For idx = 0 To cookies.Count - 1
            cookies2.Add(cookies(idx))
        Next

        WriteCookiesToDisk(cookieName, cookies2)

    End Sub

    Public Sub LogIn(ByVal URL As String, formName As String, Fields As List(Of KeyValuePair(Of String, String)), cookieName As String)

        Dim cookies As ReadOnlyCollection(Of OpenQA.Selenium.Cookie)
        Dim cookies2 As New List(Of OpenQA.Selenium.Cookie)

        Dim idx As Integer
        Dim check As String
        Dim ret As Boolean
        Dim kvp As KeyValuePair(Of String, String)
        Dim formElement As IWebElement
        Dim allFormChildElements As ReadOnlyCollection(Of IWebElement)

        Dim btn As IWebElement = Nothing

        _driver.Navigate.GoToUrl(URL)

        formElement = _driver.FindElement(By.Id(formName))
        allFormChildElements = formElement.FindElements(By.XPath("*"))

        For Each item As IWebElement In allFormChildElements

            If InnerElements(item) Then
                GetInnerElements(item, Fields)
            End If

            For idx = 0 To (Fields.Count - 1)

                kvp = Fields(idx)

                ret = ProcessElement(item, kvp)
<commit_msg>Return first form on page for Login Function.<commit_after>﻿Imports System
Imports System.IO
Imports System.Net
Imports System.Collections.Generic
Imports System.Collections.ObjectModel
Imports OpenQA.Selenium
Imports OpenQA.Selenium.PhantomJS
Imports HtmlAgilityPack
Imports System.Runtime.Serialization.Formatters.Binary

Public Class clsWeb
    Implements IDisposable

    Private _driver As IWebDriver
    Private _options As PhantomJSOptions
    Private _service As PhantomJSDriverService
    Private _cookie As Boolean = False
    'Private UserAgent As String = "Chrome/34.0.1847.131"

    Sub New()

        _service = PhantomJSDriverService. _
                   CreateDefaultService(Environment.CurrentDirectory)
        _service.HideCommandPromptWindow = True

        _options = New PhantomJSOptions
        '_options.AddAdditionalCapability("phantomjs.page.settings.userAgent", UserAgent)

        _driver = New PhantomJSDriver(_service, _options)
        _driver.Manage().Timeouts().ImplicitlyWait(New TimeSpan(0, 0, 30))

    End Sub

    Sub Dispose() Implements IDisposable.Dispose

        _driver.Quit()
        _driver = Nothing
        _options = Nothing
        _service = Nothing

    End Sub

    Private Function InnerElements(ctrl As IWebElement) As Boolean

        Dim ret As Boolean = False
        Dim html As String
        Dim doc As HtmlDocument = Nothing

        html = ctrl.GetAttribute("outerHTML")

        doc = CleanHTML(html)

        If doc.DocumentNode.FirstChild.HasChildNodes Then
            If doc.DocumentNode.FirstChild.FirstChild.Name = "#text" Then
                ret = False
            Else
                ret = True
            End If
        End If

        doc = Nothing

        Return ret

    End Function

    Private Function ProcessElement( _
                                     ByRef item As IWebElement, _
                                     ByRef kvp As KeyValuePair(Of String, String) _
                                   ) As Boolean

        Dim ret As Boolean = False
        Dim check As String

        check = item.GetAttribute("name")
        If check = kvp.Key Then
            Select Case item.GetAttribute("type")
                Case "text", "password"
                    item.SendKeys(kvp.Value)
                Case "checkbox"
                    item.Click()
            End Select
            ret = True
        End If

        Return ret

    End Function

    Private Sub GetInnerElements( _
                                  ByRef ctrl As IWebElement, _
                                  ByRef Fields As List(Of KeyValuePair(Of String, String)) _
                                )

        Dim idx As Integer
        Dim ret As Boolean

        Dim kvp As KeyValuePair(Of String, String)
        Dim ChildElements As ReadOnlyCollection(Of IWebElement)

        ChildElements = ctrl.FindElements(By.XPath("*"))

        For Each item As IWebElement In ChildElements

            If InnerElements(item) Then
                GetInnerElements(item, Fields)
            End If

            For idx = 0 To (Fields.Count - 1)

                kvp = Fields(idx)
                ret = ProcessElement(item, kvp)

                If ret = True Then
                    Exit For
                End If

            Next

        Next

    End Sub

    Public Sub FollowLink(ByVal URL As String, ByVal LinkText As String, cookieName As String)

        Dim cookies As ReadOnlyCollection(Of OpenQA.Selenium.Cookie)
        Dim cookies2 As New List(Of OpenQA.Selenium.Cookie)
        Dim link As IWebElement
        Dim ret As String

        _driver.Navigate.GoToUrl(URL)

        link = _driver.FindElement(By.LinkText(LinkText))

        link.Click()

        ret = _driver.PageSource

        cookies = _driver.Manage.Cookies.AllCookies

        For idx = 0 To cookies.Count - 1
            cookies2.Add(cookies(idx))
        Next

        WriteCookiesToDisk(cookieName, cookies2)

    End Sub

    Public Sub LogIn(ByVal URL As String, formName As String, Fields As List(Of KeyValuePair(Of String, String)), cookieName As String)

        Dim cookies As ReadOnlyCollection(Of OpenQA.Selenium.Cookie)
        Dim cookies2 As New List(Of OpenQA.Selenium.Cookie)

        Dim idx As Integer
        Dim check As String
        Dim ret As Boolean
        Dim kvp As KeyValuePair(Of String, String)
        Dim formElement As IWebElement
        Dim forms As ReadOnlyCollection(Of IWebElement)
        Dim allFormChildElements As ReadOnlyCollection(Of IWebElement)

        Dim btn As IWebElement = Nothing

        _driver.Navigate.GoToUrl(URL)

        forms = _driver.FindElements(By.TagName("form"))

        formElement = forms(0)

        allFormChildElements = formElement.FindElements(By.XPath("*"))

        For Each item As IWebElement In allFormChildElements

            If InnerElements(item) Then
                GetInnerElements(item, Fields)
            End If

            For idx = 0 To (Fields.Count - 1)

                kvp = Fields(idx)

                ret = ProcessElement(item, kvp)

{'fraction_non_alphanumeric': 0.05655680573533805, 'fraction_numerical': 0.00487902021308374, 'mean_word_length': 1.8396946564885497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4249562', 'n_tokens_mistral': 2774, 'n_tokens_neox': 2541, 'n_words': 798}