Q:

How do I use VBA to automate navigation through a webpage?

I have successfully been able to navigate to a webpage and get logged in but I am getting hung up on identifying the proper elements to follow clickable links. I ultimately would like to navigate to a specific location on a website and use excel to automate data entry from an excel file into an input box on the webpage. 
I initially just tried opening the page needed directly but am prompted with user login which ultimately opens to the default home page. I have tried to identify the HTML page elements to follow the clickable path but I'm not sure how specific I need to be or how far up the hierarchy I need to define. I have tried the following: 
This is the code that I have so far: 
Sub Test()

Dim IE As Object
Dim doc As HTMLDocument

Set IE = CreateObject("InternetExplorer.Application")

IE.Visible = True
IE.Navigate "https://iuctsi-oncore-prod.forteresearchapps.com/smrs/AdhocReportControlServlet"

Do While IE.Busy Or IE.ReadyState <> 4
   DoEvents
Loop

Set doc = IE.Document

doc.getElementById("username").Value = "Username"
doc.getElementById("submitbtn").Click
doc.getElementById("password").Value = "Password"
doc.getElementById("submitbtn").Click

Set ul = IE.Document.getElementsByTagName("ul")
   If li.class = "ui-sortable-handle" And role = "presentation" Then
        Set link = IE.Document.getElementsByTagName("a")
          For Each li In link
             If li.className = "top-level-menu-link" And li.href = "https://iuctsi-oncore-prod.forteresearchapps.com/smrs/AdhocReportControlServlet?hdn_function=SPECIMEN_SEARCH&amp;hdn_function_type=REPORT_SEARCH" Then
            link.Click
            Exit For
            End If
   Next
      End If

End Sub

I thought it would suffice to just define the classname "top-level-menu-link" and the href but there are also ul id="nav", li class ="nav iu-sortable" and role ="Presentation" elements above. I'm obviously inexperienced with this stuff but am trying to learn to save my sanity from nonstop monotonous data entry.

<a class="brand" href="/smrs/SMRSHomePageServlet?hdn_function=WELCOME">
  <img src="/smrs/images/menu/menu_logo.png?v=20190523144203">
</a>
<script type="application/json" id="app-menu-items">
  { & #34;data&# 34;: { & #34;key&# 34;: & #34;ROOT&# 34;,
      &
      #34;label&# 34;: & #34;Menu&# 34;,
      &
      #34;url&# 34;: null,
      &
      #34;active&# 34;: false,
      &
      #34;external&# 34;: false
    }, & #34;children&# 34;: [{ & #34;data&# 34;: { & #34;key&# 34;: & #34;MY_CONSOLE_ITEM&# 34;,
          &
          #34;label&# 34;: & #34;My Console&# 34;,
          &
          #34;url&# 34;: & #34;/smrs/MyConsoleControlServlet&# 34;,
          &
          #34;active&# 34;: false,
          &
          #34;external&# 34;: false
        },
        &
        #34;children&# 34;: []
      }, { & #34;data&# 34;: { & #34;key&# 34;: & #34;PROTOCOLS_GROUP&# 34;,
          &
          #34;label&# 34;: & #34;Protocols&# 34;,
          &
          #34;url&# 34;: null,
          &
          #34;active&# 34;: false,
          &
          #34;external&# 34;: false
        },
        &
        #34;children&# 34;: [{ & #34;data&# 34;: { & #34;key&# 34;: & #34;DOCUMENT_SEARCH_ITEM&# 34;,
                &
                #34;label&# 34;: & #34;Document Search&# 34;,
                &
                #34;url&# 34;: & #34;/smrs/SMRSControlServlet?hdn_function= DOCUMENT_SEARCH & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;PC_CONSOLE_ITEM & #34;,&# 34;label & #34;:&# 34;PC Console & #34;,&# 34;url & #34;:&# 34;
                /smrs/SMRSControlServlet ? hdn_function = PROTOCOL_INQUIRY & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;PROTOCOL_SEARCH_ITEM & #34;,&# 34;label & #34;:&# 34;Protocol Search & #34;,&# 34;url & #34;:&# 34;
                /smrs/SMRSControlServlet ? hdn_function = PROTOCOL_SEARCH & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;PROTOCOL_SEARCH_LAST_RESULTS & #34;,&# 34;label & #34;:&# 34;Protocol Search - Last Results & #34;,&# 34;url & #34;:&# 34;
                /smrs/PRBrowseServlet ? hdn_function = PROTOCOL_SEARCH_LAST_RESULTS & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]}]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;REPORTS_GROUP & #34;,&# 34;label & #34;:&# 34;Reports & #34;,&# 34;url & #34;:null,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[{&# 34;data & #34;:{&# 34;key & #34;:&# 34;REPORTS_ITEM & #34;,&# 34;label & #34;:&# 34;Reports & #34;,&# 34;url & #34;:&# 34;
                /smrs/SMRSControlServlet ? hdn_function = PL_REPORTS_MAINTAIN & amp;hdn_PRIVILEGE_ID,
                FUNCTION_GROUP_type = REPORTS & amp;on_success = /plinks/jsp / pl_report_browse.jsp & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]}]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SPECIMENS_GROUP & #34;,&# 34;label & #34;:&# 34;Specimens & #34;,&# 34;url & #34;:null,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[{&# 34;data & #34;:{&# 34;key & #34;:&# 34;BSM_DASHBOARD_ITEM & #34;,&# 34;label & #34;:&# 34;BSM Dashboard & #34;,&# 34;url & #34;:&# 34;
                /smrs/BSMSReportsControlServlet ? hdn_function = BSM_DASHBOARD & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SPECIMEN_BULK_MOVE_ITEM & #34;,&# 34;label & #34;:&# 34;Specimen Bulk Move & #34;,&# 34;url & #34;:&# 34;
                /smrs/BSMSSpecimenCollectionControlServlet ? hdn_function = BULK_MOVE & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SPECIMEN_COLLECTION_CONSOLE_ITEM & #34;,&# 34;label & #34;:&# 34;Specimen Collection Console & #34;,&# 34;url & #34;:&# 34;
                /smrs/BSMSSpecimenCollectionControlServlet ? hdn_function = SPECIMEN_COLLECTION & amp;hdn_function_type = INQUIRY & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SPECIMEN_CUSTOM_REPORT_ITEM & #34;,&# 34;label & #34;:&# 34;Specimen Custom Report & #34;,&# 34;url & #34;:&# 34;
                /smrs/AdhocReportControlServlet ? hdn_function = SPECIMEN_SEARCH & amp;hdn_function_type = REPORT_SEARCH & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SPECIMEN_DATA_ENTRY_ITEM & #34;,&# 34;label & #34;:&# 34;Specimen Data Entry & #34;,&# 34;url & #34;:&# 34;
                /smrs/BSMSpecimenEntryControlServlet ? hdn_function = SPECIMEN_COLLECTION & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SPECIMEN_IMPORT_ITEM & #34;,&# 34;label & #34;:&# 34;Specimen Import & #34;,&# 34;url & #34;:&# 34;
                /smrs/BSMSpecimenUploadServlet ? hdn_function = SPECIMEN_IMPORT & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SPECIMEN_REQUEST_CONSOLE_ITEM & #34;,&# 34;label & #34;:&# 34;Specimen Request Console & #34;,&# 34;url & #34;:&# 34;
                /smrs/BSMSpecimenRequestControlServlet ? hdn_function = REQUEST_LIST & amp;default_list = Y & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;STORAGE_BROWSE_ITEM & #34;,&# 34;label & #34;:&# 34;Storage Browse & #34;,&# 34;url & #34;:&# 34;
                /smrs/bsmStorage / browse /
                default.do & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;WORKING_LISTS_ITEM & #34;,&# 34;label & #34;:&# 34;Working Lists & #34;,&# 34;url & #34;:&# 34;
                /smrs/BSMSSpecimenCollectionControlServlet ? hdn_function = WORKING_LISTS & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]}]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SUBJECTS_GROUP & #34;,&# 34;label & #34;:&# 34;Subjects & #34;,&# 34;url & #34;:null,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[{&# 34;data & #34;:{&# 34;key & #34;:&# 34;CRA_CONSOLE_ITEM & #34;,&# 34;label & #34;:&# 34;CRA Console & #34;,&# 34;url & #34;:&# 34;
                /smrs/SMRSControlServlet ? hdn_function = CRA_CONSOLE & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SUBJECT_CONSOLE_ITEM & #34;,&# 34;label & #34;:&# 34;Subject Console & #34;,&# 34;url & #34;:&# 34;
                /smrs/SMRSControlServlet ? hdn_function = SUBJECT_CONSOLE & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SUBJECT_SEARCH_ITEM & #34;,&# 34;label & #34;:&# 34;Subject Search & #34;,&# 34;url & #34;:&# 34;
                /smrs/SMRSControlServlet ? hdn_function = SUBJECT_INQUIRY & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]},{&# 34;data & #34;:{&# 34;key & #34;:&# 34;SUBJECT_SEARCH_LAST_RESULTS & #34;,&# 34;label & #34;:&# 34;Subject Search - Last Results & #34;,&# 34;url & #34;:&# 34;
                /smrs/SubjectBrowseServlet ? hdn_function = SUBJECT_SEARCH_LAST_RESULTS & #34;,&# 34;active & #34;:false,&# 34;external & #34;:false},&# 34;children & #34;:[]}]}]}
</script>
<script type="application/json" id="user-favorite-keys">
  [ & #34;BSM_DASHBOARD_ITEM&# 34;, & #34;WORKING_LISTS_ITEM&# 34;, & #34;STORAGE_BROWSE_ITEM&# 34;, & #34;SPECIMEN_CUSTOM_REPORT_ITEM&# 34;, & #34;SPECIMEN_COLLECTION_CONSOLE_ITEM&# 34;, & #34;SPECIMEN_IMPORT_ITEM&# 34;]
</script>
<ul class="nav" id="nav">
  <li class="dropdown star-menu"><a href="#" class="main-menu-link dropdown-toggle" data-toggle="main-menu">Menu&nbsp;<i class="icon-menu-hamburger"></i></a>
    <ul class="dropdown-menu" role="main-menu">
      <li role="presentation" class="">
        <a href="/smrs/MyConsoleControlServlet">
          <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">My Console</span></a>
      </li>
      <li class="dropdown-submenu">
        <a href="#">
          <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Protocols</span></a>
        <ul class="dropdown-menu" role="main-menu" style="top: -6px; left: 99px;">
          <li role="presentation" class="">
            <a href="/smrs/SMRSControlServlet?hdn_function=DOCUMENT_SEARCH">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Document Search</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/SMRSControlServlet?hdn_function=PROTOCOL_INQUIRY">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">PC Console</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/SMRSControlServlet?hdn_function=PROTOCOL_SEARCH">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Protocol Search</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/PRBrowseServlet?hdn_function=PROTOCOL_SEARCH_LAST_RESULTS">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Protocol Search - Last Results</span></a>
          </li>
        </ul>
      </li>
      <li class="dropdown-submenu">
        <a href="#">
          <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Reports</span></a>
        <ul class="dropdown-menu" role="main-menu" style="top: -6px; left: 99px;">
          <li role="presentation" class="">
            <a href="/smrs/SMRSControlServlet?hdn_function=PL_REPORTS_MAINTAIN&amp;hdn_PRIVILEGE_ID,

FUNCTION_GROUP_type=REPORTS&amp;on_success=/plinks/jsp/pl_report_browse.jsp">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Reports</span></a>
          </li>
        </ul>
      </li>
      <li class="dropdown-submenu">
        <a href="#">
          <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Specimens</span></a>
        <ul class="dropdown-menu" role="main-menu" style="top: -6px; left: 99px;">
          <li role="presentation" class="">
            <a href="/smrs/BSMSReportsControlServlet?hdn_function=BSM_DASHBOARD">
              <div class="menu-icon-wrapper"><i class="icon-star"></i></div><span class="link-text">BSM Dashboard</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/BSMSSpecimenCollectionControlServlet?hdn_function=BULK_MOVE">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Specimen Bulk Move</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/BSMSSpecimenCollectionControlServlet?hdn_function=SPECIMEN_COLLECTION&amp;hdn_function_type=INQUIRY">
              <div class="menu-icon-wrapper"><i class="icon-star"></i></div><span class="link-text">Specimen Collection Console</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/AdhocReportControlServlet?hdn_function=SPECIMEN_SEARCH&amp;hdn_function_type=REPORT_SEARCH">
              <div class="menu-icon-wrapper"><i class="icon-star"></i></div><span class="link-text">Specimen Custom Report</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/BSMSpecimenEntryControlServlet?hdn_function=SPECIMEN_COLLECTION">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Specimen Data Entry</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/BSMSpecimenUploadServlet?hdn_function=SPECIMEN_IMPORT">
              <div class="menu-icon-wrapper"><i class="icon-star"></i></div><span class="link-text">Specimen Import</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/BSMSpecimenRequestControlServlet?hdn_function=REQUEST_LIST&amp;default_list=Y">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Specimen Request Console</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/bsmStorage/browse/default.do">
              <div class="menu-icon-wrapper"><i class="icon-star"></i></div><span class="link-text">Storage Browse</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/BSMSSpecimenCollectionControlServlet?hdn_function=WORKING_LISTS">
              <div class="menu-icon-wrapper"><i class="icon-star"></i></div><span class="link-text">Working Lists</span></a>
          </li>
        </ul>
      </li>
      <li class="dropdown-submenu">
        <a href="#">
          <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Subjects</span></a>
        <ul class="dropdown-menu" role="main-menu" style="top: -6px; left: 99px;">
          <li role="presentation" class="">
            <a href="/smrs/SMRSControlServlet?hdn_function=CRA_CONSOLE">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">CRA Console</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/SMRSControlServlet?hdn_function=SUBJECT_CONSOLE">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Subject Console</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/SMRSControlServlet?hdn_function=SUBJECT_INQUIRY">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Subject Search</span></a>
          </li>
          <li role="presentation" class="">
            <a href="/smrs/SubjectBrowseServlet?hdn_function=SUBJECT_SEARCH_LAST_RESULTS">
              <div class="menu-icon-wrapper"><i class="icon-star-empty"></i></div><span class="link-text">Subject Search - Last Results</span></a>
          </li>
        </ul>
      </li>
    </ul>
  </li>
  <ul class="nav ui-sortable">
    <li role="presentation" class="ui-sortable-handle"><a class="top-level-menu-link" href="/smrs/BSMSReportsControlServlet?hdn_function=BSM_DASHBOARD">BSM Dashboard</a></li>
    <li role="presentation" class="ui-sortable-handle"><a class="top-level-menu-link" href="/smrs/BSMSSpecimenCollectionControlServlet?hdn_function=WORKING_LISTS">Working Lists</a></li>
    <li role="presentation" class="ui-sortable-handle"><a class="top-level-menu-link" href="/smrs/bsmStorage/browse/default.do">Storage Browse</a></li>

``` ***Target destination is this line****        
<li role="presentation" class="ui-sortable-handle"><a class="top-level-menu-link" href="/smrs/AdhocReportControlServlet?hdn_function=SPECIMEN_SEARCH&amp;hdn_function_type=REPORT_SEARCH">Specimen Custom Report</a></li>

```
    <li role="presentation" class="ui-sortable-handle"><a class="top-level-menu-link" href="/smrs/BSMSSpecimenCollectionControlServlet?hdn_function=SPECIMEN_COLLECTION&amp;hdn_function_type=INQUIRY">Specimen Collection Console</a></li>
    <li role="presentation" class="ui-sortable-handle"><a class="top-level-menu-link" href="/smrs/BSMSpecimenUploadServlet?hdn_function=SPECIMEN_IMPORT">Specimen Import</a></li>
  </ul>
</ul>

HTML
I have modified my code with the following changes recommended by the commentors below: 
Sub Test()

Dim IE As Object
Dim doc As Object

Set IE = CreateObject("InternetExplorer.Application")

IE.Visible = True
IE.navigate "https://iuctsi-oncore-prod.forteresearchapps.com/smrs/AdhocReportControlServlet"

Call ReadyWaitingLoop(IE)

Set doc = IE.document

Dim userEle As Object, pwEle As Object, submitBtn As Object, frm As Object
Set frm = doc.getElementsByClassName("login-form")(0)
Set userEle = doc.getElementById("username")
userEle.Value = "username" ' ## MODIFY to use a real username
Set submitBtn = doc.getElementById("submitBtn")
frm.submit  ' or submitBtn.Click -- not sure which is needed, both seem to work as tested
Call ReadyWaitingLoop(IE)

Set pwEle = doc.getElementById("password")
Set submitBtn = doc.getElementById("submitBtn")  '## maybe case-sensitive
pwEle.Value = "password" ' ## MODIFY to use a real password
submitBtn.Click '-- not sure which is needed, both seem to work as tested frm.submit  ' or

    Call ReadyWaitingLoop(IE)

While IE.Busy Or IE.readyState <> 4: DoEvents: Wend
IE.document.querySelector("[href$=REPORT_SEARCH]").Click
While IE.Busy Or IE.readyState <> 4: DoEvents: Wend

While IE.Busy Or IE.readyState <> 4: DoEvents: Wend
IE.document.querySelector("[href$=CustomReportGroup_Mine_IUGBQCChecks").Click '----This reference does not seem to work. The original element is:<a id="groupHead_3" href="javascript:showhide(3,'CustomReportGroup_Mine_IUGBQCChecks' );">[-]</a> 
While IE.Busy Or IE.readyState <> 4: DoEvents: Wend

While IE.Busy Or IE.readyState <> 4: DoEvents: Wend
IE.document.querySelector("[href$=Copy of Hemoglobin QC_CO").Click
While IE.Busy Or IE.readyState <> 4: DoEvents: Wend

' Rest of your code...

End Sub

Sub ReadyWaitingLoop(IE As Object)
Do While IE.Busy Or IE.readyState <> 4
   DoEvents
Loop
End Sub

A:

Try working off IE.document after ensuring proper page load wait after prior click then avoid loop and use css attribute = value selector
doc.getElementById("submitBtn").Click
While ie.Busy Or ie.ReadyState<>4:DoEvents:Wend
ie.document.querySelector("[href$=REPORT_SEARCH]").click
While ie.Busy Or ie.ReadyState<>4:DoEvents:Wend

