<commit_before>﻿@{
    Layout = "~/Views/_Layout.cshtml";
}
@{ 
    var searchLink = $"{Context.Request.Path.Value.ToLower().Replace("/crawl", "/search")}{Context.Request.QueryString}";
    var collection = (ViewBag.Collection ?? Context.Request.Query["collection"]).ToString();
    var q = (ViewBag.Q ?? Context.Request.Query["q"]).ToString();
    var field = (ViewBag.Field ?? Context.Request.Query["field"]).ToString();
    var job = (string)ViewBag.Job;
} 
    <form action="/crawl/@Context.Request.QueryString" method="post">
        <div class="blog-wrapper">
            <a href="@searchLink">&#8592; Back to result</a>
            <h2>Crawl and index your result</h2>
            @if (ViewBag.JobValidationError != null)
            {
                <div class="validation-error">@ViewBag.JobValidationError</div>
            }
            <p>
                <label><input type="radio" name="job" value="fi" @Html.Raw((job == "fi") ? "checked" : "") /> <strong>Fetch and index all web page content from referenced WET files</strong>.</label>
            </p>
            <p>
                <label><input type="radio" name="job" value="ei" @Html.Raw((job == "ei") ? "checked" : "") /> <strong>Extract web page content from WET files and index</strong> those resources referenced in your result.</label>
            </p>
            @*<p>
            <label><input type="radio" disabled="disabled" name="job" value="wci" @Html.Raw((job == "wci") ? "checked" : "") /> <strong>Web crawl and index</strong> those resources referenced in your result. (Coming soon)</label>
        </p>
        <p>
            <label><input type="radio" disabled="disabled" name="job" value="rwci" @Html.Raw((job == "rwci") ? "checked" : "")/> <strong>Recurringly web crawl and index</strong> those resources referenced in your result. (Coming soon)</label>
        </p>*@
            <p>
                When the crawling job has completed we will have created a searchable document collection..
                We will host this document collection for your convenience.
                You will be able to query it using a web browser or HTTP client and slice, append to or download it.
            </p>
            <p>
                Name your collection carefully.
                You will need to reference your collection by name anytime you wish to interact with it.<commit_msg>improve instructions<commit_after>﻿@{
    Layout = "~/Views/_Layout.cshtml";
}
@{ 
    var searchLink = $"{Context.Request.Path.Value.ToLower().Replace("/crawl", "/search")}{Context.Request.QueryString}";
    var collection = (ViewBag.Collection ?? Context.Request.Query["collection"]).ToString();
    var q = (ViewBag.Q ?? Context.Request.Query["q"]).ToString();
    var field = (ViewBag.Field ?? Context.Request.Query["field"]).ToString();
    var job = (string)ViewBag.Job;
} 
    <form action="/crawl/@Context.Request.QueryString" method="post">
        <div class="blog-wrapper">
            <a href="@searchLink">&#8592; Back to result</a>
            <h2>Crawl and index your result</h2>
            @if (ViewBag.JobValidationError != null)
            {
                <div class="validation-error">@ViewBag.JobValidationError</div>
            }
            <p>
                <label><input type="radio" name="job" value="fi" @Html.Raw((job == "fi") ? "checked" : "") /> <strong>Fetch and index all web page content from referenced WET files</strong>.</label>
            </p>
            <p>
                <label><input type="radio" name="job" value="ei" @Html.Raw((job == "ei") ? "checked" : "") /> <strong>Extract web page content from WET files and index</strong> those resources referenced in your result.</label>
            </p>
            <p>
            <label><input type="radio" disabled="disabled" name="job" value="wci" @Html.Raw((job == "wci") ? "checked" : "") /> <strong>Web crawl and index</strong> those resources referenced in your result. (Coming soon)</label>
        </p>
        <p>
            <label><input type="radio" disabled="disabled" name="job" value="rwci" @Html.Raw((job == "rwci") ? "checked" : "")/> <strong>Recurringly web crawl and index</strong> those resources referenced in your result. (Coming soon)</label>
        </p>
            <p>
                When the crawling job has completed we will have created a searchable document collection.
                We will host this collection for your convenience so that 
                you will be able to query it using a web browser or HTTP client as well as slice it, 
                append to it or download it as JSON.
            </p>
            <p>
                Name your collection carefully.
                You will need to reference your collection by name anytime you wish to interact with it.
{'fraction_non_alphanumeric': 0.14783879724368343, 'fraction_numerical': 0.002505742326164126, 'mean_word_length': 2.5194709772226305, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3006955', 'n_tokens_mistral': 1340, 'n_tokens_neox': 1284, 'n_words': 403}