<commit_before><map>
    <entry>
        <string>plugin_status</string>
        <string>testing</string>
    </entry>
    <entry>
        <string>plugin_identifier</string>
        <string>org.lockss.plugin.michigan.UMichFulcrumBooksPlugin</string>
    </entry>
    <entry>
        <string>plugin_version</string>
        <string>8</string>
    </entry>
    <entry>
        <string>plugin_name</string>
        <string>University of Michigan Press Books Plugin</string>
    </entry>
    <entry>
        <string>au_name</string>
        <string>"University of Michigan Press Books Plugin, Base URL %s, Path %s", base_url, book_uri</string>
    </entry>
    <entry>
        <string>au_start_url</string>
        <string>"%s%s", base_url, book_uri</string>
    </entry>
    <!--  don't currently have one; work around for now -->
    <entry>
        <string>au_permission_url</string>
        <string>"http://clockss-ingest.lockss.org/lockss.txt"</string>
    </entry>
    <entry>
        <string>au_permitted_host_pattern</string>
        <list>
            <string>www\.fulcrum\.org</string>
            <string>"(.+\.cloudfront.net|use\.typekit\.net)"</string>
        </list>
    </entry>
    <entry>
        <string>au_def_new_content_crawl</string>
        <long>1209600000</long>
    </entry>
    <entry>
        <string>au_def_pause_time</string>
        <long>3000</long>
    </entry>
    <entry>
        <!-- start page to download page -->
        <string>au_refetch_depth</string>
        <int>2</int>
    </entry>
    <entry>
        <string>au_crawlrules</string>
        <list>
            <!-- curated CDN list with terminators -->
            <string>1,"^https?://(.+\.cloudfront.net/assets|use\.typekit\.net)/[^?]*\.(css|js)", url_host(base_url)</string>
            <string>1,"^https?://%s/[^?]*\.(bmp|css|eot|gif|ico|jpe?g|js|otf|png|svg|tif?f|ttf|woff2?)([?][0-9]+)?$", url_host(base_url)</string>
            <string>2,"^https?://%s/%s\.(atom|json|rss)", url_host(base_url), book_uri</string>
            <string>2,"^https?://%s/%s.*&amp;(sort|per_page|view)=", url_host(base_url), book_uri</string>
            <!--  %% is the way to represent a percent in a crawl rule -->
            <string>2,"^https?://%s/%s\?f%%5B", url_host(base_url), book_uri</string>
            <!--  a strange redundant argument that causes a 503 now normalized-->
            <!-- <string>2,"^https?://%s/%s.*\?utf8=", url_host(base_url), book_uri</string>-->
            <string>1,"^https?://%s/%s", url_host(base_url), book_uri</string>
            <string>1,"^https?://%s/(concern/file_sets|downloads|epubs|image-service)/", url_host(base_url)</string>
        </list>
    </entry>
    <entry>
        <string>au_substance_url_pattern</string>
        <list>
            <!--yes to: https://www.fulcrum.org/downloads/00000070p and no to https://www.fulcrum.org/downloads/00000070p?file=thumbnail -->
            <!-- there is no way to differentiate the book substance from supporting assets -->
            <string>"^https?://%s/downloads/[^?/]+$", url_host(base_url)</string>
        </list>
    </entry>
    <entry>
        <string>plugin_config_props</string>
        <list>
            <org.lockss.daemon.ConfigParamDescr>
                <key>base_url</key>
                <displayName>Base URL</displayName>
                <description>Usually of the form http://&lt;journal-name&gt;.com/</description>
                <type>3</type>
                <size>40</size>
                <definitional>true</definitional>
                <defaultOnly>false</defaultOnly>
            </org.lockss.daemon.ConfigParamDescr>
            <org.lockss.daemon.ConfigParamDescr>
                <key>book_uri</key>
                <displayName>Path</displayName>
                <description>Path for book (used as part of file name)</description>
                <type>1</type>
                <size>200</size>
                <definitional>true</definitional>
                <defaultOnly>false</defaultOnly>
            </org.lockss.daemon.ConfigParamDescr>
        </list>
    </entry>
    <entry>
        <string>text/html_filter_factory</string>
        <string>org.lockss.plugin.michigan.UMichHtmlHashFilterFactory</string>
    </entry>
    <entry>
        <string>au_url_normalizer</string>
        <string>org.lockss.plugin.michigan.UMichUrlNormalizer</string>
    </entry>
    <entry>
        <string>text/html_link_extractor_factory</string>
        <string>org.lockss.plugin.michigan.UMichHtmlLinkExtractorFactory</string>
    </entry>
    <entry>
        <string>application/json_link_extractor_factory</string>
        <string>org.lockss.plugin.michigan.IiifJsonLinkExtractorFactory</string>
    </entry>
    <entry>
        <string>application/ld+json_link_extractor_factory</string>
        <string>org.lockss.plugin.michigan.IiifJsonLinkExtractorFactory</string>
    </entry>
    <entry>
        <string>application/json_link_rewriter_factory</string>
        <string>org.lockss.plugin.michigan.UMichJsonLinkRewriterFactory</string>
    </entry>
    <entry>
        <string>application/ld+json_link_rewriter_factory</string>
        <string>org.lockss.plugin.michigan.UMichJsonLinkRewriterFactory</string>
    </entry>
    <entry>
        <string>plugin_article_iterator_factory</string>
        <string>org.lockss.plugin.michigan.UMichBookIteratorFactory</string>
    </entry>
    <entry>
        <string>plugin_article_metadata_extractor_factory</string>
        <string>org.lockss.plugin.michigan.UMichBookIteratorFactory</string>
    </entry>
    <entry>
        <string>text/html_metadata_extractor_factory_map</string>
        <map>
            <entry>
                <string>*;DublinCore</string>
                <string>org.lockss.plugin.michigan.UMichHtmlMetadataExtractorFactory</string>
            </entry>
        </map>
    </entry>
    <entry>
        <string>text/javascript_link_rewriter_factory</string>
        <string>org.lockss.plugin.michigan.UMichJavaScriptLinkRewriterFactory</string>
    </entry>
    <entry>
        <string>au_repair_from_peer_if_missing_url_pattern</string>
        <list>
            <string>.+[.](bmp|css|dfont|eot|gif|ico|jpe?g|js|otf|png|svg|tif?f|ttc|ttf|woff.?)(\?.*)?$"</string>
        </list>
    </entry>
    <entry>
        <!--these will not count towards poll results but will replicate from peer if above-->
        <string>au_url_poll_result_weight</string>
        <list>
            <string>".+[.](bmp|css|dfont|eot|gif|ico|jpe?g|js|otf|png|svg|tif?f|ttc|ttf|woff.?)(\?.*)?$", 0</string>
            <!--  do not replicate, just don't count in poll; it's a frame for the epub viewer and hashes to zero -->
            <string>"^%sepubs/[^/]+$", base_url, 0</string>
        </list>
    </entry>
    <!--  tell the default base.MimeTypeContentValidator what url patterns to validate by mime type -->
    <!--  this won't work - it could be PDF also -->
    <entry>
        <string>au_url_mime_validation_map</string>
        <list>
            <string>/download/, application/epub+zip</string>
        </list>
    </entry>
</map><commit_msg>Better encapsulation of external resources (Google Fonts, etc.). Not currently included: Typekit.<commit_after><map>
    <entry>
        <string>plugin_status</string>
        <string>testing</string>
    </entry>
    <entry>
        <string>plugin_identifier</string>
        <string>org.lockss.plugin.michigan.UMichFulcrumBooksPlugin</string>
    </entry>
    <entry>
        <string>plugin_version</string>
        <string>8</string>
    </entry>
    <entry>
        <string>plugin_name</string>
        <string>University of Michigan Press Books Plugin</string>
    </entry>
    <entry>
        <string>au_name</string>
        <string>"University of Michigan Press Books Plugin, Base URL %s, Path %s", base_url, book_uri</string>
    </entry>
    <entry>
        <string>au_start_url</string>
        <string>"%s%s", base_url, book_uri</string>
    </entry>
    <!--  don't currently have one; work around for now -->
    <entry>
        <string>au_permission_url</string>
        <string>"http://clockss-ingest.lockss.org/lockss.txt"</string>
    </entry>
    <entry>
        <string>au_permitted_host_pattern</string>
        <list>
            <string>www\.fulcrum\.org</string>
            <string>"(.+\.cloudfront.net|use\.typekit\.net)"</string>
        </list>
    </entry>
    <entry>
        <string>au_def_new_content_crawl</string>
        <long>1209600000</long>
    </entry>
    <entry>
        <string>au_def_pause_time</string>
        <long>3000</long>
    </entry>
    <entry>
        <!-- start page to download page -->
        <string>au_refetch_depth</string>
        <int>2</int>
    </entry>
    <entry>
        <string>au_crawlrules</string>
        <list>
            <string>1,"^https?://%s/[^?]*\.(bmp|css|eot|gif|ico|jpe?g|js|otf|png|svg|tif?f|ttf|woff2?)([?][0-9]+)?$", url_host(base_url)</string>
            <string>1,"^https?://([^.]+\.cloudfront\.net|(fonts|ajax)\.(googleapis|gstatic)\.com)/"</string>
            <string>2,"^https?://%s/%s\.(atom|json|rss)", url_host(base_url), book_uri</string>
            <string>2,"^https?://%s/%s.*&amp;(sort|per_page|view)=", url_host(base_url), book_uri</string>
            <!--  %% is the way to represent a percent in a crawl rule -->
            <string>2,"^https?://%s/%s\?f%%5B", url_host(base_url), book_uri</string>
            <!--  a strange redundant argument that causes a 503 now normalized-->
            <!-- <string>2,"^https?://%s/%s.*\?utf8=", url_host(base_url), book_uri</string>-->
            <string>1,"^https?://%s/%s", url_host(base_url), book_uri</string>
            <string>1,"^https?://%s/(concern/file_sets|downloads|epubs|image-service)/", url_host(base_url)</string>
        </list>
    </entry>
    <entry>
        <string>au_substance_url_pattern</string>
        <list>
            <!--yes to: https://www.fulcrum.org/downloads/00000070p and no to https://www.fulcrum.org/downloads/00000070p?file=thumbnail -->
            <!-- there is no way to differentiate the book substance from supporting assets -->
            <string>"^https?://%s/downloads/[^?/]+$", url_host(base_url)</string>
        </list>
    </entry>
    <entry>
        <string>plugin_config_props</string>
        <list>
            <org.lockss.daemon.ConfigParamDescr>
                <key>base_url</key>
                <displayName>Base URL</displayName>
                <description>Usually of the form http://&lt;journal-name&gt;.com/</description>
                <type>3</type>
                <size>40</size>
                <definitional>true</definitional>
                <defaultOnly>false</defaultOnly>
            </org.lockss.daemon.ConfigParamDescr>
            <org.lockss.daemon.ConfigParamDescr>
                <key>book_uri</key>
                <displayName>Path</displayName>
                <description>Path for book (used as part of file name)</description>
                <type>1</type>
                <size>200</size>
                <definitional>true</definitional>
                <defaultOnly>false</defaultOnly>
            </org.lockss.daemon.ConfigParamDescr>
        </list>
    </entry>
    <entry>
        <string>text/html_filter_factory</string>
        <string>org.lockss.plugin.michigan.UMichHtmlHashFilterFactory</string>
    </entry>
    <entry>
        <string>au_url_normalizer</string>
        <string>org.lockss.plugin.michigan.UMichUrlNormalizer</string>
    </entry>
    <entry>
        <string>text/html_link_extractor_factory</string>
        <string>org.lockss.plugin.michigan.UMichHtmlLinkExtractorFactory</string>
    </entry>
    <entry>
        <string>application/json_link_extractor_factory</string>
        <string>org.lockss.plugin.michigan.IiifJsonLinkExtractorFactory</string>
    </entry>
    <entry>
        <string>application/ld+json_link_extractor_factory</string>
        <string>org.lockss.plugin.michigan.IiifJsonLinkExtractorFactory</string>
    </entry>
    <entry>
        <string>application/json_link_rewriter_factory</string>
        <string>org.lockss.plugin.michigan.UMichJsonLinkRewriterFactory</string>
    </entry>
    <entry>
        <string>application/ld+json_link_rewriter_factory</string>
        <string>org.lockss.plugin.michigan.UMichJsonLinkRewriterFactory</string>
    </entry>
    <entry>
        <string>plugin_article_iterator_factory</string>
        <string>org.lockss.plugin.michigan.UMichBookIteratorFactory</string>
    </entry>
    <entry>
        <string>plugin_article_metadata_extractor_factory</string>
        <string>org.lockss.plugin.michigan.UMichBookIteratorFactory</string>
    </entry>
    <entry>
        <string>text/html_metadata_extractor_factory_map</string>
        <map>
            <entry>
                <string>*;DublinCore</string>
                <string>org.lockss.plugin.michigan.UMichHtmlMetadataExtractorFactory</string>
            </entry>
        </map>
    </entry>
    <entry>
        <string>text/javascript_link_rewriter_factory</string>
        <string>org.lockss.plugin.michigan.UMichJavaScriptLinkRewriterFactory</string>
    </entry>
    <entry>
        <string>au_repair_from_peer_if_missing_url_pattern</string>
        <list>
            <string>.+[.](bmp|css|dfont|eot|gif|ico|jpe?g|js|otf|png|svg|tif?f|ttc|ttf|woff.?)(\?.*)?$"</string>
        </list>
    </entry>
    <entry>
        <!--these will not count towards poll results but will replicate from peer if above-->
        <string>au_url_poll_result_weight</string>
        <list>
            <string>".+[.](bmp|css|dfont|eot|gif|ico|jpe?g|js|otf|png|svg|tif?f|ttc|ttf|woff.?)(\?.*)?$", 0</string>
            <!--  do not replicate, just don't count in poll; it's a frame for the epub viewer and hashes to zero -->
            <string>"^%sepubs/[^/]+$", base_url, 0</string>
        </list>
    </entry>
    <!--  tell the default base.MimeTypeContentValidator what url patterns to validate by mime type -->
    <!--  this won't work - it could be PDF also -->
    <entry>
        <string>au_url_mime_validation_map</string>
        <list>
            <string>/download/, application/epub+zip</string>
        </list>
    </entry>
  <entry>
    <string>au_permitted_host_pattern</string>
    <list>
      <string>www\.fulcrum\.org|[^.]+\.cloudfront\.net|(fonts|ajax)\.(googleapis|gstatic)\.com</string>
    </list>
  </entry>
</map>
{'fraction_non_alphanumeric': 0.16172600083114005, 'fraction_numerical': 0.007895830447430393, 'mean_word_length': 2.526868588177821, 'pattern_counts': {'":': 0, '<': 510, '<?xml version=': 0, '>': 510, 'https://': 4, 'lorem ipsum': 0, 'www.': 7, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2404517', 'n_tokens_mistral': 4891, 'n_tokens_neox': 4694, 'n_words': 650}