<commit_before><map>
  <entry>
    <string>plugin_status</string>
    <string>down</string>
  </entry>
  <entry>
    <string>plugin_identifier</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagPlugin</string>
  </entry>
  <entry>
    <string>plugin_name</string>
    <string>Georg Thieme Verlag Plugin</string>
  </entry>
  <entry>
    <string>plugin_version</string>
    <string>8</string>
  </entry>
  <entry>
    <string>plugin_feature_version_map</string>
    <map>
      <entry>
        <string>Metadata</string>
        <string>4</string>
      </entry>
      <entry>
        <string>Poll</string>
        <string>6</string>
      </entry>
      <entry>
        <string>Substance</string>
        <string>2</string>
      </entry>
    </map>
  </entry>
  <entry>
    <string>required_daemon_version</string>
    <string>1.64.0</string>
  </entry>
  <entry>
    <string>au_name</string>
    <string>"Georg Thieme Verlag Plugin, Base URL %s, Journal ID %s, Volume %s", base_url, journal_id, volume_name</string>
  </entry>
  <entry>
    <string>au_start_url</string>
    <string>"%sejournals/issues/%s/%s", base_url, journal_id, volume_name</string>
  </entry>
  <entry>
    <string>au_feature_urls</string>
    <map>
      <entry>
        <string>au_title</string>
        <string>"%sejournals/journal/%s", base_url, journal_id</string>
      </entry>
      <entry>
        <string>au_volume</string>
        <string>"%sejournals/issues/%s/%s", base_url, journal_id, volume_name</string>
      </entry>
    </map>
  </entry>
  <entry>
    <string>au_crawlrules</string>
    <list>
      <string>1,"^%s.*[.](bmp|css|gif|ico|jpe?g|js|png|tif?f)$", base_url</string>
      <string>4,"^%s.*/10[.][0-9a-z]{4,6}/", base_url</string>
      <string>2,"^%s.*/10[.][0-9a-z]{4,6}/www[.]", base_url</string>
      <string>2,"^%s(?:rss/thieme|ejournals/(?:journal|issues|sample|recommend|topten|linkout))", base_url</string>
      <string>1,"^%sejournals/(?:issue|abstract|html|pdf|ris)/10[.][0-9a-z]{4,6}/[.a-z0-9-]+(?:/BIB)?$", base_url</string>
      <string>1,"^%smedia/.*/supmat/", base_url</string>
    </list>
  </entry>
  <entry>
    <string>plugin_config_props</string>
    <list>
      <org.lockss.daemon.ConfigParamDescr>
        <key>base_url</key>
        <displayName>Base URL</displayName>
        <description>Usually of the form http://&lt;journal-name&gt;.com/</description>
        <type>3</type>
        <size>40</size>
        <definitional>true</definitional>
        <defaultOnly>false</defaultOnly>
      </org.lockss.daemon.ConfigParamDescr>
      <org.lockss.daemon.ConfigParamDescr>
        <key>journal_id</key>
        <displayName>Journal Identifier</displayName>
        <description>Identifier for journal (often used as part of file names)</description>
        <type>1</type>
        <size>40</size>
        <definitional>true</definitional>
        <defaultOnly>false</defaultOnly>
      </org.lockss.daemon.ConfigParamDescr>
      <org.lockss.daemon.ConfigParamDescr>
        <key>volume_name</key>
        <displayName>Volume Name</displayName>
        <type>1</type>
        <size>20</size>
        <definitional>true</definitional>
        <defaultOnly>false</defaultOnly>
      </org.lockss.daemon.ConfigParamDescr>
    </list>
  </entry>
  <entry>
    <string>au_substance_url_pattern</string>
    <list>
      <string>"^%sejournals/html/10[.][0-9a-z]{4,6}/[a-z0-9-]+$", base_url</string>
      <string>"^%sejournals/pdf/10[.][0-9a-z]{4,6}/[a-z0-9-]+[.]pdf$", base_url</string>
    </list>
  </entry>
  <entry>
    <string>au_url_normalizer</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagUrlNormalizer</string>
  </entry>
  <entry>
    <string>text/html_filter_factory</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagHtmlFilterFactory</string>
  </entry>
  <entry>
    <string>application/pdf_filter_factory</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagPdfFilterFactory</string>
  </entry>
  <entry>
    <string>plugin_article_iterator_factory</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagArticleIteratorFactory</string>
  </entry>
  <entry>
    <string>plugin_article_metadata_extractor_factory</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagArticleIteratorFactory</string>
  </entry>
  <entry>
    <string>text/html_metadata_extractor_factory_map</string>
    <map>
      <entry>
        <string>*</string>
        <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagHtmlMetadataExtractorFactory</string>
      </entry>
    </map>
  </entry>
  <entry>
    <string>application/pdf_metadata_extractor_factory_map</string>
    <map>
      <entry>
        <string>*</string>
        <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagPdfMetadataExtractorFactory</string>
      </entry>
    </map>
  </entry>
  <entry>
    <string>au_def_pause_time</string>
    <long>3000</long>
  </entry>
  <entry>
    <string>au_def_new_content_crawl</string>
    <long>1209600000</long>
  </entry>
</map>
<commit_msg>Remove au_url_normalizer as it caused overcrawl<commit_after><map>
  <entry>
    <string>plugin_status</string>
    <string>down</string>
  </entry>
  <entry>
    <string>plugin_identifier</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagPlugin</string>
  </entry>
  <entry>
    <string>plugin_name</string>
    <string>Georg Thieme Verlag Plugin</string>
  </entry>
  <entry>
    <string>plugin_version</string>
    <string>8</string>
  </entry>
  <entry>
    <string>plugin_feature_version_map</string>
    <map>
      <entry>
        <string>Metadata</string>
        <string>4</string>
      </entry>
      <entry>
        <string>Poll</string>
        <string>6</string>
      </entry>
      <entry>
        <string>Substance</string>
        <string>2</string>
      </entry>
    </map>
  </entry>
  <entry>
    <string>required_daemon_version</string>
    <string>1.64.0</string>
  </entry>
  <entry>
    <string>au_name</string>
    <string>"Georg Thieme Verlag Plugin, Base URL %s, Journal ID %s, Volume %s", base_url, journal_id, volume_name</string>
  </entry>
  <entry>
    <string>au_start_url</string>
    <string>"%sejournals/issues/%s/%s", base_url, journal_id, volume_name</string>
  </entry>
  <entry>
    <string>au_feature_urls</string>
    <map>
      <entry>
        <string>au_title</string>
        <string>"%sejournals/journal/%s", base_url, journal_id</string>
      </entry>
      <entry>
        <string>au_volume</string>
        <string>"%sejournals/issues/%s/%s", base_url, journal_id, volume_name</string>
      </entry>
    </map>
  </entry>
  <entry>
    <string>au_crawlrules</string>
    <list>
      <string>1,"^%s.*[.](bmp|css|gif|ico|jpe?g|js|png|tif?f)$", base_url</string>
      <string>4,"^%s.*/10[.][0-9a-z]{4,6}/", base_url</string>
      <string>2,"^%s.*/10[.][0-9a-z]{4,6}/www[.]", base_url</string>
      <string>2,"^%s(?:rss/thieme|ejournals/(?:journal|issues|sample|recommend|topten|linkout))", base_url</string>
      <string>1,"^%sejournals/(?:issue|abstract|html|pdf|ris)/10[.][0-9a-z]{4,6}/[.a-z0-9-]+(?:/BIB)?$", base_url</string>
      <string>1,"^%smedia/.*/supmat/", base_url</string>
    </list>
  </entry>
  <entry>
    <string>plugin_config_props</string>
    <list>
      <org.lockss.daemon.ConfigParamDescr>
        <key>base_url</key>
        <displayName>Base URL</displayName>
        <description>Usually of the form http://&lt;journal-name&gt;.com/</description>
        <type>3</type>
        <size>40</size>
        <definitional>true</definitional>
        <defaultOnly>false</defaultOnly>
      </org.lockss.daemon.ConfigParamDescr>
      <org.lockss.daemon.ConfigParamDescr>
        <key>journal_id</key>
        <displayName>Journal Identifier</displayName>
        <description>Identifier for journal (often used as part of file names)</description>
        <type>1</type>
        <size>40</size>
        <definitional>true</definitional>
        <defaultOnly>false</defaultOnly>
      </org.lockss.daemon.ConfigParamDescr>
      <org.lockss.daemon.ConfigParamDescr>
        <key>volume_name</key>
        <displayName>Volume Name</displayName>
        <type>1</type>
        <size>20</size>
        <definitional>true</definitional>
        <defaultOnly>false</defaultOnly>
      </org.lockss.daemon.ConfigParamDescr>
    </list>
  </entry>
  <entry>
    <string>au_substance_url_pattern</string>
    <list>
      <string>"^%sejournals/html/10[.][0-9a-z]{4,6}/[a-z0-9-]+$", base_url</string>
      <string>"^%sejournals/pdf/10[.][0-9a-z]{4,6}/[a-z0-9-]+[.]pdf$", base_url</string>
    </list>
  </entry>
  <entry>
    <string>text/html_filter_factory</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagHtmlFilterFactory</string>
  </entry>
  <entry>
    <string>application/pdf_filter_factory</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagPdfFilterFactory</string>
  </entry>
  <entry>
    <string>plugin_article_iterator_factory</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagArticleIteratorFactory</string>
  </entry>
  <entry>
    <string>plugin_article_metadata_extractor_factory</string>
    <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagArticleIteratorFactory</string>
  </entry>
  <entry>
    <string>text/html_metadata_extractor_factory_map</string>
    <map>
      <entry>
        <string>*</string>
        <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagHtmlMetadataExtractorFactory</string>
      </entry>
    </map>
  </entry>
  <entry>
    <string>application/pdf_metadata_extractor_factory_map</string>
    <map>
      <entry>
        <string>*</string>
        <string>org.lockss.plugin.georgthiemeverlag.GeorgThiemeVerlagPdfMetadataExtractorFactory</string>
      </entry>
    </map>
  </entry>
  <entry>
    <string>au_def_pause_time</string>
    <long>3000</long>
  </entry>
  <entry>
    <string>au_def_new_content_crawl</string>
    <long>1209600000</long>
  </entry>
</map>

{'fraction_non_alphanumeric': 0.17345325163075706, 'fraction_numerical': 0.014429729195493181, 'mean_word_length': 3.3938341293964394, 'pattern_counts': {'":': 0, '<': 461, '<?xml version=': 0, '>': 461, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6649826', 'n_tokens_mistral': 3773, 'n_tokens_neox': 3678, 'n_words': 409}