<commit_before><meta_metadata_repository xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  name="googleImageSearch" package="ecologylab.bigsemantics.generated.library.search"
  default_user_agent_name="chrome_2">

  <meta_metadata name="google_image_search" extends="image_search" comment="Google image search."
    parser="xpath" extract_with="service">
<!--     <selector url_stripped="https://www.google.com/search"> -->
<!--       <param name="tbm" value="isch" /> -->
<!--     </selector>     -->
<!--     <selector url_stripped="http://www.google.com/search"> -->
<!--       <param name="tbm" value="isch" /> -->
<!--     </selector> -->
    <selector url_regex="https?://www.google.com/search\?.*" domain="google.com">
      <param name="tbm" value="isch" />
    </selector>
    <selector url_regex="https?://www.google.com/webhp\?.*" domain="google.com">
      <param name="tbm" value="isch" />
    </selector>
    <example_url url="https://www.google.com/search?tbm=isch&amp;hl=en&amp;q=watergate" />
    <example_url url="http://www.google.com/search?hl=en&amp;tbm=isch&amp;q=earth" />

<!--     <filter_location> -->
<!--       <regex match="^https://" replace="http://" /> -->
<!--       <set_param name="hl" value="en" /> -->
<!--       <set_param name="gbv" value="1" /> -->
<!--       <set_param name="sei" value="1" /> -->
<!--     </filter_location> -->

    <filter_location>
      <override_params />
      <strip_param name="source" />
      <strip_param name="newwindow" />
      <strip_param name="biw" />
      <strip_param name="bih" />
      <strip_param name="oq" />
    </filter_location>
	
    <collection name="search_results">
      <xpath>//div[@id='ires']//td</xpath>
      <scalar name="location" always_show="true">
        <xpath>./a/@href</xpath>
        <regex_op regex="^/url\?q=" replace="" />
      </scalar>
      	<scalar name ="title">
      		<xpath>./cite</xpath>
      	</scalar>
      <scalar name="thumb_width">
        <xpath>./a/img/@width</xpath>
      </scalar>
      <scalar name="thumb_height">
        <xpath>./a/img/@height</xpath>
      </scalar>
      <scalar name="description">
      	<xpath>./b/following-sibling::text()[1]</xpath>
      </scalar>
      <collection name="main_images">
      	<xpath>./a</xpath>
      	<scalar name="location">
      		<xpath>./img/@src</xpath>
      		<regex_op match="https://encrypted-tbn([0-9]+)\.google\.com/images" replace="http://t$1.gstatic.com/images" />
      	</scalar>
      </collection>
    </collection>
    
    <semantic_actions>
      <get_field name="search_results" />
      <for_each collection="search_results" as="result">
        <get_field object="result" name="location" />
        <parse_document>
          <arg name="location" value="location" />
        </parse_document>

        <get_field object="result" name="ref_location" />
        <parse_document>
          <arg name="location" value="ref_location" />
        </parse_document>
      </for_each>
    </semantic_actions>
-
  </meta_metadata>

</meta_metadata_repository><commit_msg>fixed issue with image serach urls<commit_after><meta_metadata_repository xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  name="googleImageSearch" package="ecologylab.bigsemantics.generated.library.search"
  default_user_agent_name="chrome_2">

  <meta_metadata name="google_image_search" extends="image_search" comment="Google image search."
    parser="xpath" extract_with="service">
<!--     <selector url_stripped="https://www.google.com/search"> -->
<!--       <param name="tbm" value="isch" /> -->
<!--     </selector>     -->
<!--     <selector url_stripped="http://www.google.com/search"> -->
<!--       <param name="tbm" value="isch" /> -->
<!--     </selector> -->
    <selector url_regex="https?://www.google.com/search\?.*" domain="google.com">
      <param name="tbm" value="isch" />
    </selector>
    <selector url_regex="https?://www.google.com/webhp\?.*" domain="google.com">
      <param name="tbm" value="isch" />
    </selector>
    <example_url url="https://www.google.com/search?tbm=isch&amp;hl=en&amp;q=watergate" />
    <example_url url="http://www.google.com/search?hl=en&amp;tbm=isch&amp;q=earth" />

<!--     <filter_location> -->
<!--       <regex match="^https://" replace="http://" /> -->
<!--       <set_param name="hl" value="en" /> -->
<!--       <set_param name="gbv" value="1" /> -->
<!--       <set_param name="sei" value="1" /> -->
<!--     </filter_location> -->

    <filter_location>
      <override_params />
      <strip_param name="source" />
      <strip_param name="newwindow" />
      <strip_param name="biw" />
      <strip_param name="bih" />
      <strip_param name="oq" />
    </filter_location>
	
    <collection name="search_results">
      <xpath>//div[@id='ires']//td</xpath>
      <scalar name="location" always_show="true">
        <xpath>./a/@href</xpath>
        <location_op extract_param="q" decode_url="true" />
        
      </scalar>
      	<scalar name ="title">
      		<xpath>./cite</xpath>
      	</scalar>
      <scalar name="thumb_width">
        <xpath>./a/img/@width</xpath>
      </scalar>
      <scalar name="thumb_height">
        <xpath>./a/img/@height</xpath>
      </scalar>
      <scalar name="description">
      	<xpath>./b/following-sibling::text()[1]</xpath>
      </scalar>
      <collection name="main_images">
      	<xpath>./a</xpath>
      	<scalar name="location">
      		<xpath>./img/@src</xpath>
      		<regex_op match="https://encrypted-tbn([0-9]+)\.google\.com/images" replace="http://t$1.gstatic.com/images" />
      	</scalar>
      </collection>
    </collection>
    
    <semantic_actions>
      <get_field name="search_results" />
      <for_each collection="search_results" as="result">
        <get_field object="result" name="location" />
        <parse_document>
          <arg name="location" value="location" />
        </parse_document>

        <get_field object="result" name="ref_location" />
        <parse_document>
          <arg name="location" value="ref_location" />
        </parse_document>
      </for_each>
    </semantic_actions>
-
  </meta_metadata>

</meta_metadata_repository>
{'fraction_non_alphanumeric': 0.20533980582524272, 'fraction_numerical': 0.003883495145631068, 'mean_word_length': 2.780428134556575, 'pattern_counts': {'":': 0, '<': 185, '<?xml version=': 0, '>': 185, 'https://': 8, 'lorem ipsum': 0, 'www.': 14, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3748343', 'n_tokens_mistral': 2076, 'n_tokens_neox': 2046, 'n_words': 276}