<commit_before>#!/bin/bash
#
# Copyright © 2012-2013 <NAME> <<EMAIL>>
# 
# Permission to use, copy, modify, and/or distribute this software 
# for any purpose with or without fee is hereby granted, provided that 
# the above copyright notice and this permission notice appear in all copies.
# 
# THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES 
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, 
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE 
# OR PERFORMANCE OF THIS SOFTWARE.
#
# This script gets all the bicycles from
# Biciescapa store !
#
URL="www.mundoebikes.es"
ONLY_DOMAIN="mundoebikes.es"
MAX_TRIES=10
MAX_TIMEOUT=10

function get_page()
{
  BASE_URL="$1"
  PAGES="$2"
  if [ "${PAGES}" = "" ];
  then
    wget -U 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4' \
      --tries=${MAX_TRIES} --timeout=${MAX_TIMEOUT} "${BASE_URL}" 
  else
    for page in ${PAGES};
    do
      wget -U 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.6) Gecko/20070802 SeaMonkey/1.1.4' \
        --tries=${MAX_TRIES} --timeout=${MAX_TIMEOUT} "${BASE_URL}${page}" 
    done 
  fi
}

BMX_BIKES_BASE="${URL}/bicicletas-bmx"
BMX_BIKES_PAGES=""

ROAD_BIKES_BASE="${URL}/bicicletas-de-carretera?limit=20&limitstart="
ROAD_BIKES_PAGES="0 20 40"

ROAD_TRIATLON_BASE="${URL}/bicicletas-triatlon"
ROAD_TRIATLON_PAGES=""

URBAN_ELECTRIC_BIKES_BASE="${URL}/bicicletas-electricas-deportivas?limit=20&limitstart="
URBAN_ELECTRIC_BIKES_PAGES="0 20"

URBAN_FOLDING_BIKES_BASE="${URL}/bicicletas-urbanas-plegables"
URBAN_FOLDING_BIKES_PAGES=""

URBAN_EFOLDING_BIKES_BASE="${URL}/bicicletas-electricas-plegables"
URBAN_EFOLDING_BIKES_PAGES=""

URBAN_EBIKES_BASE="${URL}/bicicletas-electricas-urbanas?limit=20&limitstart="
URBAN_EBIKES_PAGES="0 20 40"

URBAN_FIXIE_BASE="${URL}/bicicletas-fixed-and-single-speed"
URBAN_FIXIE_PAGES=""<commit_msg>Adding some sleeps and randoms to avoid robots detectors<commit_after>#!/bin/bash
#
# Copyright © 2012-2013 <NAME> <<EMAIL>>
# 
# Permission to use, copy, modify, and/or distribute this software 
# for any purpose with or without fee is hereby granted, provided that 
# the above copyright notice and this permission notice appear in all copies.
# 
# THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES 
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY 
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, 
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM 
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, 
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE 
# OR PERFORMANCE OF THIS SOFTWARE.
#
# This script gets all the bicycles from
# Biciescapa store !
#
URL="www.mundoebikes.es"
ONLY_DOMAIN="mundoebikes.es"
MAX_TRIES=10
MAX_TIMEOUT=10

function get_page()
{
  BASE_URL="$1"
  PAGES="$2"
  sleep 5
  if [ "${PAGES}" = "" ];
  then
    wget -w5 --random-wait -e robots=off -U 'mozilla' \
      --tries=${MAX_TRIES} --timeout=${MAX_TIMEOUT} "${BASE_URL}" 
  else
    for page in ${PAGES};
    do
      wget -w6 --random-wait -e robots=off -U 'mozilla' \
        --tries=${MAX_TRIES} --timeout=${MAX_TIMEOUT} "${BASE_URL}${page}" 
    done 
  fi
}

BMX_BIKES_BASE="${URL}/bicicletas-bmx"
BMX_BIKES_PAGES=""

ROAD_BIKES_BASE="${URL}/bicicletas-de-carretera?limit=20&limitstart="
ROAD_BIKES_PAGES="0 20 40"

ROAD_TRIATLON_BASE="${URL}/bicicletas-triatlon"
ROAD_TRIATLON_PAGES=""

URBAN_ELECTRIC_BIKES_BASE="${URL}/bicicletas-electricas-deportivas?limit=20&limitstart="
URBAN_ELECTRIC_BIKES_PAGES="0 20"

URBAN_FOLDING_BIKES_BASE="${URL}/bicicletas-urbanas-plegables"
URBAN_FOLDING_BIKES_PAGES=""

URBAN_EFOLDING_BIKES_BASE="${URL}/bicicletas-electricas-plegables"
URBAN_EFOLDING_BIKES_PAGES=""

URBAN_EBIKES_BASE="${URL}/bicicletas-electricas-urbanas?limit=20&limitstart="
URBAN_EBIKES_PAGES="0 20 40"

URBAN_FIXIE_BASE="${URL}/bicicletas-fixed-and-single-speed"
URBAN_FIXIE_PAGES=""
{'fraction_non_alphanumeric': 0.1226000462641684, 'fraction_numerical': 0.026139255146888734, 'mean_word_length': 5.115983026874116, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '217442', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1604, 'n_words': 389}