<commit_before>#!/usr/bin/perl
#
# Copyright © 2012-2014 <NAME> <<EMAIL>>
#
# Permission to use, copy, modify, and/or distribute this software
# for any purpose with or without fee is hereby granted, provided that
# the above copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
# OR PERFORMANCE OF THIS SOFTWARE.
#
require "CommonGet.pl";
use constant STORE => "Zermatt";
use constant URL => "http://www.zermattbike.es";
use constant ONLY_DOMAIN => "zermattbike.es";
use constant EXCLUDE => "-Rgif -Rpng -Rjpg";
use constant MAX_TRIES => 10;
use constant MAX_TIMEOUT => 10;
use constant OUTPUT_FILE => "output";
use constant MAX_PRICE => 30;

my $TRADEMARK_KEY="TRADEMARK";
my $SUBURL_KEY="SUBURL";
my $STORE_KEY="STORE";
my $PRICE_KEY="PRICE";
my $KIND_KEY="KIND";

sub dump_model {
  my $file = $_[0];
  my $trademark_model = $_[1];
  my $type = $_[2];
  my $store = $_[3];
  my $trimmed_trademark_model = $trademark_model;
  $trimmed_trademark_model =~ s/^\s+|\s+$//g;
  my @trade_model = split(/\s+/, $trimmed_trademark_model);
  my $model_search_trademark = sprintf('>%s<', $trimmed_trademark_model);
  my $search_trademark_cmd = "grep \"$model_search_trademark\" $file -A10 | grep alt";
  my $search_trademark = `$search_trademark_cmd`;
  ($search_first, $search_sec) = split('alt=', $search_trademark, -1);
  my @trademark_all = split(' ', $search_sec);
  my $trademark = $trademark_all[0];
  $trademark =~ s/"//g;
  my $model;
  my $url;

  foreach my $elem (@trade_model) {
    if ($elem ne $trademark) {
      $model .= "$elem ";
    }
  }
  if ($model eq "") {
    $model = $trademark;
  }

  # URL PARSING
  my $url_cmd = "grep \"$trimmed_trademark_model\" $file | grep href";
  $url_all = `$url_cmd`;
  ($url_first, $url_no_html) = split('href=', $url_all, -1);
  ($url_no_dash, $url_later) = split(' ', $url_no_html, -1);
  chomp($url_no_dash);
  my $url = $url_no_dash;
  $url =~ s/"/\\"/g;

  # PRICE PARSING
  my $price_cmd = sprintf("grep -A%d \"$trimmed_trademark_model\" $file " .
"| grep 'itemprop=\"price\"' | tail -1", MAX_PRICE);
  my $price_html = `$price_cmd`;
  if ($price_html eq "") {
    $price_cmd = sprintf("grep -A%d \"$trimmed_trademark_model\" $file " .
"| grep '<strike>' | tail -1", MAX_PRICE);
    $price_html = `$price_cmd`;
  }
  $price_html =~ s/<.+?>//sg;
  $price_html =~ s/\s+//sg;
  my $price = $price_html;
  $price =~ s/\.//sg;
  chomp($price);

  # TRADEMARK/MODEL processing
  my $clean_trademark = `bash -c 'source ./common_proc; bubic_clean "$trademark"'`;
  my $clean_model = `bash -c 'source ./common_proc; bubic_clean "$model"'`;
  chomp($clean_trademark);
  chomp($clean_model);
  my $camel_trademark = `bash -c 'source ./common_proc; bubic_camel "$clean_trademark"'`;
  my $camel_model = `bash -c 'source ./common_proc; bubic_camel "$clean_model"'`;
  chomp($camel_trademark);
  chomp($camel_model);
  #print "================================";
  #print "\nTRADEMARK:$trademark\n";
  #print "MODEL_SEARCH_TRADEMARK:$model_search_trademark\n";
  #print "MODEL_SEARCH_CMD:$search_trademark_cmd\n";
  #print "SEARCH FIRST:$search_first\n";
  #print "SEARCH SEC:$search_sec\n";
  #print "MODEL:$model\n";
  #print "TYPE:$type\n";<commit_msg>Fixing price in Zermattbike<commit_after>#!/usr/bin/perl
#
# Copyright © 2012-2014 <NAME> <<EMAIL>>
#
# Permission to use, copy, modify, and/or distribute this software
# for any purpose with or without fee is hereby granted, provided that
# the above copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
# AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
# LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
# OR PERFORMANCE OF THIS SOFTWARE.
#
require "CommonGet.pl";
use constant STORE => "Zermatt";
use constant URL => "http://www.zermattbike.es";
use constant ONLY_DOMAIN => "zermattbike.es";
use constant EXCLUDE => "-Rgif -Rpng -Rjpg";
use constant MAX_TRIES => 10;
use constant MAX_TIMEOUT => 10;
use constant OUTPUT_FILE => "output";
use constant MAX_PRICE => 30;

my $TRADEMARK_KEY="TRADEMARK";
my $SUBURL_KEY="SUBURL";
my $STORE_KEY="STORE";
my $PRICE_KEY="PRICE";
my $KIND_KEY="KIND";

sub dump_model {
  my $file = $_[0];
  my $trademark_model = $_[1];
  my $type = $_[2];
  my $store = $_[3];
  my $trimmed_trademark_model = $trademark_model;
  $trimmed_trademark_model =~ s/^\s+|\s+$//g;
  my @trade_model = split(/\s+/, $trimmed_trademark_model);
  my $model_search_trademark = sprintf('>%s<', $trimmed_trademark_model);
  my $search_trademark_cmd = "grep \"$model_search_trademark\" $file -A10 | grep alt";
  my $search_trademark = `$search_trademark_cmd`;
  ($search_first, $search_sec) = split('alt=', $search_trademark, -1);
  my @trademark_all = split(' ', $search_sec);
  my $trademark = $trademark_all[0];
  $trademark =~ s/"//g;
  my $model;
  my $url;

  foreach my $elem (@trade_model) {
    if ($elem ne $trademark) {
      $model .= "$elem ";
    }
  }
  if ($model eq "") {
    $model = $trademark;
  }

  # URL PARSING
  my $url_cmd = "grep \"$trimmed_trademark_model\" $file | grep href";
  $url_all = `$url_cmd`;
  ($url_first, $url_no_html) = split('href=', $url_all, -1);
  ($url_no_dash, $url_later) = split(' ', $url_no_html, -1);
  chomp($url_no_dash);
  my $url = $url_no_dash;
  $url =~ s/"/\\"/g;

  # PRICE PARSING
  my $price_cmd = sprintf("grep -A%d \"$trimmed_trademark_model\" $file " .
"| grep 'itemprop=\"price\"' | tail -1", MAX_PRICE);
  my $price_html = `$price_cmd`;
  if ($price_html eq "") {
    $price_cmd = sprintf("grep -A%d \"$trimmed_trademark_model\" $file " .
"| grep '<strike>' | tail -1", MAX_PRICE);
    $price_html = `$price_cmd`;
  }
  $price_html =~ s/<.+?>//sg;
  $price_html =~ s/\s+//sg;
  my $price = $price_html;
  $price =~ s/\.//sg;
  chomp($price);
  $price =~ s/[^0-9,\,]//sg;

  # TRADEMARK/MODEL processing
  my $clean_trademark = `bash -c 'source ./common_proc; bubic_clean "$trademark"'`;
  my $clean_model = `bash -c 'source ./common_proc; bubic_clean "$model"'`;
  chomp($clean_trademark);
  chomp($clean_model);
  my $camel_trademark = `bash -c 'source ./common_proc; bubic_camel "$clean_trademark"'`;
  my $camel_model = `bash -c 'source ./common_proc; bubic_camel "$clean_model"'`;
  chomp($camel_trademark);
  chomp($camel_model);
  #print "================================";
  #print "\nTRADEMARK:$trademark\n";
  #print "MODEL_SEARCH_TRADEMARK:$model_search_trademark\n";
  #print "MODEL_SEARCH_CMD:$search_trademark_cmd\n";
  #print "SEARCH FIRST:$search_first\n";
  #print "SEARCH SEC:$search_sec\n";
  #print "MODEL:$model\n";
  #print "TYPE:$type\n";
{'fraction_non_alphanumeric': 0.16966580976863754, 'fraction_numerical': 0.007306183195778649, 'mean_word_length': 4.451327433628318, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4128578', 'n_tokens_mistral': 2913, 'n_tokens_neox': 2785, 'n_words': 718}