<commit_before>#!/usr/bin/env node
var Crawler = require("crawler").Crawler;
var moment = require('moment');
var argv = require('yargs').argv;

if (argv.h || argv.help) {
  console.log("Get UK tide times on the command line\nUsage: tidetimes @falmouth \ntidetimes tomorrow @falmouth\ntidetimes saturday @cowes")
  process.exit(code=0);
}

function TideCrawler(argv) {

  //inherit from node-crawler
  Crawler.call(this);

  this.location = 'falmouth';  //Default location
  this.date = '';

  // if the location argument starts with @ remove it
  this.formatLocation = function(location) {
    if (location[0] === '@') {
      return location.substring(1)
    } else {
      return location
    }
  }

  // takes tomorrow or a day of the week and converts it to a date url fragment 
  this.formatDate = function(day) {
    // Vaid Day arguments
    var dayArgs = ['tomorrow', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'];

    var index = dayArgs.indexOf(day);
    if (index > -1 ){
      if (index === 0) {
        var date = moment().add('days', 1).format('-YYYYMMDD');
      } else {
        var date = moment().day(index).format('-YYYYMMDD');
      }
      return date;
    } 
  }

  // Format the scaped tide times by removing extra spaces, adding a line break after m, and removing the leading spaces
  this.formatTideTimes = function(times) {
    return times.replace(/\s+/g, " ").replace(/m /g, "m\n").substring(1)
  }

  //Set location and date via values passed from args
  if (argv._.length == 1) {
    // is the argument a date or a location?
    var d = this.formatDate(argv._[0])
    if (typeof(d) != "undefined"){
      this.date = d
    } else {
      location = this.formatLocation(argv._[0])
    }
    } else if (argv._.length == 2) {
    // two arguments provided
    // first argument should be date related
    // second argument shoudl be a location
    this.date = this.formatDate(argv._[0])

    this.location = this.formatLocation(argv._[1])
  }

  // the uri of page where we will scrape the data
  this.uri = 'http://www.tidetimes.co.uk/'+this.location+'-tide-times'+this.date;


  this.getTimes = function(){

    // Scrape the data
    var c = new Crawler({
    "maxConnections":1,

    "callback":function(error,result,$) {
        
        if(!error && result.statusCode === 200){
          return $(".times").text();
        }else{
          return 'No data available for this location'
        }
    }
    });

    // !!!! Need to call formatTideTimes after c.queue completes !!!!        
    this.formatTideTimes(c.queue(this.uri));
  }


}

var tides = new TideCrawler(argv);
console.log(tides.getTimes());<commit_msg>no need to inherit from node-crawler<commit_after>#!/usr/bin/env node
var Crawler = require("crawler").Crawler;
var moment = require('moment');
var argv = require('yargs').argv;

if (argv.h || argv.help) {
  console.log("Get UK tide times on the command line\nUsage: tidetimes @falmouth \ntidetimes tomorrow @falmouth\ntidetimes saturday @cowes")
  process.exit(code=0);
}

function TideCrawler(argv) {

  this.location = 'falmouth';  //Default location
  this.date = '';

  // if the location argument starts with @ remove it
  this.formatLocation = function(location) {
    if (location[0] === '@') {
      return location.substring(1)
    } else {
      return location
    }
  }

  // takes tomorrow or a day of the week and converts it to a date url fragment 
  this.formatDate = function(day) {
    // Vaid Day arguments
    var dayArgs = ['tomorrow', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday'];

    var index = dayArgs.indexOf(day);
    if (index > -1 ){
      if (index === 0) {
        var date = moment().add('days', 1).format('-YYYYMMDD');
      } else {
        var date = moment().day(index).format('-YYYYMMDD');
      }
      return date;
    } 
  }

  // Format the scaped tide times by removing extra spaces, adding a line break after m, and removing the leading spaces
  this.formatTideTimes = function(times) {
    return times.replace(/\s+/g, " ").replace(/m /g, "m\n").substring(1)
  }

  //Set location and date via values passed from args
  if (argv._.length == 1) {
    // is the argument a date or a location?
    var d = this.formatDate(argv._[0])
    if (typeof(d) != "undefined"){
      this.date = d
    } else {
      location = this.formatLocation(argv._[0])
    }
    } else if (argv._.length == 2) {
    // two arguments provided
    // first argument should be date related
    // second argument shoudl be a location
    this.date = this.formatDate(argv._[0])

    this.location = this.formatLocation(argv._[1])
  }

  // the uri of page where we will scrape the data
  this.uri = 'http://www.tidetimes.co.uk/'+this.location+'-tide-times'+this.date;


  this.getTimes = function(){

    // Scrape the data
    var c = new Crawler({
    "maxConnections":1,

    "callback":function(error,result,$) {
        
        if(!error && result.statusCode === 200){
          return $(".times").text();
        }else{
          return 'No data available for this location'
        }
    }
    });

    // TODO !!!! Need to call formatTideTimes after c.queue completes !!!!        
    this.formatTideTimes(c.queue(this.uri));
  }


}

var tides = new TideCrawler(argv);
console.log(tides.getTimes());
{'fraction_non_alphanumeric': 0.1477463998503834, 'fraction_numerical': 0.0063587058163456145, 'mean_word_length': 2.726829268292683, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2152783', 'n_tokens_mistral': 1763, 'n_tokens_neox': 1686, 'n_words': 477}