<commit_before>'use strict';

var expect = require("chai").expect,
    fs = require('fs'),
    Dom = require(__dirname + "/../lib/util/Dom.js"),
    BaseScraper = require(__dirname + "/../lib/scraper/BaseScraper.js"),
    htmlTest1 = fs.readFileSync(__dirname + "/assets/BaseScraper_Test1.html"),
    htmlTest2 = fs.readFileSync(__dirname + "/assets/BaseScraper_Test2.html");

describe("BaseScraper.scrapeHtml", function() {
  
  it("should return 3 rows of scraped data for test 1", function(){
    var colSelectors = {
      id: "tr td:nth-child(1)",
      name: "tr td:nth-child(2)",
      memo: "tr td:nth-child(3)"
    }
    
    var results = BaseScraper.scrapeHtml(htmlTest1, colSelectors);
    
    expect(results.length).to.equal(3);
  });
  
  it("should return column based on custom callback for test 2", function(){
    var colSelectors = {
      first: "tr td:nth-child(1)",
      second: "tr td:nth-child(2)",
      sum: function(window, currentIndex, fullElementArray){
        var currentRow = fullElementArray[currentIndex],
            num1 = parseInt(currentRow[0].innerHTML),
            num2 = parseInt(currentRow[1].innerHTML);
        
        return num1 + num2; 
      }
    }
    
    var results = BaseScraper.scrapeHtml(htmlTest2, colSelectors);
    
    expect(results[0][3]).to.equal(2);
    expect(results[1][3]).to.equal(4);
    expect(results[2][3]).to.equal(6);
  });
  
  it("should return column based on custom callback for test 2 in the right column position", function(){
    var colSelectors = {
      first: "tr td:nth-child(1)",
      sum: function(window, currentRowIndex, fullElementArray){
        console.log(currentRowIndex)
        var element1 = fullElementArray[0][currentRowIndex], 
            element2 = fullElementArray[1][currentRowIndex],
            num1 = element1 ? element1.innerHTML : 0,
            num2 = element2 ? element2.innerHTML : 0;
        return num1 + num2; 
      },
      second: "tr td:nth-child(2)",
    }
    
    var results = BaseScraper.scrapeHtml(htmlTest2, colSelectors);
    
    expect(results[0][2]).to.equal(2);
    expect(results[1][2]).to.equal(4);
    expect(results[2][2]).to.equal(6);
  });
  
});

describe("BaseScraper.getElementArray", function(){
  it("should get an element array for test1", function(){
    var colSelectors = {
      id: "tr td:nth-child(1)",
      name: "tr td:nth-child(2)",
      random: function(){return "foo"},
      memo: "tr td:nth-child(3)"
    };
    
    var window = Dom.getDomWithJquery(htmlTest1),
        elementArray = BaseScraper.getElementArray(window, colSelectors);
    
    expect(elementArray[0][2].innerHTML).to.equal('77de68daecd823babbb58edb1c8e14d7106e83bb');
    expect(elementArray[1].length).to.equal(3);
    expect(typeof elementArray[2]).to.equal('function');
    expect(elementArray[3].length).to.equal(3);

  });
});<commit_msg>all tests passing<commit_after>'use strict';

var expect = require("chai").expect,
    fs = require('fs'),
    Dom = require(__dirname + "/../lib/util/Dom.js"),
    BaseScraper = require(__dirname + "/../lib/scraper/BaseScraper.js"),
    htmlTest1 = fs.readFileSync(__dirname + "/assets/BaseScraper_Test1.html"),
    htmlTest2 = fs.readFileSync(__dirname + "/assets/BaseScraper_Test2.html");

describe("BaseScraper.scrapeHtml", function() {
  
  it("should return 3 rows of scraped data for test 1", function(){
    var colSelectors = {
      id: "tr td:nth-child(1)",
      name: "tr td:nth-child(2)",
      memo: "tr td:nth-child(3)"
    }
    
    var results = BaseScraper.scrapeHtml(htmlTest1, colSelectors);
    
    expect(results.length).to.equal(3);
  });
  
  it("should return column based on custom callback for test 2", function(){
    var colSelectors = {
      first: "tr td:nth-child(1)",
      second: "tr td:nth-child(2)",
      sum: function(window, currentRowIndex, fullElementArray){
        var elementFirst = fullElementArray[0][currentRowIndex],
            elementSecond = fullElementArray[1][currentRowIndex],
            numFirst = parseInt(elementFirst.innerHTML),
            numSecond = parseInt(elementSecond.innerHTML);
        
        return numFirst + numSecond; 
      }
    }
    
    var results = BaseScraper.scrapeHtml(htmlTest2, colSelectors);
    console.log(results)
    expect(results[2][0]).to.equal(2);
    expect(results[2][1]).to.equal(4);
    expect(results[2][2]).to.equal(6);
  });
  
});

describe("BaseScraper.getElementArray", function(){
  it("should get an element array for test1", function(){
    var colSelectors = {
      id: "tr td:nth-child(1)",
      name: "tr td:nth-child(2)",
      random: function(){return "foo"},
      memo: "tr td:nth-child(3)"
    };
    
    var window = Dom.getDomWithJquery(htmlTest1),
        elementArray = BaseScraper.getElementArray(window, colSelectors);
    
    expect(elementArray[0][2].innerHTML).to.equal('77de68daecd823babbb58edb1c8e14d7106e83bb');
    expect(elementArray[1].length).to.equal(3);
    expect(typeof elementArray[2]).to.equal('function');
    expect(elementArray[3].length).to.equal(3);

  });
});
{'fraction_non_alphanumeric': 0.15399725004910628, 'fraction_numerical': 0.028481634256531132, 'mean_word_length': 3.2825904121110177, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4704368', 'n_tokens_mistral': 1723, 'n_tokens_neox': 1686, 'n_words': 303}