<commit_before>/*
 * Given a test result data, save it.
 * Asked for a test result, retrieve it.
 */

var debug   = require('debug')('wptc:data_store_fs');
var moment  = require('moment');
var request = require('request');
var mkdirp  = require('mkdirp');
var fs      = require('fs');
var path    = require('path');
var jf      = require('jsonfile');
var os      = require('os');
var junk    = require('junk');

//this should probably come from config
var resultsPath = 'public' + path.sep + 'results' + path.sep;

dataStore = {

  /*
   * Given a test and some results, save to the file system:
   * the json
   * filmstrip image
   * waterfall image
   * for the intial view and the refresh view.
   */
  saveDatapoint: function saveDatapointTest_anon(test, results) {

    //the internet is flaky sometimes
    if (!goodTestResults(results)){
      console.error('Test Died on: ' + results.response.data.testUrl);
      return;
    }

    var response = results.response.data
      , datePath = moment().format('YYYY-MM-DD-HH-mm-ss')
      , datapointPath = test.suitePathName + path.sep + test.testId + path.sep + datePath
      , datapointDir  = resultsPath + datapointPath
      , firstViewFilmstripImg = 'http://www.webpagetest.org/video/filmstrip.php?tests=' + response.testId + '-r:1-c:0&thumbSize=100&ival=500&end=visual&text=ffffff&bg=000000'
      , repeatFilmstripImg = 'http://www.webpagetest.org/video/filmstrip.php?tests=' + response.testId + '-r:1-c:1&thumbSize=100&ival=500&end=visual&text=ffffff&bg=000000'
      , firstViewFilmstrip = 'http://www.webpagetest.org/video/compare.php?tests=' + response.testId + '-r:1-c:0'
      , repeatViewFilmstrip = 'http://www.webpagetest.org/video/compare.php?tests=' + response.testId + '-r:1-c:1'
      ;

    //make the new dir structure
    mkdirp.sync(datapointDir);

    //grab the waterfall image, and save it with the date into the name-d folder
    request(response.run.firstView.images.waterfall).pipe(fs.createWriteStream(datapointDir + path.sep + 'fv_waterfall.png'));
    request(response.run.repeatView.images.waterfall).pipe(fs.createWriteStream(datapointDir + path.sep + 'rv_waterfall.png'));

    //grab the fv filmstrip image, and save a url to the filmstrip page in the data
    request(firstViewFilmstripImg).pipe(fs.createWriteStream(datapointDir + path.sep + 'fv_filmstrip.png'));
    response.run.firstView.pages.fvFilmstrip = datapointPath + path.sep + 'fv_filmstrip.png';

    //grab the rv filmstrip image, and save a url to the filmstrip page in the data
    request(repeatFilmstripImg).pipe(fs.createWriteStream(datapointDir + path.sep + 'rv_filmstrip.png'));
    response.run.firstView.pages.rvFilmstrip = datapointPath + path.sep + 'rv_filmstrip.png';

    //save the json test data to a file
    jf.writeFile(datapointDir + path.sep + 'results.json', results, function(err) {
      if (err) console.error(err);
    });

    debug('Saved results for ' + results.response.testUrl);

  },

  getDatapoint: function getTestData_anon(suiteId, testId, datapointId) {
  
    var tests = fs.readdirSync(resultsPath + suiteId + path.sep + testId).filter(junk.not)
      , testIndex = tests.indexOf(datapointId)
      , testDir = resultsPath + suiteId + path.sep + testId + path.sep + tests[testIndex] + path.sep
      , baseTestPath = path.sep + 'suite' + path.sep + suiteId + path.sep + testId + path.sep
      , publicPath = path.sep + 'results' + path.sep + suiteId + path.sep + testId + path.sep + tests[testIndex] + path.sep
      , data = {}
      , resourceBase = '/results/' + suiteId + '/' + testId + '/' + datapointId + '/'
      ;
    
    data = {
      datapointId: datapointId,
      suiteId: suiteId,
      testId: testId,
      firstViewWaterfallImage: resourceBase + 'fv_waterfall.png',
      repeatViewWaterfallImage: resourceBase + 'rv_waterfall.png',
      firstViewfilmstripImage: resourceBase + 'fv_filmstrip.png',
      repeatViewFilmstripImage: resourceBase + 'rv_filmstrip.png',
      jsonLink: resourceBase + 'results.json',
      testResults: jf.readFileSync(testDir + 'results.json'),
      testDate: tests[testIndex],
      nextTest: testIndex < tests.length - 1 ?  {suiteId: suiteId, testId: testId, datapointId: tests[testIndex + 1]} : null,
      prevTest: testIndex > 0 ? {suiteId: suiteId, testId: testId, datapointId: tests[testIndex - 1]} : null,
    };
    
    return data;
  },

  /*
   * Return the data for a suite of tests
   */
  getSuite: function (suiteName) {
    debug("getting suite: " + suiteName);

    var suiteDir = resultsPath + suiteName
      , testDirs = fs.readdirSync(suiteDir).filter(junk.not)
      ;

    suite = {
      suite: suiteName,
      tests: testDirs
    };

    return suite;
  },

  getSuiteTest: function getChartData_anon (suiteName, testName) {
    
    debug("getting suite test: " + suiteName + ' - ' + testName);

    suiteTests = {
      suite: suiteName,
      testName: testName,
      datapoints: []
    };

    var testDirBase = resultsPath + suiteName + path.sep + testName;

    testDirs = fs.readdirSync(testDirBase).filter(junk.not);
    testDirs.forEach(function(testDir){
      var dirVals = testDir.split('-')
        , dirDate = new Date(dirVals[0], (dirVals[1] - 1), dirVals[2], dirVals[3], dirVals[4], dirVals[5])
        , datapoint = {
            id: testDir,
            //sync to keep the array of results in order. lil lazy/slow
            data: jf.readFileSync(testDirBase + path.sep + testDir + path.sep + 'results.json').response.data
          }
        ;
      suiteTests.datapoints.push(datapoint);
    });
    return suiteTests;
  }
};


module.exports = dataStore;

/*
 * An overly verbose debugged method for helping
 * with occasional network service inconsistencies
 */
function goodTestResults (results) {
    var msg = 'goodTestResults suceeeded'
      , res = true
      ;

    if (!results.response.data.run) {
      msg = 'no results.response.data.run';
      res = false;
    } else if (!results.response.data.run.firstView) {
      msg = 'no results.response.data.run.firstView';
      res = false;
    } else if (!results.response.data.run.repeatView) {
      msg = 'no results.response.data.run.repeatView';
      res = false;
    } else if (!results.response.data.run.firstView.images) {
      msg = 'no results.response.data.run.firstView.images';
      res = false;
    } else if (!results.response.data.run.repeatView.images) {
      msg = 'no results.response.data.run.repeatView.images';
      res = false;
    } else if (!results.response.data.run.firstView.results.SpeedIndex) {
      msg = 'no results.response.data.run.firstView.results.SpeedIndex';
      res = false;
    } else if (!results.response.data.run.repeatView.results.SpeedIndex) {
      msg = 'no results.response.data.run.repeatView.results.SpeedIndex';
      res = false;
    }

    debug(msg);
    debug(results.response.data.run);
    return res;
}<commit_msg>no longer need to load these files, getting them from the json due to potential strain on WPT generating lots of filmstrips<commit_after>/*
 * Given a test result data, save it.
 * Asked for a test result, retrieve it.
 */

var debug   = require('debug')('wptc:data_store_fs');
var moment  = require('moment');
var request = require('request');
var mkdirp  = require('mkdirp');
var fs      = require('fs');
var path    = require('path');
var jf      = require('jsonfile');
var os      = require('os');
var junk    = require('junk');

//this should probably come from config
var resultsPath = 'public' + path.sep + 'results' + path.sep;

dataStore = {

  /*
   * Given a test and some results, save to the file system:
   * the json
   * filmstrip image
   * waterfall image
   * for the intial view and the refresh view.
   */
  saveDatapoint: function saveDatapointTest_anon(test, results) {

    //the internet is flaky sometimes
    if (!goodTestResults(results)){
      console.error('Test Died on: ' + results.response.data.testUrl);
      return;
    }

    var response = results.response.data
      , datePath = moment().format('YYYY-MM-DD-HH-mm-ss')
      , datapointPath = test.suitePathName + path.sep + test.testId + path.sep + datePath
      , datapointDir  = resultsPath + datapointPath
      ;

    //make the new dir structure
    mkdirp.sync(datapointDir);

    //save the json test data to a file
    jf.writeFile(datapointDir + path.sep + 'results.json', results, function(err) {
      if (err) console.error(err);
    });

    debug('Saved results for ' + results.response.testUrl);

  },

  getDatapoint: function getTestData_anon(suiteId, testId, datapointId) {
  
    var tests = fs.readdirSync(resultsPath + suiteId + path.sep + testId).filter(junk.not)
      , testIndex = tests.indexOf(datapointId)
      , testDir = resultsPath + suiteId + path.sep + testId + path.sep + tests[testIndex] + path.sep
      , baseTestPath = path.sep + 'suite' + path.sep + suiteId + path.sep + testId + path.sep
      , publicPath = path.sep + 'results' + path.sep + suiteId + path.sep + testId + path.sep + tests[testIndex] + path.sep
      , data = {}
      , resourceBase = '/results/' + suiteId + '/' + testId + '/' + datapointId + '/'
      ;
    
    data = {
      datapointId: datapointId,
      suiteId: suiteId,
      testId: testId,
      firstViewWaterfallImage: resourceBase + 'fv_waterfall.png',
      repeatViewWaterfallImage: resourceBase + 'rv_waterfall.png',
      firstViewfilmstripImage: resourceBase + 'fv_filmstrip.png',
      repeatViewFilmstripImage: resourceBase + 'rv_filmstrip.png',
      jsonLink: resourceBase + 'results.json',
      testResults: jf.readFileSync(testDir + 'results.json'),
      testDate: tests[testIndex],
      nextTest: testIndex < tests.length - 1 ?  {suiteId: suiteId, testId: testId, datapointId: tests[testIndex + 1]} : null,
      prevTest: testIndex > 0 ? {suiteId: suiteId, testId: testId, datapointId: tests[testIndex - 1]} : null,
    };
    
    return data;
  },

  /*
   * Return the data for a suite of tests
   */
  getSuite: function (suiteName) {
    debug("getting suite: " + suiteName);

    var suiteDir = resultsPath + suiteName
      , testDirs = fs.readdirSync(suiteDir).filter(junk.not)
      ;

    suite = {
      suite: suiteName,
      tests: testDirs
    };

    return suite;
  },

  getSuiteTest: function getChartData_anon (suiteName, testName) {
    
    debug("getting suite test: " + suiteName + ' - ' + testName);

    suiteTests = {
      suite: suiteName,
      testName: testName,
      datapoints: []
    };

    var testDirBase = resultsPath + suiteName + path.sep + testName;

    testDirs = fs.readdirSync(testDirBase).filter(junk.not);
    testDirs.forEach(function(testDir){
      var dirVals = testDir.split('-')
        , dirDate = new Date(dirVals[0], (dirVals[1] - 1), dirVals[2], dirVals[3], dirVals[4], dirVals[5])
        , datapoint = {
            id: testDir,
            //sync to keep the array of results in order. lil lazy/slow
            data: jf.readFileSync(testDirBase + path.sep + testDir + path.sep + 'results.json').response.data
          }
        ;
      suiteTests.datapoints.push(datapoint);
    });
    return suiteTests;
  }
};


module.exports = dataStore;

/*
 * An overly verbose debugged method for helping
 * with occasional network service inconsistencies
 */
function goodTestResults (results) {
    var msg = 'goodTestResults suceeeded'
      , res = true
      ;

    if (!results.response.data.run) {
      msg = 'no results.response.data.run';
      res = false;
    } else if (!results.response.data.run.firstView) {
      msg = 'no results.response.data.run.firstView';
      res = false;
    } else if (!results.response.data.run.repeatView) {
      msg = 'no results.response.data.run.repeatView';
      res = false;
    } else if (!results.response.data.run.firstView.images) {
      msg = 'no results.response.data.run.firstView.images';
      res = false;
    } else if (!results.response.data.run.repeatView.images) {
      msg = 'no results.response.data.run.repeatView.images';
      res = false;
    } else if (!results.response.data.run.firstView.results.SpeedIndex) {
      msg = 'no results.response.data.run.firstView.results.SpeedIndex';
      res = false;
    } else if (!results.response.data.run.repeatView.results.SpeedIndex) {
      msg = 'no results.response.data.run.repeatView.results.SpeedIndex';
      res = false;
    }

    debug(msg);
    debug(results.response.data.run);
    return res;
}
{'fraction_non_alphanumeric': 0.13020166453265045, 'fraction_numerical': 0.004321382842509603, 'mean_word_length': 2.9926517571884985, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7176108', 'n_tokens_mistral': 4098, 'n_tokens_neox': 3987, 'n_words': 907}