<commit_before>'use strict';

/* Controllers */


function MyCtrl1() {}
MyCtrl1.$inject = [];


function MyCtrl2() {
}
MyCtrl2.$inject = [];

function EdidCtrl($scope) {
  // Populated test EDID
  /*
  $scope.originalEdid = 
"00 FF FF FF FF FF FF 00 4D D9 FA 06 00 00 00 00 \n" +
"2D 0C 01 03 90 1F 11 00 EA A8 E0 99 57 4B 92 25 \n" +
"1C 50 54 00 00 00 01 01 01 01 01 01 01 01 01 01 \n" +
"01 01 01 01 01 01 40 38 40 F6 63 84 13 30 52 52 \n" +
"66 50 36 AE 10 00 00 18 00 00 00 FC 00 4E 76 69 \n" +
"64 69 61 20 44 65 66 61 75 6C 00 00 00 FC 00 74 \n" +
"20 46 6C 61 74 20 50 61 6E 65 6C 00 00 00 00 FD \n" +
"00 00 3D 1D 38 0B 00 00 20 20 20 20 20 00 00 48";
*/
/* New Test Edid
 * http://www.avsforum.com/t/1253912/first-hdmi-1-4-edid-report
 */
$scope.originalEdid =  "00,FF,FF,FF,FF,FF,FF,00,4C,2D,9B,06,01,00,00,00, \n" +
                       "33,13,01,03,80,59,32,78,0A,EE,91,A3,54,4C,99,26, \n" +
                       "0F,50,54,BD,EF,80,71,4F,81,00,81,40,81,80,95,00, \n" +
                       "95,0F,B3,00,A9,40,02,3A,80,18,71,38,2D,40,58,2C, \n" +
                       "45,00,A0,5A,00,00,00,1E,66,21,50,B0,51,00,1B,30, \n" +
                       "40,70,36,00,A0,5A,00,00,00,1E,00,00,00,FD,00,18, \n" +
                       "4B,1A,51,17,00,0A,20,20,20,20,20,20,00,00,00,FC, \n" +
                       "00,53,41,4D,53,55,4E,47,0A,20,20,20,20,20,01,7F, \n" +
                       "02,03,2E,F1,4B,90,1F,04,13,05,14,03,12,20,21,22, \n" +
                       "23,09,07,07,83,01,00,00,E2,00,0F,E3,05,03,01,6E, \n" +
                       "03,0C,00,20,00,B8,2D,20,D0,04,01,40,07,3F,02,3A, \n" +
                       "80,D0,72,38,2D,40,10,2C,45,80,A0,5A,00,00,00,1E, \n" +
                       "01,1D,00,BC,52,D0,1E,20,B8,28,55,40,A0,5A,00,00, \n" +
                       "00,1E,01,1D,80,D0,72,1C,16,20,10,2C,25,80,A0,5A, \n" +
                       "00,00,00,9E,00,00,00,00,00,00,00,00,00,00,00,00, \n" +
                       "00,00,00,00,00,00,00,00,00,00,00,00,00,00,00,74";
    
    $scope.edid = new Edid();

    $scope.validChecksums = new Array();
    $scope.dtdBit2Text = new Array();
    $scope.dtdBit1Text = new Array();
    
    //*********************************
    // Start Actually Parsing
    $scope.parseEdid = function() {
    // Remove 0x, and Commas
    $scope.scrubbedEdid = scrubEdid($scope.originalEdid);
    
    $scope.edid.setEdidData($scope.scrubbedEdid);
    $scope.edid.parse();
    
    $scope.updateChecksums();
    $scope.edidHeaderInfo = 'partials/edidHeaderInfo.html';
    if($scope.edid.bdp.digitalInput == true)
    {
      $scope.bdpPage = 'partials/bdp/digital.html';
    }
    else
    {
      $scope.bdpPage = 'partials/bdp/analog.html';
    }
    $scope.chromacityCoords = 'partials/chromacityCoords.html';
    $scope.timingBitmap = 'partials/timingBitmap.html';
    $scope.standardTimingInfo = 'partials/standardTimingInformation.html';
    $scope.stdDisplayModes = 'partials/stdDisplayModes.html'
    $scope.updateDtdBit2Text();
    $scope.updateDtdBit1Text();
    $scope.detailedTimingDescriptors = 'partials/detailedTimingDescriptors.html'
  };
  $scope.updateChecksums = function()  
  {
    var numberBlocks = $scope.edid.edidData.length / $scope.edid.EDID_BLOCK_LENGTH;
    for(var index = 0; index < numberBlocks; index++)
    {
      $scope.validChecksums[index] = $scope.edid.validChecksum(index).toString().toUpperCase();
    }
  }
  $scope.isTimingBitmapSet = function(index)
  {
    var tbLength = $scope.edid.establishedTimingBitmaps.length;
    var msb = 0x800000;
    var isSet = ($scope.edid.timingBitmap & (msb >> index))?true:false;
    return isSet;
  }
  $scope.updateDtdBit2Text = function()
  {
    for(var index = 0; index < $scope.edid.dtds.length; index++)
    {
      var bit2Text = new String();
      if($scope.edid.dtds[index].syncType == 
                $scope.edid.syncTypeEnum.DIGITAL_SEPARATE)
      {
        bit2Text = "Vertical Sync Polarity: ";
        bit2Text += $scope.edid.dtds[index].vSyncPolarity;
      }
      else
      {
        bit2Text = "Vertical Sync Serrated: ";
        bit2Text += $scope.edid.dtds[index].vSyncSerrated;
      } 
      $scope.dtdBit2Text[index] = bit2Text;
    }
  }
  $scope.updateDtdBit1Text = function()
  {
    for(var index = 0; index < $scope.edid.dtds.length; index++)
    {
      var bit1Text = new String();
      if(($scope.edid.dtds[index].syncType == 
                $scope.edid.syncTypeEnum.ANALOG_COMPOSITE) ||
                ($scope.edid.dtds[index].syncType == 
                $scope.edid.syncTypeEnum.BIPOLAR_ANALOG_COMPOSITE))
      {
        bit1Text = "Sync on all 3 RGB lines: ";
        bit1Text += $scope.edid.dtds[index].syncAllRGBLines;
      }
      else
      {
        bit1Text = "HSync polarity: ";
        bit1Text += $scope.edid.dtds[index].hSyncPolarity;
      } 
      $scope.dtdBit1Text[index] = bit1Text;
    }
  }
}

function scrubEdid(edid)
{
  var scrubbedEdid;
  
  // Remove commas, replace with spaces
  scrubbedEdid = edid.replace(/,/g,' ');
  // Remove 0x for hex
  scrubbedEdid = scrubbedEdid.replace(/0x/g,' ');
  // Remove multiple spaces
  scrubbedEdid = scrubbedEdid.replace( / +/g,' ');
  // Make hex upper case
  scrubbedEdid = scrubbedEdid.toUpperCase(); 

  return scrubbedEdid; 
}
/* Test Data
 * 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F
 * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F

 * Internet Edid
00 FF FF FF FF FF FF 00 3A C4 00 04 00 00 00 00
2D 0C 01 03 80 20 18 00 EA A8 E0 99 57 4B 92 25
1C 50 54 00 00 00 01 01 01 01 01 01 01 01 01 01
01 01 01 01 01 01 48 3F 40 30 62 B0 32 40 4C C0
13 00 42 F3 10 00 00 1E 00 00 00 FC 00 4E 76 69
64 69 61 20 44 65 66 61 75 6C 00 00 00 FC 00 74
20 46 6C 61 74 20 50 61 6E 65 6C 00 00 00 00 FD
00 00 3C 1D 4C 11 00 00 20 20 20 20 20 00 00 9C

 * <NAME>
00 FF FF FF FF FF FF 00 4D D9 FA 06 00 00 00 00
2D 0C 01 03 90 1F 11 00 EA A8 E0 99 57 4B 92 25
1C 50 54 00 00 00 01 01 01 01 01 01 01 01 01 01
01 01 01 01 01 01 40 38 40 F6 63 84 13 30 52 52
66 50 36 AE 10 00 00 18 00 00 00 FC 00 4E 76 69
64 69 61 20 44 65 66 61 75 6C 00 00 00 FC 00 74
20 46 6C 61 74 20 50 61 6E 65 6C 00 00 00 00 FD
00 00 3D 1D 38 0B 00 00 20 20 20 20 20 00 00 48
 * 
 */<commit_msg>Improve EDID scrubber<commit_after>'use strict';

/* Controllers */


function MyCtrl1() {}
MyCtrl1.$inject = [];


function MyCtrl2() {
}
MyCtrl2.$inject = [];

function EdidCtrl($scope) {
  // Populated test EDID
  /*
  $scope.originalEdid = 
"00 FF FF FF FF FF FF 00 4D D9 FA 06 00 00 00 00 \n" +
"2D 0C 01 03 90 1F 11 00 EA A8 E0 99 57 4B 92 25 \n" +
"1C 50 54 00 00 00 01 01 01 01 01 01 01 01 01 01 \n" +
"01 01 01 01 01 01 40 38 40 F6 63 84 13 30 52 52 \n" +
"66 50 36 AE 10 00 00 18 00 00 00 FC 00 4E 76 69 \n" +
"64 69 61 20 44 65 66 61 75 6C 00 00 00 FC 00 74 \n" +
"20 46 6C 61 74 20 50 61 6E 65 6C 00 00 00 00 FD \n" +
"00 00 3D 1D 38 0B 00 00 20 20 20 20 20 00 00 48";
*/
/* New Test Edid
 * http://www.avsforum.com/t/1253912/first-hdmi-1-4-edid-report
 */
/*
$scope.originalEdid =  "00,FF,FF,FF,FF,FF,FF,00,4C,2D,9B,06,01,00,00,00, \n" +
                       "33,13,01,03,80,59,32,78,0A,EE,91,A3,54,4C,99,26, \n" +
                       "0F,50,54,BD,EF,80,71,4F,81,00,81,40,81,80,95,00, \n" +
                       "95,0F,B3,00,A9,40,02,3A,80,18,71,38,2D,40,58,2C, \n" +
                       "45,00,A0,5A,00,00,00,1E,66,21,50,B0,51,00,1B,30, \n" +
                       "40,70,36,00,A0,5A,00,00,00,1E,00,00,00,FD,00,18, \n" +
                       "4B,1A,51,17,00,0A,20,20,20,20,20,20,00,00,00,FC, \n" +
                       "00,53,41,4D,53,55,4E,47,0A,20,20,20,20,20,01,7F, \n" +
                       "02,03,2E,F1,4B,90,1F,04,13,05,14,03,12,20,21,22, \n" +
                       "23,09,07,07,83,01,00,00,E2,00,0F,E3,05,03,01,6E, \n" +
                       "03,0C,00,20,00,B8,2D,20,D0,04,01,40,07,3F,02,3A, \n" +
                       "80,D0,72,38,2D,40,10,2C,45,80,A0,5A,00,00,00,1E, \n" +
                       "01,1D,00,BC,52,D0,1E,20,B8,28,55,40,A0,5A,00,00, \n" +
                       "00,1E,01,1D,80,D0,72,1C,16,20,10,2C,25,80,A0,5A, \n" +
                       "00,00,00,9E,00,00,00,00,00,00,00,00,00,00,00,00, \n" +
                       "00,00,00,00,00,00,00,00,00,00,00,00,00,00,00,74";
 */   
    $scope.edid = new Edid();

    $scope.validChecksums = new Array();
    $scope.dtdBit2Text = new Array();
    $scope.dtdBit1Text = new Array();
    
    //*********************************
    // Start Actually Parsing
    $scope.parseEdid = function() {
    // Remove 0x, and Commas
    $scope.scrubbedEdid = scrubEdid($scope.originalEdid);
    
    $scope.edid.setEdidData($scope.scrubbedEdid);
    $scope.edid.parse();
    
    $scope.updateChecksums();
    $scope.edidHeaderInfo = 'partials/edidHeaderInfo.html';
    if($scope.edid.bdp.digitalInput == true)
    {
      $scope.bdpPage = 'partials/bdp/digital.html';
    }
    else
    {
      $scope.bdpPage = 'partials/bdp/analog.html';
    }
    $scope.chromacityCoords = 'partials/chromacityCoords.html';
    $scope.timingBitmap = 'partials/timingBitmap.html';
    $scope.standardTimingInfo = 'partials/standardTimingInformation.html';
    $scope.stdDisplayModes = 'partials/stdDisplayModes.html'
    $scope.updateDtdBit2Text();
    $scope.updateDtdBit1Text();
    $scope.detailedTimingDescriptors = 'partials/detailedTimingDescriptors.html'
  };
  $scope.updateChecksums = function()  
  {
    var numberBlocks = $scope.edid.edidData.length / $scope.edid.EDID_BLOCK_LENGTH;
    for(var index = 0; index < numberBlocks; index++)
    {
      $scope.validChecksums[index] = $scope.edid.validChecksum(index).toString().toUpperCase();
    }
  }
  $scope.isTimingBitmapSet = function(index)
  {
    var tbLength = $scope.edid.establishedTimingBitmaps.length;
    var msb = 0x800000;
    var isSet = ($scope.edid.timingBitmap & (msb >> index))?true:false;
    return isSet;
  }
  $scope.updateDtdBit2Text = function()
  {
    for(var index = 0; index < $scope.edid.dtds.length; index++)
    {
      var bit2Text = new String();
      if($scope.edid.dtds[index].syncType == 
                $scope.edid.syncTypeEnum.DIGITAL_SEPARATE)
      {
        bit2Text = "Vertical Sync Polarity: ";
        bit2Text += $scope.edid.dtds[index].vSyncPolarity;
      }
      else
      {
        bit2Text = "Vertical Sync Serrated: ";
        bit2Text += $scope.edid.dtds[index].vSyncSerrated;
      } 
      $scope.dtdBit2Text[index] = bit2Text;
    }
  }
  $scope.updateDtdBit1Text = function()
  {
    for(var index = 0; index < $scope.edid.dtds.length; index++)
    {
      var bit1Text = new String();
      if(($scope.edid.dtds[index].syncType == 
                $scope.edid.syncTypeEnum.ANALOG_COMPOSITE) ||
                ($scope.edid.dtds[index].syncType == 
                $scope.edid.syncTypeEnum.BIPOLAR_ANALOG_COMPOSITE))
      {
        bit1Text = "Sync on all 3 RGB lines: ";
        bit1Text += $scope.edid.dtds[index].syncAllRGBLines;
      }
      else
      {
        bit1Text = "HSync polarity: ";
        bit1Text += $scope.edid.dtds[index].hSyncPolarity;
      } 
      $scope.dtdBit1Text[index] = bit1Text;
    }
  }
}

function scrubEdid(edid)
{
  var scrubbedEdid;
  
  // Remove commas, replace with spaces
  scrubbedEdid = edid.replace(/,/g,' ');
  // Remove 0x for hex
  scrubbedEdid = scrubbedEdid.replace(/0x/g,' ');
  // Replace Tabs
  scrubbedEdid = scrubbedEdid.replace('\t',' ');
  // Remove all line returns
  scrubbedEdid = scrubbedEdid.replace(/(\r\n|\n|\r)/gm,' ');
  // Remove multiple spaces
  scrubbedEdid = scrubbedEdid.replace( / +/g,' ');
  // Make hex upper case
  scrubbedEdid = scrubbedEdid.toUpperCase();
  return scrubbedEdid; 
}
/* Test Data
 * 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 0x09 0x0A 0x0B 0x0C 0x0D 0x0E 0x0F
 * 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F

 * Internet Edid
00 FF FF FF FF FF FF 00 3A C4 00 04 00 00 00 00
2D 0C 01 03 80 20 18 00 EA A8 E0 99 57 4B 92 25
1C 50 54 00 00 00 01 01 01 01 01 01 01 01 01 01
01 01 01 01 01 01 48 3F 40 30 62 B0 32 40 4C C0
13 00 42 F3 10 00 00 1E 00 00 00 FC 00 4E 76 69
64 69 61 20 44 65 66 61 75 6C 00 00 00 FC 00 74
20 46 6C 61 74 20 50 61 6E 65 6C 00 00 00 00 FD
00 00 3C 1D 4C 11 00 00 20 20 20 20 20 00 00 9C

 * <NAME>
00 FF FF FF FF FF FF 00 4D D9 FA 06 00 00 00 00
2D 0C 01 03 90 1F 11 00 EA A8 E0 99 57 4B 92 25
1C 50 54 00 00 00 01 01 01 01 01 01 01 01 01 01
01 01 01 01 01 01 40 38 40 F6 63 84 13 30 52 52
66 50 36 AE 10 00 00 18 00 00 00 FC 00 4E 76 69
64 69 61 20 44 65 66 61 75 6C 00 00 00 FC 00 74
20 46 6C 61 74 20 50 61 6E 65 6C 00 00 00 00 FD
00 00 3D 1D 38 0B 00 00 20 20 20 20 20 00 00 48
 * 
 */
{'fraction_non_alphanumeric': 0.14768627450980393, 'fraction_numerical': 0.18776470588235294, 'mean_word_length': 2.4640043466449333, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1962040', 'n_tokens_mistral': 7176, 'n_tokens_neox': 5325, 'n_words': 1308}