<commit_before>module.exports = function(line, format) {

  var match, times, expanded, i;

  // Expand groups: A2,3(1X,I2) => A2,1X,I2,1X,I2,1X,I2
  while ( ( match = /(\d+)\((.*)\)/.exec(format) ) !== null ) {

    times = parseInt(match[1], 10);

    expanded = '';

    for (i = 0; i < times; i++) {
      expanded += match[2];
      if ( i !== times - 1 ) {
        expanded += ',';
      }
    }

    format = format.replace(match[0], expanded);

    if ( format.slice(-1) === ',' ) {
      format = format.slice(0, -1);
    }

  }

  // Expand repetitions: 3F6.4 => F6.4,F6.4,F6.4
  while ( ( match = /(\d+)([AFIX]{1}[\d\.]*)/.exec(format) ) !== null ) {

    times = parseInt(match[1], 10);

    expanded = '';

    for (i = 0; i < times; i++) {
      expanded += match[2];
      if ( i !== times - 1 ) {
        expanded += ',';
      }
    }

    format = format.replace(match[0], expanded);

    if ( format.slice(-1) === ',' ) {
      format = format.slice(0, -1);
    }

  }

  var elements = format.split(',');

  var startIndex = 0;

  var result = elements.map( function(e) {

    var match = /(\d+)?([AFEIX]{1})((\d+){1}[.\d]*)?/.exec(e);

    var repetitions = parseInt(match[1], 10) || 1,
        type = match[2],
        width = parseInt(match[4], 10) || 1;

    var sliced = line.slice(startIndex, startIndex+width);

    // console.log(repetitions, type, width, '=' + sliced + '=');

    var foo = null;

    switch ( type ) {

      case 'A':
        foo = sliced;
        break;

      case 'F':
      case 'E':

        if ( sliced.trim().length !== 0 ) {
          foo = parseFloat( sliced );
        }
        break;

      case 'I':

        if ( sliced.trim().length !== 0 ) {
          foo = parseInt( sliced, 10 );
        }
        break;

      case 'X':
        foo = '';
        break;

    }

    startIndex += width;

    return foo;<commit_msg>Allow D as a stand in for E,F<commit_after>module.exports = function(line, format) {

  var match, times, expanded, i;

  // Expand groups: A2,3(1X,I2) => A2,1X,I2,1X,I2,1X,I2
  while ( ( match = /(\d+)\((.*)\)/.exec(format) ) !== null ) {

    times = parseInt(match[1], 10);

    expanded = '';

    for (i = 0; i < times; i++) {
      expanded += match[2];
      if ( i !== times - 1 ) {
        expanded += ',';
      }
    }

    format = format.replace(match[0], expanded);

    if ( format.slice(-1) === ',' ) {
      format = format.slice(0, -1);
    }

  }

  // Expand repetitions: 3F6.4 => F6.4,F6.4,F6.4
  while ( ( match = /(\d+)([ADEFIX]{1}[\d\.]*)/.exec(format) ) !== null ) {

    times = parseInt(match[1], 10);

    expanded = '';

    for (i = 0; i < times; i++) {
      expanded += match[2];
      if ( i !== times - 1 ) {
        expanded += ',';
      }
    }

    format = format.replace(match[0], expanded);

    if ( format.slice(-1) === ',' ) {
      format = format.slice(0, -1);
    }

  }

  var elements = format.split(',');

  var startIndex = 0;

  var result = elements.map( function(e) {

    var match = /(\d+)?([ADEFIX]{1})((\d+){1}[.\d]*)?/.exec(e);

    var repetitions = parseInt(match[1], 10) || 1,
        type = match[2],
        width = parseInt(match[4], 10) || 1;

    var sliced = line.slice(startIndex, startIndex+width);

    // console.log(repetitions, type, width, '=' + sliced + '=');

    var foo = null;

    switch ( type ) {

      case 'A':
        foo = sliced;
        break;

      case 'D':
      case 'E':
      case 'F':

        if ( sliced.trim().length !== 0 ) {
          foo = parseFloat( sliced.replace(/[DF]/i, 'e') );
        }
        break;

      case 'I':

        if ( sliced.trim().length !== 0 ) {
          foo = parseInt( sliced, 10 );
        }
        break;

      case 'X':
        foo = '';
        break;

    }

    startIndex += width;

    return foo;
{'fraction_non_alphanumeric': 0.21041612143417954, 'fraction_numerical': 0.02983512169589113, 'mean_word_length': 1.8522388059701493, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5156217', 'n_tokens_mistral': 1474, 'n_tokens_neox': 1368, 'n_words': 299}