<commit_before>// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:io' as io;
import 'dart:convert';

Map<String, Map<String, int>> typeIndex = {};

/// Export the type usage distribution from the results of the map
main(List<String> args) {
  if (args.length != 1) {
    print ("Usage: typeusage_reducer [path_to_features]");
    io.exit(1);
  }

  List<io.FileSystemEntity> fses =
    new io.Directory(args[0]).listSync(recursive: true);

  for (var fse in fses) {
    if (!fse.path.endsWith(".gz")) continue;

    var f = fse as io.File;
    var data = JSON.decode(f.readAsStringSync());
    for (var dataPath in data['result'].keys) {

      var fileName =
        f.path.split("2016-01").last + dataPath.split("data_working").last;
      var discoveryFeatures = data['result'][dataPath]['discovery_features'];
      for (var typeUsageKV in discoveryFeatures) {
        var type = typeUsageKV['TypeUsage'];
        typeIndex.putIfAbsent(type, () => {}).putIfAbsent(fileName, () => 0);
        typeIndex[type][fileName]++;
      }<commit_msg>Add TODO<commit_after>// Copyright (c) 2016, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:io' as io;
import 'dart:convert';

Map<String, Map<String, int>> typeIndex = {};

/// Export the type usage distribution from the results of the map
main(List<String> args) {
  if (args.length != 1) {
    print ("Usage: typeusage_reducer [path_to_features]");
    io.exit(1);
  }

  List<io.FileSystemEntity> fses =
    new io.Directory(args[0]).listSync(recursive: true);

  for (var fse in fses) {
    if (!fse.path.endsWith(".gz")) continue;

    var f = fse as io.File;
    var data = JSON.decode(f.readAsStringSync());
    for (var dataPath in data['result'].keys) {

      // TODO: This form of path mangling should be replaced with something
      // that reads the input field of the results to give a stable mapping
      // to their source

      var fileName =
        f.path.split("2016-01").last + dataPath.split("data_working").last;
      var discoveryFeatures = data['result'][dataPath]['discovery_features'];
      for (var typeUsageKV in discoveryFeatures) {
        var type = typeUsageKV['TypeUsage'];
        typeIndex.putIfAbsent(type, () => {}).putIfAbsent(fileName, () => 0);
        typeIndex[type][fileName]++;
      }
{'fraction_non_alphanumeric': 0.1355479190530737, 'fraction_numerical': 0.010691103474608629, 'mean_word_length': 3.388609715242881, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3821014', 'n_tokens_mistral': 834, 'n_tokens_neox': 802, 'n_words': 254}