<commit_before>import 'dart:collection';
import 'dart:convert';
import 'dart:io';

import 'package:pana/src/analyzer_output.dart';
import 'package:pana/src/summary.dart';
import 'package:path/path.dart' as p;

main() async {
  var dir = new Directory("results");

  var items = dir
      .listSync()
      .where((fse) => fse is File && p.extension(fse.path) == '.json')
      .toList();

  for (File jsonFile in items) {
    _process(jsonFile.readAsStringSync());
  }
}

String _prettyString(Object o) => const JsonEncoder.withIndent(' ').convert(o);

_process(String content) {
  var output = JSON.decode(content);
  var summary = new Summary.fromJson(output);
  assert(_prettyString(summary.toJson()) == _prettyString(output));

  print(JSON.encode(new _MiniSum.fromSummary(summary)));
}

class _MiniSum {
  final String name;
  final String version;
  final int unformattedFiles;
  final Set<AnalyzerOutput> analyzerItems;
  final bool pubClean;
  final Set<String> authorDomains;

  _MiniSum._(this.name, this.version, this.unformattedFiles, this.analyzerItems,
      this.pubClean, this.authorDomains);

  factory _MiniSum.fromSummary(Summary summary) {
    var authorDomains =
        summary.pubSummary.authors.map(_domainFromAuthor).toSet();

    return new _MiniSum._(
        summary.packageName,
        summary.packageVersion.toString(),
        summary.unformattedFiles.length,
        summary.analyzerItems,
        summary.pubSummary.exitCode == 0,
        authorDomains);
  }

  Map<String, dynamic> toJson() => <String, dynamic>{
        'name': name,
        'version': version,
        'unformattedFiles': unformattedFiles,
        'analyzerErrors': _analyzerItemJson.length,
        'pubClean': pubClean,
        'authorDomains': (authorDomains.toList()..sort()).join(', ')
      };

  Iterable<Map<String, Object>> get _analyzerItemJson sync* {
    var errorTypes = new SplayTreeMap<String, int>();
    for (var a in analyzerItems) {
      if (a.type.startsWith('INFO|')) {
        continue;
      }

      var itemPath = p.split(a.file).first;
      if (const ['bin', 'lib'].contains(itemPath)) {
        errorTypes[a.type] = 1 + errorTypes.putIfAbsent(a.type, () => 0);<commit_msg>authors smarter<commit_after>import 'dart:collection';
import 'dart:convert';
import 'dart:io';

import 'package:pana/src/analyzer_output.dart';
import 'package:pana/src/summary.dart';
import 'package:path/path.dart' as p;

main() async {
  var dir = new Directory("results");

  var items = dir
      .listSync()
      .where((fse) => fse is File && p.extension(fse.path) == '.json')
      .toList();

  for (File jsonFile in items) {
    _process(jsonFile.readAsStringSync());
  }
}

String _prettyString(Object o) => const JsonEncoder.withIndent(' ').convert(o);

_process(String content) {
  var output = JSON.decode(content);
  var summary = new Summary.fromJson(output);
  assert(_prettyString(summary.toJson()) == _prettyString(output));

  print(JSON.encode(new _MiniSum.fromSummary(summary)));
}

class _MiniSum {
  final String name;
  final String version;
  final int unformattedFiles;
  final Set<AnalyzerOutput> analyzerItems;
  final bool pubClean;
  final Set<String> authorDomains;

  _MiniSum._(this.name, this.version, this.unformattedFiles, this.analyzerItems,
      this.pubClean, this.authorDomains);

  factory _MiniSum.fromSummary(Summary summary) {
    var authorDomains = new SplayTreeSet<String>.from(
        summary.pubSummary.authors.map(_domainFromAuthor));

    return new _MiniSum._(
        summary.packageName,
        summary.packageVersion.toString(),
        summary.unformattedFiles.length,
        summary.analyzerItems,
        summary.pubSummary.exitCode == 0,
        authorDomains);
  }

  Map<String, dynamic> toJson() => <String, dynamic>{
        'name': name,
        'version': version,
        'unformattedFiles': unformattedFiles,
        'analyzerErrors': _analyzerItemJson.length,
        'pubClean': pubClean,
        'authorDomains': authorDomains.join(', ')
      };

  Iterable<Map<String, Object>> get _analyzerItemJson sync* {
    var errorTypes = new SplayTreeMap<String, int>();
    for (var a in analyzerItems) {
      if (a.type.startsWith('INFO|')) {
        continue;
      }

      var itemPath = p.split(a.file).first;
      if (const ['bin', 'lib'].contains(itemPath)) {
        errorTypes[a.type] = 1 + errorTypes.putIfAbsent(a.type, () => 0);
{'fraction_non_alphanumeric': 0.1374235906724021, 'fraction_numerical': 0.0013583880461851936, 'mean_word_length': 3.5972944849115507, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7614706', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1401, 'n_words': 290}