<commit_before>#!/usr/bin/python
#
# ==-- process-stats-dir - summarize one or more Swift -stats-output-dirs --==#
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014-2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ==------------------------------------------------------------------------==#
#
# This file processes the contents of one or more directories generated by
# `swiftc -stats-output-dir` and emits summary data, traces etc. for analysis.

import argparse
import csv
import io
import itertools
import json
import os
import platform
import re
import sys
import time
import urllib
if sys.version_info[0] < 3:
    import urllib2
    Request = urllib2.Request
    URLOpen = urllib2.urlopen
else:
    import urllib.request, urllib.parse, urllib.error
    Request = urllib.request.Request
    URLOpen = urllib.request.urlopen
from collections import namedtuple
from operator import attrgetter

from jobstats import (list_stats_dir_profiles,
                      load_stats_dir, merge_all_jobstats)


MODULE_PAT = re.compile(r'^(\w+)\.')


def module_name_of_stat(name):
    return re.match(MODULE_PAT, name).groups()[0]
<commit_msg>test: adjust style for python linter (NFC)<commit_after>#!/usr/bin/python
#
# ==-- process-stats-dir - summarize one or more Swift -stats-output-dirs --==#
#
# This source file is part of the Swift.org open source project
#
# Copyright (c) 2014-2017 Apple Inc. and the Swift project authors
# Licensed under Apache License v2.0 with Runtime Library Exception
#
# See https://swift.org/LICENSE.txt for license information
# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ==------------------------------------------------------------------------==#
#
# This file processes the contents of one or more directories generated by
# `swiftc -stats-output-dir` and emits summary data, traces etc. for analysis.

import argparse
import csv
import io
import itertools
import json
import os
import platform
import re
import sys
import time
import urllib
from collections import namedtuple
from operator import attrgetter

from jobstats import (list_stats_dir_profiles,
                      load_stats_dir, merge_all_jobstats)

if sys.version_info[0] < 3:
    import urllib2
    Request = urllib2.Request
    URLOpen = urllib2.urlopen
else:
    import urllib.request
    import urllib.parse
    import urllib.error
    Request = urllib.request.Request
    URLOpen = urllib.request.urlopen

MODULE_PAT = re.compile(r'^(\w+)\.')


def module_name_of_stat(name):
    return re.match(MODULE_PAT, name).groups()[0]

{'fraction_non_alphanumeric': 0.13586378148279532, 'fraction_numerical': 0.011351543100390209, 'mean_word_length': 3.09288824383164, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3597000', 'n_tokens_mistral': 872, 'n_tokens_neox': 817, 'n_words': 278}