<commit_before><?php
$path = 'data/tx';
$files = scandir($path);

foreach($files as $file) {
    if(!is_dir($file) && !preg_match('/^\./', $file)) {
        $row = 1;
        $res = '';
        $capacity = '';
        if (($handle = fopen($path . '/' . $file, "r")) !== FALSE) {
            $months = array();
            $months_list = array();
            $fh = fopen('data/tx_m/' . $file, 'wb');
            while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
                if($row == 1) {
                    fputcsv($fh, $data);
                } else {
                    $date = explode('/', $data[4]);
                    $date_parts = $date[0] . '/' . $date[2];
                    $months[$date_parts][] = $data[1];
                }
                $res = $data[0];
                $capacity = $data[2];
                $row++;
            }

            foreach($months as $key => $month) {
                $monthly_avg = round(array_sum($month) / count($month));
                $monthly_avg_pct = round(($monthly_avg / $capacity) * 100, 1);

                fputcsv($fh, array($res, $monthly_avg, $capacity, $monthly_avg_pct, $key));
            }
            echo $res . "\n";

            fclose($fh);
            fclose($handle);
        }
    }
}<commit_msg>switch to just looking for letters<commit_after><?php
$path = 'data/tx';
$files = scandir($path);

foreach($files as $file) {
    if(!is_dir($file) && !preg_match('/^\./', $file)) {
        $row = 1;
        $res = '';
        $capacity = '';
        if (($handle = fopen($path . '/' . $file, "r")) !== FALSE) {
            $months = array();
            $months_list = array();
            $fh = fopen('data/tx_m/' . $file, 'wb');
            while (($data = fgetcsv($handle, 1000, ",")) !== FALSE) {
                if($row == 1) {
                    fputcsv($fh, $data);
                } else {
                    $date = explode('/', $data[4]);
                    $date_parts = $date[0] . '/' . $date[2];
                    $months[$date_parts][] = $data[1];
                }
                $res = $data[0];
                $capacity = $data[2];
                $row++;
            }

            foreach($months as $key => $month) {
                $monthly_avg = round(array_sum($month) / count($month));
                $monthly_avg_pct = round(($monthly_avg / $capacity) * 100, 1);

                if(preg_match('/20\d{2}$/', $key) && !preg_match('/[A-Za-z]/', $key)) {
                    fputcsv($fh, array($res, $monthly_avg, $capacity, $monthly_avg_pct, $key));
                }

            }
            echo $res . "\n";

            fclose($fh);
            fclose($handle);
        }
    }
}
{'fraction_non_alphanumeric': 0.20022205773501112, 'fraction_numerical': 0.012953367875647668, 'mean_word_length': 1.381497797356828, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7342741', 'n_tokens_mistral': 917, 'n_tokens_neox': 844, 'n_words': 159}