<commit_before>drop table if exists downloads;
create table downloads(year int, month int, country_from varchar(255), country_to varchar(255), type char(1), count int);

load data infile '/tmp/aggregated_download_info.csv' into table downloads fields terminated by ';' lines terminated by '\n';

delete from downloads where country_to in (select country_to from (select country_to, SUM(count) as cnt from downloads group by country_to having cnt < 1000 order by cnt asc limit 100) as z);

delete from downloads where country_from in ('None', 'Unknown');

update downloads set country_from='USA'                          where country_from='United States';
update downloads set country_from='UK'	                         where country_from='United Kingdom';
update downloads set country_from='Moldova'                      where country_from='Republic of Moldova';
update downloads set country_from='Lithuania'                    where country_from='Republic of Lithuania';
update downloads set country_from='Burma'                        where country_from='Myanmar [Burma]';
update downloads set country_from='China'                        where country_from='Hong Kong';
update downloads set country_from='China'                        where country_from='Macao';
update downloads set country_from='South Korea'                  where country_from='Republic of Korea';
update downloads set country_from='Jordan'                       where country_from='Hashemite Kingdom of Jordan';
update downloads set country_from='Italy'                        where country_from='San Marino';
update downloads set country_from='Micronesia'                   where country_from='Federated States of Micronesia';
update downloads set country_from='United States Virgin Islands' where country_from='U.S. Virgin Islands';<commit_msg>Fixed whitespace.<commit_after>drop table if exists downloads;
create table downloads(year int, month int, country_from varchar(255), country_to varchar(255), type char(1), count int);

load data infile '/tmp/aggregated_download_info.csv' into table downloads fields terminated by ';' lines terminated by '\n';

delete from downloads where country_to in (select country_to from (select country_to, SUM(count) as cnt from downloads group by country_to having cnt < 1000 order by cnt asc limit 100) as z);

delete from downloads where country_from in ('None', 'Unknown');

update downloads set country_from='USA'                          where country_from='United States';
update downloads set country_from='UK'                           where country_from='United Kingdom';
update downloads set country_from='Moldova'                      where country_from='Republic of Moldova';
update downloads set country_from='Lithuania'                    where country_from='Republic of Lithuania';
update downloads set country_from='Burma'                        where country_from='Myanmar [Burma]';
update downloads set country_from='China'                        where country_from='Hong Kong';
update downloads set country_from='China'                        where country_from='Macao';
update downloads set country_from='South Korea'                  where country_from='Republic of Korea';
update downloads set country_from='Jordan'                       where country_from='Hashemite Kingdom of Jordan';
update downloads set country_from='Italy'                        where country_from='San Marino';
update downloads set country_from='Micronesia'                   where country_from='Federated States of Micronesia';
update downloads set country_from='United States Virgin Islands' where country_from='U.S. Virgin Islands';
{'fraction_non_alphanumeric': 0.07434842249657064, 'fraction_numerical': 0.007681755829903978, 'mean_word_length': 3.361244019138756, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4257698', 'n_tokens_mistral': 954, 'n_tokens_neox': 832, 'n_words': 332}