<commit_before>import os
import sys
import math
import json


os.chdir(os.path.abspath(os.path.dirname(__file__)))


def convert_countries():
    countries = {}

    with open('raw/countryInfo.txt', 'rb') as f:
        for line in f:
            line = line.decode('utf-8').strip().split('\t')
            if not line or line[0][:1] == '#':
                continue

            country_code = line[0]
            country = line[4]
            capital = line[5]
            countries[country_code] = {
                'name': country,
                'capital': capital,
                'code': country_code
            }

    return countries


def convert_cities():
    cities = {}

    with open('raw/cities15000.txt', 'rb') as f:
        for line in f:
            line = line.decode('utf-8').strip().split('\t')
            if not line:
                continue

            main_name = line[2]
            country = line[8]
            state = country == 'US' and line[10] or None
            population = int(line[14])
            timezone = line[17]
            is_capital = line[7] == 'PPLC'

            city_key = ('%s/%s%s' % (country, main_name,
                state and '/' + state or '')).replace(' ', '_')
            old_city = cities.get(city_key)

            # There was already a city with that name, let the one
            # with the higher population win.
            if old_city is not None:
                if population < old_city['population']:
                    continue

            cities[city_key] = {
                'country': country,
                'state': state,
                'name': main_name,
                'timezone': timezone,
                'population': population,
                'is_capital': is_capital,
            }

    return cities


def combine_data(countries, cities, timezone_data):
    selectables = []
    timezones_found = set()

    def record_selectable(key, name, display_name, type, tz, sortinfo=None):
        selectables.append({
            'k': key,
            'n': name,
            'd': display_name,
            'z': tz,
            't': type,
            'sortinfo': sortinfo or {},
        })

    for city in cities.itervalues():
        key = \
            city['country'].lower() + ':' + \
            (city['name'] + ':' + (city['state'] or '')).rstrip(':').lower() \
                .replace(' ', '-') \
                .replace('_', '-') \
                .replace('\'', '') \
                .replace(',', '') \
                .replace('(', '') \
                .replace(')', '')
        display_parts = [countries[city['country']]['name']]
        if city['state']:
            display_parts.append(city['state'])
        display_parts.append(city['name'])
        record_selectable(key, city['name'], ', '.join(display_parts),
                          'C', city['timezone'], sortinfo={'city': city})
        timezones_found.add(city['timezone'])

    for name in timezone_data['meta']:
        if name in timezones_found or \
           not (name.lower().startswith('etc/') or not '/' in name):
            continue
        key = name.lower() \
            .replace('_', '-') \
            .replace('/', ':') \
            .replace(',', '') \
            .replace('\'', '')
        record_selectable(key, name, name.split('/', 1)[-1], 'T', name)

    def _sort_key(x):
        city = x['sortinfo'].get('city')
        name = x['n'].lower()
        importance = 0
        if city:
            if city['is_capital']:
                importance += 1
            importance += math.log(max(city['population'], 1)) / 50.0
        return -importance, name
    selectables.sort(key=_sort_key)
    for selectable in selectables:
        selectable.pop('sortinfo', None)

    return {
        'timezones': timezone_data,
        'selectables': selectables,
        'countries': dict((k, v['name']) for k, v in countries.iteritems()),
    }
<commit_msg>Fixed an issue with regular timezones<commit_after>import os
import math
import json


os.chdir(os.path.abspath(os.path.dirname(__file__)))


def convert_countries():
    countries = {}

    with open('raw/countryInfo.txt', 'rb') as f:
        for line in f:
            line = line.decode('utf-8').strip().split('\t')
            if not line or line[0][:1] == '#':
                continue

            country_code = line[0]
            country = line[4]
            capital = line[5]
            countries[country_code] = {
                'name': country,
                'capital': capital,
                'code': country_code
            }

    return countries


def convert_cities():
    cities = {}

    with open('raw/cities15000.txt', 'rb') as f:
        for line in f:
            line = line.decode('utf-8').strip().split('\t')
            if not line:
                continue

            main_name = line[2]
            country = line[8]
            state = country == 'US' and line[10] or None
            population = int(line[14])
            timezone = line[17]
            is_capital = line[7] == 'PPLC'

            city_key = ('%s/%s%s' % (country, main_name,
                state and '/' + state or '')).replace(' ', '_')
            old_city = cities.get(city_key)

            # There was already a city with that name, let the one
            # with the higher population win.
            if old_city is not None:
                if population < old_city['population']:
                    continue

            cities[city_key] = {
                'country': country,
                'state': state,
                'name': main_name,
                'timezone': timezone,
                'population': population,
                'is_capital': is_capital,
            }

    return cities


def combine_data(countries, cities, timezone_data):
    selectables = []
    timezones_found = set()

    def record_selectable(key, name, full_name, type, tz, sortinfo=None):
        selectables.append({
            'k': key,
            'n': name,
            'd': full_name,
            'z': tz,
            't': type,
            'sortinfo': sortinfo or {},
        })

    for city in cities.itervalues():
        key = \
            city['country'].lower() + ':' + \
            (city['name'] + ':' + (city['state'] or '')).rstrip(':').lower() \
                .replace(' ', '-') \
                .replace('_', '-') \
                .replace('\'', '') \
                .replace(',', '') \
                .replace('(', '') \
                .replace(')', '')
        display_parts = [countries[city['country']]['name']]
        if city['state']:
            display_parts.append(city['state'])
        display_parts.append(city['name'])
        record_selectable(key, city['name'], ', '.join(display_parts),
                          'C', city['timezone'], sortinfo={'city': city})
        timezones_found.add(city['timezone'])

    for name in timezone_data['meta']:
        if name in timezones_found or \
           not (name.lower().startswith('etc/') or not '/' in name):
            continue
        key = name.lower() \
            .replace('_', '-') \
            .replace('/', ':') \
            .replace(',', '') \
            .replace('\'', '')
        record_selectable(key, name.split('/', 1)[-1], name, 'T', name)

    def _sort_key(x):
        city = x['sortinfo'].get('city')
        name = x['n'].lower()
        importance = 0
        if city:
            if city['is_capital']:
                importance += 1
            importance += math.log(max(city['population'], 1)) / 50.0
        return -importance, name
    selectables.sort(key=_sort_key)
    for selectable in selectables:
        selectable.pop('sortinfo', None)

    return {
        'timezones': timezone_data,
        'selectables': selectables,
        'countries': dict((k, v['name']) for k, v in countries.iteritems()),
    }

{'fraction_non_alphanumeric': 0.15552454649245212, 'fraction_numerical': 0.0073576049727261195, 'mean_word_length': 1.5985497692814765, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4961360', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2242, 'n_words': 519}