<commit_before>#!/usr/bin/env python

import pymongo

mongo_client = pymongo.MongoClient()
db = mongo_client['eumssi_db']
col = db['content_items']


def source_map(item):
    mapping = {
        'Twitter':'Twitter',
        'Twitter-DW':'Twitter',
        'DW-de_DE':'DW article',
        'DW-en_GB':'DW article',
        'DW-es_ES':'DW article',
        'DW-fr_FR':'DW article',
        'DW-vid_EN':'DW video',
        'Eumssi-News-Crawler':'News',
        'Youtube-video-GeneralChannel':'Youtube',
        'Youtube-video-dwEnglishChannel':'DW (Youtube)',
        'Youtube-video-theguardianChannel':'Guardian (Youtube)',
        'DW-en':'DW ?',
        'DW-DE':'DW ?',
        'DW-de':'DW ?',
        'DW-EN':'DW ?',
        'DW-es':'DW ?',
        'DW-ES':'DW ?',
        'DW-FR':'DW ?'
    }
    new_source = item['source']
    try:
        new_source = mapping[item['source']]
    except Exception as e:
        print e
    if (new_source == "News"):
        try:
            new_source = item['meta.source.publisher'].title() # initial cap version of news source
        except Exception:
            pass
    if (new_source == "DW ?"):
        try:
            new_source = 'DW video' if (item['meta.original_format'] == 'DW-video-may-release') else \
                         'DW article' if item['meta.original_format'] == 'DW-news-may-release' else \
                         new_source
        except Exception:
            pass
    return new_source


def main():
    for item in col.find({},projection=['_id','source','meta.source.publisher','meta.original_format']):
        res = col.update({'_id': item['_id']},{'$set': {'source': source_map(item),'old_source':item['source']}})<commit_msg>rename-sources: hopefully fixed access to meta.original_format etc.<commit_after>#!/usr/bin/env python

import pymongo

mongo_client = pymongo.MongoClient()
db = mongo_client['eumssi_db']
col = db['content_items']


def source_map(item):
    mapping = {
        'Twitter':'Twitter',
        'Twitter-DW':'Twitter',
        'DW-de_DE':'DW article',
        'DW-en_GB':'DW article',
        'DW-es_ES':'DW article',
        'DW-fr_FR':'DW article',
        'DW-vid_EN':'DW video',
        'Eumssi-News-Crawler':'News',
        'Youtube-video-GeneralChannel':'Youtube',
        'Youtube-video-dwEnglishChannel':'DW (Youtube)',
        'Youtube-video-theguardianChannel':'Guardian (Youtube)',
        'DW-en':'DW ?',
        'DW-DE':'DW ?',
        'DW-de':'DW ?',
        'DW-EN':'DW ?',
        'DW-es':'DW ?',
        'DW-ES':'DW ?',
        'DW-FR':'DW ?'
    }
    new_source = item['source']
    try:
        new_source = mapping[item['source']]
    except Exception as e:
        print e
    if (new_source == "News"):
        try:
            new_source = item['meta']['source']['publisher'].title() # initial cap version of news source
        except Exception:
            print e
    if (new_source == "DW ?"):
        try:
            new_source = 'DW video' if (item['meta']['original_format'] == 'DW-video-may-release') else \
                         'DW article' if item['meta']['original_format'] == 'DW-news-may-release' else \
                         new_source
        except Exception:
            print e
    return new_source


def main():
    for item in col.find({},projection=['_id','source','meta.source.publisher','meta.original_format']):
        res = col.update({'_id': item['_id']},{'$set': {'source': source_map(item),'old_source':item['source']}})
{'fraction_non_alphanumeric': 0.18025258323765786, 'fraction_numerical': 0.0, 'mean_word_length': 2.4166666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3918102', 'n_tokens_mistral': 1238, 'n_tokens_neox': 1110, 'n_words': 203}