<commit_before><commit_msg>add final target as example `lumberjack.py`<commit_after>import time

from pymongo import MongoClient

from dcard import Dcard, logger


client = MongoClient()
db = client['dcard']
collct = None


def should_it_update(metas, forum):
    results = []
    for meta in metas:
        found = db[forum].find_one({'id': meta['id']})
        if not found:# or found['updatedAt'] < meta['updatedAt']:
            results.append(meta['id'])
    return results


def store_to_db(posts, forum):
    result = db[forum].insert_many([p for p in posts])
    logger.info('[database] #Forum {}: insert {} items'.format(forum, len(result.inserted_ids)))


def main():
    dcard = Dcard()
    forums = dcard.forums.get(no_school=True)

    for forum in forums:
        name = forum['alias']

        bound = 30  # let it be infinity later!
        metas = dcard.forums(name).get_metas(
                    num=bound, callback=lambda metas, forum=name: \
                        should_it_update(metas, forum)
                )
        if metas:
            dcard.posts(metas).get(
                callback=lambda posts, forum=name: store_to_db(posts, forum))

        # db[name].drop()

if __name__ == '__main__':
    s = time.time()
    main()
    print('{:.05} sec'.format(time.time() - s))
{'fraction_non_alphanumeric': 0.13604919292851653, 'fraction_numerical': 0.0030745580322828594, 'mean_word_length': 2.636871508379888, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3705164', 'n_tokens_mistral': 435, 'n_tokens_neox': 411, 'n_words': 101}