<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#

from __future__ import unicode_literals, absolute_import

import time

from elasticsearch.helpers import scan
from elasticsearch.exceptions import NotFoundError

from oclubs.app import app
from oclubs.access import database, elasticsearch, done
from oclubs.objs import Activity, Club

clses = [Club, Activity]

with app.app_context():
    for cls in clses:
        db_ids = database.fetch_onecol(
            cls.table,
            cls.identifier,
            {}
        )
        db_ids = set(int(x) for x in db_ids)
        db_max = max(db_ids)

        try:
            es_ids = scan(
                elasticsearch.es,
                index='oclubs',
                doc_type=cls.table,
                size=10000000,
                query={
                    'query': {'match_all': {}},
                    'size': 10000,
                    'fields': ['_id']
                })
            es_ids = (d['_id'] for d in es_ids)
        except NotFoundError:
            es_ids = []

        es_ids = set(int(x) for x in es_ids)

        if es_ids:
            es_max = max(es_ids)
        else:<commit_msg>Fix db2es.py<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#

from __future__ import unicode_literals, absolute_import

import time

from elasticsearch.helpers import scan
from elasticsearch.exceptions import NotFoundError

from oclubs.app import app
from oclubs.access import database, elasticsearch, done
from oclubs.objs import Activity, Club

clses = [Club, Activity]

with app.app_context():
    for cls in clses:
        db_ids = database.fetch_onecol(
            cls.table,
            cls.identifier,
            {}
        )
        db_ids = set(int(x) for x in db_ids)
        db_max = max(db_ids)

        try:
            es_ids = scan(
                elasticsearch.es,
                index='oclubs',
                doc_type=cls.table,
                size=10000000,
                query={
                    'query': {'match_all': {}},
                    'size': 10000,
                    'fields': ['_id']
                })
            es_ids = (d['_id'] for d in es_ids)
            es_ids = set(int(x) for x in es_ids)
        except NotFoundError:
            es_ids = []

        if es_ids:
            es_max = max(es_ids)
        else:
{'fraction_non_alphanumeric': 0.09821428571428571, 'fraction_numerical': 0.012329931972789115, 'mean_word_length': 1.5971302428256071, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '992277', 'n_tokens_mistral': 751, 'n_tokens_neox': 697, 'n_words': 178}