<commit_before>#!/usr/bin/env python3
#-*- encoding: utf-8 -*-

# Copyright 2016 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from urllib import parse as urlparse
import argparse
import collections
import difflib
import elasticsearch
import json
import logging

logger = logging.getLogger(__name__)


def get_id_keys(es, index, type_, id_key):
    body = {
        "query": {"type": {"value": type_}},
        "fields": [id_key, "_id"],
    }

    page = es.search(index=index, scroll='2m', search_type='scan', size=1000, body=body)
    sid = page['_scroll_id']
    scroll_size = page['hits']['total']

    results = {}
    while (scroll_size > 0):
        page = es.scroll(scroll_id=sid, scroll='2m')
        sid = page['_scroll_id']
        scroll_size = len(page['hits']['hits'])
        results.update({h['fields'][id_key][0]: h['_id'] for h in page['hits']['hits']})

    return results

def print_only(only, side):
    for element in only:
        print('only in {0}: {1}'.format(side, element))

def make_ordered(d):
    def handle(e):
        if isinstance(e, dict):
            return collections.OrderedDict(sorted(e.items(), key=lambda t: handle(t[0])))
        if isinstance(e, (list, tuple)):
            return [handle(x) for x in e]
        return e
    return handle(d)

def diff_common(es_left, left_index, es_right, right_index, common, quiet):
    diff_entries = 0
    for key_, es_ids in common.items():
        left = make_ordered(es_left.get(index=left_index, id=es_ids[0])['_source'])
        right = make_ordered(es_right.get(index=right_index, id=es_ids[1])['_source'])
        if left != right:
            print('entries for key {} differ'.format(key_))
            if not quiet:
                leftj = json.dumps(left, indent=2)
                rightj = json.dumps(right, indent=2)
                for line in difflib.unified_diff(leftj.splitlines(), rightj.splitlines()):
                    print(line)
    return diff_entries

def diff(es_left, left_index, es_right, right_index, type_, id_key, quiet):
    left = get_id_keys(es_left, left_index, type_, id_key)
    right = get_id_keys(es_right, right_index, type_, id_key)

    common_keys = set(left.keys()) & set(right.keys())
    only_left_keys = set(left.keys()) - common_keys
    only_right_keys = set(right.keys()) - common_keys

    print_only(only_left_keys, 'left')
    print_only(only_right_keys, 'right')

    common = collections.defaultdict(lambda: [None, None])
    for k, v in left.items():
        common[k][0] = v
    for k, v in right.items():
        common[k][1] = v
    common = {k: v for k, v in common.items() if None not in v}
    entries_differ = diff_common(es_left, left_index, es_right, right_index, common, quiet)

    print('Summary:')
    print('{} entries only in left index'.format(len(only_left_keys)))
    print('{} entries only in right index'.format(len(only_right_keys)))
    print('{} common entries'.format(len(common_keys)))
    print('  {} of them are the same'.format(len(common_keys) - entries_differ))
    print('  {} of them differ'.format(entries_differ))

def main(left_index_url, right_index_url, type_, id_key, quiet):
    left_index_url = urlparse.urlparse(left_index_url)
    left_es_url = left_index_url.scheme + '://' +  left_index_url.netloc
    es_left = elasticsearch.Elasticsearch(left_es_url)
    if len(left_index_url.path.split('/')) != 2:
        raise RuntimeError('wrong src index path: {}'.format(left_index_url.path))
    right_index_url = urlparse.urlparse(right_index_url)
    right_es_url = right_index_url.scheme + '://' +  right_index_url.netloc
    es_right = elasticsearch.Elasticsearch(right_es_url)
    if len(right_index_url.path.split('/')) != 2:
        raise RuntimeError('wrong dest index path: {}'.format(right_index_url.path))

    left_indices = [x for x in es_left.indices.get_aliases().keys()]
    left_index = left_index_url.path[1:]

    if left_index not in left_indices:
        raise RuntimeError('left index does not exist!')

    right_indices = [x for x in es_right.indices.get_aliases().keys()]
    right_index = right_index_url.path[1:]

    if right_index not in right_indices:
        raise RuntimeError('right index does not exist!')

    return diff(es_left, left_index, es_right, right_index, type_, id_key, quiet)


if __name__ == "__main__":
    argparser = argparse.ArgumentParser(description='elasticdiff')
    argparser.add_argument('left_index_url')
    argparser.add_argument('right_index_url')
    argparser.add_argument('type')
    argparser.add_argument('id_key')
    argparser.add_argument('-q', '--quiet', action='store_true')
    args = argparser.parse_args()
    main(args.left_index_url, args.right_index_url, args.type, args.id_key, args.quiet)<commit_msg>changed type and id-key args to ones with dashes<commit_after>#!/usr/bin/env python3
#-*- encoding: utf-8 -*-

# Copyright 2016 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from urllib import parse as urlparse
import argparse
import collections
import difflib
import elasticsearch
import json
import logging

logger = logging.getLogger(__name__)


def get_id_keys(es, index, type_, id_key):
    body = {
        "query": {"type": {"value": type_}},
        "fields": [id_key, "_id"],
    }

    page = es.search(index=index, scroll='2m', search_type='scan', size=1000, body=body)
    sid = page['_scroll_id']
    scroll_size = page['hits']['total']

    results = {}
    while (scroll_size > 0):
        page = es.scroll(scroll_id=sid, scroll='2m')
        sid = page['_scroll_id']
        scroll_size = len(page['hits']['hits'])
        results.update({h['fields'][id_key][0]: h['_id'] for h in page['hits']['hits']})

    return results

def print_only(only, side):
    for element in only:
        print('only in {0}: {1}'.format(side, element))

def make_ordered(d):
    def handle(e):
        if isinstance(e, dict):
            return collections.OrderedDict(sorted(e.items(), key=lambda t: handle(t[0])))
        if isinstance(e, (list, tuple)):
            return [handle(x) for x in e]
        return e
    return handle(d)

def diff_common(es_left, left_index, es_right, right_index, common, quiet):
    diff_entries = 0
    for key_, es_ids in common.items():
        left = make_ordered(es_left.get(index=left_index, id=es_ids[0])['_source'])
        right = make_ordered(es_right.get(index=right_index, id=es_ids[1])['_source'])
        if left != right:
            print('entries for key {0} differ'.format(key_))
            if not quiet:
                leftj = json.dumps(left, indent=2)
                rightj = json.dumps(right, indent=2)
                for line in difflib.unified_diff(leftj.splitlines(), rightj.splitlines()):
                    print(line)
    return diff_entries

def diff(es_left, left_index, es_right, right_index, type_, id_key, quiet):
    left = get_id_keys(es_left, left_index, type_, id_key)
    right = get_id_keys(es_right, right_index, type_, id_key)

    common_keys = set(left.keys()) & set(right.keys())
    only_left_keys = set(left.keys()) - common_keys
    only_right_keys = set(right.keys()) - common_keys

    print_only(only_left_keys, 'left')
    print_only(only_right_keys, 'right')

    common = collections.defaultdict(lambda: [None, None])
    for k, v in left.items():
        common[k][0] = v
    for k, v in right.items():
        common[k][1] = v
    common = {k: v for k, v in common.items() if None not in v}
    entries_differ = diff_common(es_left, left_index, es_right, right_index, common, quiet)

    print('Summary:')
    print('{} entries only in left index'.format(len(only_left_keys)))
    print('{} entries only in right index'.format(len(only_right_keys)))
    print('{} common entries'.format(len(common_keys)))
    print('  {} of them are the same'.format(len(common_keys) - entries_differ))
    print('  {} of them differ'.format(entries_differ))

def main(left_index_url, right_index_url, type_, id_key, quiet):
    left_index_url = urlparse.urlparse(left_index_url)
    left_es_url = left_index_url.scheme + '://' +  left_index_url.netloc
    es_left = elasticsearch.Elasticsearch(left_es_url)
    if len(left_index_url.path.split('/')) != 2:
        raise RuntimeError('wrong src index path: {}'.format(left_index_url.path))
    right_index_url = urlparse.urlparse(right_index_url)
    right_es_url = right_index_url.scheme + '://' +  right_index_url.netloc
    es_right = elasticsearch.Elasticsearch(right_es_url)
    if len(right_index_url.path.split('/')) != 2:
        raise RuntimeError('wrong dest index path: {}'.format(right_index_url.path))

    left_indices = [x for x in es_left.indices.get_aliases().keys()]
    left_index = left_index_url.path[1:]

    if left_index not in left_indices:
        raise RuntimeError('left index does not exist!')

    right_indices = [x for x in es_right.indices.get_aliases().keys()]
    right_index = right_index_url.path[1:]

    if right_index not in right_indices:
        raise RuntimeError('right index does not exist!')

    return diff(es_left, left_index, es_right, right_index, type_, id_key, quiet)


if __name__ == "__main__":
    argparser = argparse.ArgumentParser(description='elasticdiff')
    argparser.add_argument('left_index_url')
    argparser.add_argument('right_index_url')
    argparser.add_argument('-t', '--type')
    argparser.add_argument('-i', '--id-key')
    argparser.add_argument('-q', '--quiet', action='store_true')
    args = argparser.parse_args()
    main(args.left_index_url, args.right_index_url, args.type, args.id_key, args.quiet)
{'fraction_non_alphanumeric': 0.12986767485822306, 'fraction_numerical': 0.006143667296786389, 'mean_word_length': 3.7005775211017324, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6667440', 'n_tokens_mistral': 3470, 'n_tokens_neox': 3404, 'n_words': 866}