<commit_before>#!/usr/bin/env python

'''
Created on Feb 24, 2012
'''

from __future__ import division

__author__ = "<NAME>"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__date__ = "Feb 24, 2012"

import logging
import json
import datetime
import collections

from pymatgen.symmetry.finder import SymmetryFinder
from pymatgen.core.structure_modifier import StructureEditor
from pymatgen.analysis.structure_fitter import StructureFitter
from pymatgen.serializers.json_coders import PMGJSONEncoder, PMGJSONDecoder

logger = logging.getLogger(__name__)


def _get_host(structure, species_to_remove):
    if species_to_remove:
        editor = StructureEditor(structure)
        editor.remove_species(species_to_remove)
        return editor.modified_structure
    else:
        return structure


def _perform_grouping(args):
    (entries_json, hosts_json, anonymized, symmetry_tol, fitting_accuracy,
     groups) = args
    entries = json.loads(entries_json, cls=PMGJSONDecoder)
    hosts = json.loads(hosts_json, cls=PMGJSONDecoder)
    unmatched = zip(entries, hosts)
    while len(unmatched) > 0:
        ref_host = unmatched[0][1]
        logger.info(
            "Reference tid = {}, formula = {}".format(unmatched[0][0].entry_id,
                                                      ref_host.formula)
        )
        ref_formula = ref_host.composition.anonymized_formula if anonymized \
            else ref_host.composition.reduced_formula
        logger.info("Reference host = {}".format(ref_formula))
        matches = [unmatched[0]]
        for i in xrange(1, len(unmatched)):
            test_host = unmatched[i][1]
            logger.info("Testing tid = {}, formula = {}"
                         .format(unmatched[i][0].entry_id, test_host.formula))
            test_formula = test_host.composition.anonymized_formula \
                if anonymized else test_host.composition.reduced_formula
            logger.info("Test host = {}".format(test_formula))
            if test_formula == ref_formula:
                fitter = StructureFitter(ref_host, test_host,
                                         anonymized=anonymized,
                                         symmetry_tol=symmetry_tol,
                                         fitting_accuracy=fitting_accuracy)
                if fitter.fit_found:
                    logger.info("Fit found")
                    matches.append(unmatched[i])
        groups.append(json.dumps([m[0] for m in matches], cls=PMGJSONEncoder))
        unmatched = filter(lambda x: x not in matches, unmatched)
        logger.info("{} unmatched remaining".format(len(unmatched)))


def group_entries_by_structure(entries, species_to_remove=None,
                               anonymized=False, symmetry_tol=0,
                               fitting_accuracy=StructureFitter.FAST_FIT,
                               ncpus=None):
    """
    Given a sequence of ComputedStructureEntries, use structure fitter to group
    them by structural similarity.

    Args:
        entries:
            Sequence of entries.
        species_to_remove:
            Sometimes you want to compare a host framework (e.g., in Li-ion
            battery analysis). This allows you to specify species to remove
            before structural comparison.
        anonymized:
            Allow anonymized fitting.
        fitting_accuracy:
            The fitting accuracy to use for StructureFitter. Defaults to
            FAST_FIT.
        ncpus:
            Number of cpus to use. Use of multiple cpus can greatly improve
            fitting speed. Default of None means serial processing.

    Returns:
        Sequence of sequence of entries by structural similarity. e.g,
        [[ entry1, entry2], [entry3, entry4, entry5]]
    """

    start = datetime.datetime.now()
    logger.info("Started at {}".format(start))
    entries_host = [(entry, _get_host(entry.structure, species_to_remove))<commit_msg>Make doc clearer that ComputedStructureEntries are needed.<commit_after>#!/usr/bin/env python

'''
Created on Feb 24, 2012
'''

from __future__ import division

__author__ = "<NAME>"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"
__date__ = "Feb 24, 2012"

import logging
import json
import datetime
import collections

from pymatgen.symmetry.finder import SymmetryFinder
from pymatgen.core.structure_modifier import StructureEditor
from pymatgen.analysis.structure_fitter import StructureFitter
from pymatgen.serializers.json_coders import PMGJSONEncoder, PMGJSONDecoder

logger = logging.getLogger(__name__)


def _get_host(structure, species_to_remove):
    if species_to_remove:
        editor = StructureEditor(structure)
        editor.remove_species(species_to_remove)
        return editor.modified_structure
    else:
        return structure


def _perform_grouping(args):
    (entries_json, hosts_json, anonymized, symmetry_tol, fitting_accuracy,
     groups) = args
    entries = json.loads(entries_json, cls=PMGJSONDecoder)
    hosts = json.loads(hosts_json, cls=PMGJSONDecoder)
    unmatched = zip(entries, hosts)
    while len(unmatched) > 0:
        ref_host = unmatched[0][1]
        logger.info(
            "Reference tid = {}, formula = {}".format(unmatched[0][0].entry_id,
                                                      ref_host.formula)
        )
        ref_formula = ref_host.composition.anonymized_formula if anonymized \
            else ref_host.composition.reduced_formula
        logger.info("Reference host = {}".format(ref_formula))
        matches = [unmatched[0]]
        for i in xrange(1, len(unmatched)):
            test_host = unmatched[i][1]
            logger.info("Testing tid = {}, formula = {}"
                         .format(unmatched[i][0].entry_id, test_host.formula))
            test_formula = test_host.composition.anonymized_formula \
                if anonymized else test_host.composition.reduced_formula
            logger.info("Test host = {}".format(test_formula))
            if test_formula == ref_formula:
                fitter = StructureFitter(ref_host, test_host,
                                         anonymized=anonymized,
                                         symmetry_tol=symmetry_tol,
                                         fitting_accuracy=fitting_accuracy)
                if fitter.fit_found:
                    logger.info("Fit found")
                    matches.append(unmatched[i])
        groups.append(json.dumps([m[0] for m in matches], cls=PMGJSONEncoder))
        unmatched = filter(lambda x: x not in matches, unmatched)
        logger.info("{} unmatched remaining".format(len(unmatched)))


def group_entries_by_structure(entries, species_to_remove=None,
                               anonymized=False, symmetry_tol=0,
                               fitting_accuracy=StructureFitter.FAST_FIT,
                               ncpus=None):
    """
    Given a sequence of ComputedStructureEntries, use structure fitter to group
    them by structural similarity.

    Args:
        entries:
            Sequence of ComputedStructureEntries.
        species_to_remove:
            Sometimes you want to compare a host framework (e.g., in Li-ion
            battery analysis). This allows you to specify species to remove
            before structural comparison.
        anonymized:
            Allow anonymized fitting.
        fitting_accuracy:
            The fitting accuracy to use for StructureFitter. Defaults to
            FAST_FIT.
        ncpus:
            Number of cpus to use. Use of multiple cpus can greatly improve
            fitting speed. Default of None means serial processing.

    Returns:
        Sequence of sequence of entries by structural similarity. e.g,
        [[ entry1, entry2], [entry3, entry4, entry5]]
    """

    start = datetime.datetime.now()
    logger.info("Started at {}".format(start))
    entries_host = [(entry, _get_host(entry.structure, species_to_remove))
{'fraction_non_alphanumeric': 0.08236601630032107, 'fraction_numerical': 0.008397135095085207, 'mean_word_length': 2.349462365591398, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5085630', 'n_tokens_mistral': 2262, 'n_tokens_neox': 2132, 'n_words': 549}