<commit_before>#!/usr/bin/env python

"""
Classes and methods related to the Structure Notation Language (SNL)
"""

from pymatgen.core.structure import Structure
from collections import namedtuple
import re
import datetime
from pybtex.database.input import bibtex
import tempfile
import sys

__author__ = '<NAME>, <NAME>'
__credits__ = '<NAME>'
__copyright__ = 'Copyright 2013, The Materials Project'
__version__ = '0.1'
__maintainer__ = '<NAME>'
__email__ = '<EMAIL>'
__date__ = 'Feb 11, 2013'

MAX_HNODE_SIZE = 64000  # maximum size (bytes) of SNL HistoryNode
MAX_DATA_SIZE = 256000  # maximum size (bytes) of SNL data field
MAX_HNODES = 100  # maximum number of HistoryNodes in SNL file
MAX_BIBTEX_CHARS = 20000  # maximum number of characters for BibTeX reference


def is_valid_bibtex(reference):
    """
    Use pybtex to validate that a reference is in proper BibTeX format

    :param reference: a String reference in BibTeX format
    """
    parser = bibtex.Parser()
    temp = tempfile.NamedTemporaryFile(delete=True)
    temp.write(reference)
    temp.seek(0)
    bib_data = None
    try:
        bib_data = parser.parse_file(temp.name)
        if len(bib_data.entries) > 0:
            return True
    except:
        pass
    finally:
        temp.close()
    return False


class HistoryNode(namedtuple('HistoryNode', ['name', 'url', 'description'])):
    """
    A HistoryNode represents a step in the chain of events that lead to a
    Structure. HistoryNodes leave 'breadcrumbs' so that you can trace back how
    a Structure was created. For example, a HistoryNode might represent pulling
    a Structure from an external database such as the ICSD or CSD. Or, it might
    represent the application of a code (e.g. pymatgen) to the Structure, with
    a custom description of how that code was applied (e.g. a site removal
    Transformation was applied).

    A HistoryNode contains three fields:
    - name: the name of a code or resource that this Structure encountered in
            its history (String)
    - url: the URL of that code/resource (String)
    - description: a free-form description of how the code/resource is
                   related to the Structure (dict)
    """

    @property
    def to_dict(self):
        return dict(self._asdict())

    @staticmethod
    def from_dict(h_node):
        return HistoryNode(h_node['name'], h_node['url'], h_node['description'])

    @staticmethod
    def parse_historynode(h_node):
        """
        Parses a History Node object from either a dict or a tuple.

        :param h_node: a dict with name/url/description fields or a 3-element
                      tuple
        """
        if isinstance(h_node, dict):
            return HistoryNode.from_dict(h_node)

        else:
            if len(h_node) != 3:
                raise ValueError("Invalid History node, "
                                 "should be dict or (name, version, "
                                 "description) tuple: {}".format(h_node))
            return HistoryNode(h_node[0], h_node[1], h_node[2])


class Author(namedtuple('Author', ['name', 'email'])):
    """
    An Author contains two fields:

    - name: (String)
    - email: (String)
    """

    def __str__(self):
        """
        String representation of an Author
        """
        return '{} <{}>'.format(self.name, self.email)

    @property
    def to_dict(self):
        return dict(self._asdict())

    @staticmethod
    def from_dict(d):
        return Author(d['name'], d['email'])

    @staticmethod
    def parse_author(author):
        """
        Parses an Author object from either a String, dict, or tuple

        :param author: A String formatted as "NAME <<EMAIL>>",
                       (name, email) tuple, or a dict with name and email keys
        """
        if isinstance(author, basestring):
            # Regex looks for whitespace, (any name), whitespace, <, (email),
            # >, whitespace
            m = re.match('\s*(.*?)\s*<(.*?@.*?)>\s*', author)
            if not m or m.start() != 0 or m.end() != len(author):
                raise ValueError("Invalid author format! {}".format(author))
            return Author(m.groups()[0], m.groups()[1])

        elif isinstance(author, dict):
            return Author.from_dict(author)

        else:
            if len(author) != 2:
                raise ValueError("Invalid author, should be String or (name, "
                                 "email) tuple: {}".format(author))
            return Author(author[0], author[1])


class StructureNL(Structure):
    """
    The Structure Notation Language (SNL, pronounced 'snail') is a pymatgen
    Structure object with some additional fields for enhanced provenance. It
    is meant to be imported/exported in a JSON file format with the following
    structure:
    - about
        - created_at
        - authors
        - projects
        - references
        - remarks
        - data
        - history
    - lattice
    - sites
    """

    def __init__(self, structure, authors, projects=None, references='',
                 remarks=None, data=None, history=None, created_at=None):
        """
        :param structure: a pymatgen.core.structure Structure object
        :param authors: *list* of {"name":'', "email":''} dicts,
                        *list* of Strings as '<NAME> <<EMAIL>>',
                        or a single String with commas separating authors
        :param projects: list of Strings ['Project A', 'Project B']
        :param references: a String in BibTeX format
        :param remarks: list of Strings ['Remark A', 'Remark B']
        :param data: a free form dict. Namespaced at the root level with an
                     underscore, e.g. {"_materialsproject":<custom data>}
        :param history: list of dicts - [{'name':'', 'url':'',
                                          'description':{}}]
        :param created_at: a datetime object
        """

        # initialize root-level structure keys
        Structure.__init__(self, structure.lattice,
                           [site.species_and_occu for site in structure],
                           structure.frac_coords)

        # turn authors into list of Author objects
        authors = authors.split(',')\
            if isinstance(authors, basestring) else authors
        self.authors = [Author.parse_author(a) for a in authors]

        # turn projects into list of Strings
        projects = projects if projects else []
        self.projects = [projects] \
            if isinstance(projects, basestring) else projects

        # check that references are valid BibTeX
        if references and not is_valid_bibtex(references):
            raise ValueError("Invalid format for SNL reference! Should be "
                             "BibTeX string.")
        if len(references) > MAX_BIBTEX_CHARS:
            raise ValueError("The BibTeX string must be fewer than {} chars "
                             ", you have {}"
                             .format(MAX_BIBTEX_CHARS, len(references)))

        self.references = references

        # turn remarks into list of Strings
        remarks = remarks if remarks else []
        self.remarks = [remarks] if isinstance(remarks, basestring) else remarks
<commit_msg>Use StringIO instead of temp file for reference checking.<commit_after>#!/usr/bin/env python

"""
Classes and methods related to the Structure Notation Language (SNL)
"""

__author__ = '<NAME>, <NAME>'
__credits__ = '<NAME>'
__copyright__ = 'Copyright 2013, The Materials Project'
__version__ = '0.1'
__maintainer__ = '<NAME>'
__email__ = '<EMAIL>'
__date__ = 'Feb 11, 2013'

import sys
import cStringIO
import re
import datetime
from collections import namedtuple

from pybtex.database.input import bibtex

from pymatgen.core.structure import Structure

MAX_HNODE_SIZE = 64000  # maximum size (bytes) of SNL HistoryNode
MAX_DATA_SIZE = 256000  # maximum size (bytes) of SNL data field
MAX_HNODES = 100  # maximum number of HistoryNodes in SNL file
MAX_BIBTEX_CHARS = 20000  # maximum number of characters for BibTeX reference


def is_valid_bibtex(reference):
    """
    Use pybtex to validate that a reference is in proper BibTeX format

    Args:
        reference:
            A String reference in BibTeX format
    """

    sio = cStringIO.StringIO(reference)
    try:
        parser = bibtex.Parser()
        bib_data = parser.parse_stream(sio)
        if len(bib_data.entries) > 0:
            return True
    except:
        pass
    return False


class HistoryNode(namedtuple('HistoryNode', ['name', 'url', 'description'])):
    """
    A HistoryNode represents a step in the chain of events that lead to a
    Structure. HistoryNodes leave 'breadcrumbs' so that you can trace back how
    a Structure was created. For example, a HistoryNode might represent pulling
    a Structure from an external database such as the ICSD or CSD. Or, it might
    represent the application of a code (e.g. pymatgen) to the Structure, with
    a custom description of how that code was applied (e.g. a site removal
    Transformation was applied).

    A HistoryNode contains three fields:
    - name: the name of a code or resource that this Structure encountered in
            its history (String)
    - url: the URL of that code/resource (String)
    - description: a free-form description of how the code/resource is
                   related to the Structure (dict)
    """

    @property
    def to_dict(self):
        return dict(self._asdict())

    @staticmethod
    def from_dict(h_node):
        return HistoryNode(h_node['name'], h_node['url'], h_node['description'])

    @staticmethod
    def parse_historynode(h_node):
        """
        Parses a History Node object from either a dict or a tuple.

        :param h_node: a dict with name/url/description fields or a 3-element
                      tuple
        """
        if isinstance(h_node, dict):
            return HistoryNode.from_dict(h_node)

        else:
            if len(h_node) != 3:
                raise ValueError("Invalid History node, "
                                 "should be dict or (name, version, "
                                 "description) tuple: {}".format(h_node))
            return HistoryNode(h_node[0], h_node[1], h_node[2])


class Author(namedtuple('Author', ['name', 'email'])):
    """
    An Author contains two fields:

    - name: (String)
    - email: (String)
    """

    def __str__(self):
        """
        String representation of an Author
        """
        return '{} <{}>'.format(self.name, self.email)

    @property
    def to_dict(self):
        return dict(self._asdict())

    @staticmethod
    def from_dict(d):
        return Author(d['name'], d['email'])

    @staticmethod
    def parse_author(author):
        """
        Parses an Author object from either a String, dict, or tuple

        :param author: A String formatted as "NAME <<EMAIL>>",
                       (name, email) tuple, or a dict with name and email keys
        """
        if isinstance(author, basestring):
            # Regex looks for whitespace, (any name), whitespace, <, (email),
            # >, whitespace
            m = re.match('\s*(.*?)\s*<(.*?@.*?)>\s*', author)
            if not m or m.start() != 0 or m.end() != len(author):
                raise ValueError("Invalid author format! {}".format(author))
            return Author(m.groups()[0], m.groups()[1])

        elif isinstance(author, dict):
            return Author.from_dict(author)

        else:
            if len(author) != 2:
                raise ValueError("Invalid author, should be String or (name, "
                                 "email) tuple: {}".format(author))
            return Author(author[0], author[1])


class StructureNL(Structure):
    """
    The Structure Notation Language (SNL, pronounced 'snail') is a pymatgen
    Structure object with some additional fields for enhanced provenance. It
    is meant to be imported/exported in a JSON file format with the following
    structure:
    - about
        - created_at
        - authors
        - projects
        - references
        - remarks
        - data
        - history
    - lattice
    - sites
    """

    def __init__(self, structure, authors, projects=None, references='',
                 remarks=None, data=None, history=None, created_at=None):
        """
        Args:
            structure:
                A pymatgen.core.structure Structure object
            authors:
                *List* of {"name":'', "email":''} dicts,
                *list* of Strings as '<NAME> <<EMAIL>>',
                or a single String with commas separating authors
            projects:
                List of Strings ['Project A', 'Project B']
            references:
                A String in BibTeX format
            remarks:
                List of Strings ['Remark A', 'Remark B']
            data:
                A free form dict. Namespaced at the root level with an
                underscore, e.g. {"_materialsproject":<custom data>}
            history:
                List of dicts - [{'name':'', 'url':'', 'description':{}}]
            created_at:
                A datetime object
        """
        # initialize root-level structure keys
        Structure.__init__(self, structure.lattice,
                           [site.species_and_occu for site in structure],
                           structure.frac_coords)

        # turn authors into list of Author objects
        authors = authors.split(',')\
            if isinstance(authors, basestring) else authors
        self.authors = [Author.parse_author(a) for a in authors]

        # turn projects into list of Strings
        projects = projects if projects else []
        self.projects = [projects] \
            if isinstance(projects, basestring) else projects

        # check that references are valid BibTeX
        if references and not is_valid_bibtex(references):
            raise ValueError("Invalid format for SNL reference! Should be "
                             "BibTeX string.")
        if len(references) > MAX_BIBTEX_CHARS:
            raise ValueError("The BibTeX string must be fewer than {} chars "
                             ", you have {}"
                             .format(MAX_BIBTEX_CHARS, len(references)))

        self.references = references

        # turn remarks into list of Strings
        remarks = remarks if remarks else []
        self.remarks = [remarks] if isinstance(remarks, basestring) else remarks

{'fraction_non_alphanumeric': 0.09597969264544456, 'fraction_numerical': 0.005968715697036224, 'mean_word_length': 2.154512010387362, 'pattern_counts': {'":': 6, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6683313', 'n_tokens_mistral': 4049, 'n_tokens_neox': 3838, 'n_words': 1373}