# Path: chemdataextractor/scrape/base.py
# class BaseField(six.with_metaclass(ABCMeta)):
#     """Base class for all fields."""
# 
#     # This is assigned by EntityMeta to match the attribute on the Entity
#     name = None
# 
#     def __init__(self, selection, xpath=False, re=None, all=False, default=None, null=False, raw=False):
#         """
# 
#         :param string selection: The CSS selector or XPath expression used to select the content to scrape.
#         :param bool xpath: (Optional) Whether selection is an XPath expression instead of a CSS selector. Default False.
#         :param re: (Optional) Regular expression to apply to scraped content.
#         :param bool all: (Optional) Whether to scrape all occurrences instead of just the first. Default False.
#         :param default: (Optional) The default value for this field if none is set.
#         :param bool null: (Optional) Include in serialized output even if value is None. Default False.
#         :param bool raw: (Optional) Whether to scrape the raw HTML/XML instead of the text contents. Default False.
#         """
#         self.selection = selection
#         self.xpath = xpath
#         self.re = re
#         self.all = all
#         self.default = default
#         self.null = null
#         self.raw = raw
# 
#     def __get__(self, instance, owner):
#         """Descriptor for retrieving a value from a field in an Entity."""
#         # Check if Entity class is being called, rather than Entity instance
#         if instance is None:
#             return self
#         # Get value from Entity instance if available
#         value = instance._values.get(self.name)
#         # If value is None, empty list or empty string return the default value if set
#         if value in [None, [], ''] and self.default is not None:
#             return self.default
#         # Otherwise if value is None and all, return empty list
#         if self.all and value is None:
#             return []
#         return value
# 
#     def __set__(self, instance, value):
#         """Descriptor for assigning a value to a field in a Entity."""
#         instance._values[self.name] = value
# 
#     def _post_scrape(self, value, processor=None):
#         """Apply processing to the scraped value."""
#         # Pass each value through the field's clean method
#         value = [self.process(v) for v in value]
#         # Filter None values
#         value = [v for v in value if v is not None]
#         # Pass each value through processors defined on the entity
#         if processor:
#             value = [processor(v) for v in value]
#             value = [v for v in value if v is not None]
#         # Take first unless all is specified
#         if not self.all:
#             value = value[0] if value else None
#         log.debug('Scraped %s: %s from %s' % (self.name, value, self.selection))
#         return value
# 
#     def scrape(self, selector, cleaner=None, processor=None):
#         """Scrape the value for this field from the selector."""
#         # Apply CSS or XPath expression to the selector
#         selected = selector.xpath(self.selection) if self.xpath else selector.css(self.selection)
#         # Extract the value and apply regular expression if specified
#         value = selected.re(self.re) if self.re else selected.extract(raw=self.raw, cleaner=cleaner)
#         return self._post_scrape(value, processor=processor)
# 
#     def serialize(self, value):
#         """Serialize this field."""
#         if hasattr(value, 'serialize'):
#             return value.serialize()
#         else:
#             return value
# 
#     def process(self, value):
#         """Override to perform custom processing of a value."""
#         return value
# 
# Path: chemdataextractor/text/processors.py
# def strip_querystring(url):
#     """Remove the querystring from the end of a URL."""
#     p = six.moves.urllib.parse.urlparse(url)
#     return p.scheme + "://" + p.netloc + p.path


import logging
import re
import dateutil.parser
import six
from .base import BaseField
from ..text.processors import strip_querystring


class EntityField(BaseField):
    """A field that contains another Entity."""

    def __init__(self, entity, selection, **kwargs):
        """

        :param entity: The embedded entity.
        """
        self.entity = entity
        super(EntityField, self).__init__(selection, **kwargs)

    def scrape(self, selector, cleaner=None, processor=None):
        """Scrape the value for this field from the selector."""
        value = self.entity.scrape(selector, root=self.selection, xpath=self.xpath)
        return self._post_scrape(value, processor=processor)


class IntField(BaseField):
    """An integer number field."""

    def process(self, value):
        """Convert value to an int."""
        try:
            return int(value)
        except (ValueError, TypeError):
            return None



# Path: chemdataextractor/scrape/base.py
# class BaseField(six.with_metaclass(ABCMeta)):
#     """Base class for all fields."""
# 
#     # This is assigned by EntityMeta to match the attribute on the Entity
#     name = None
# 
#     def __init__(self, selection, xpath=False, re=None, all=False, default=None, null=False, raw=False):
#         """
# 
#         :param string selection: The CSS selector or XPath expression used to select the content to scrape.
#         :param bool xpath: (Optional) Whether selection is an XPath expression instead of a CSS selector. Default False.
#         :param re: (Optional) Regular expression to apply to scraped content.
#         :param bool all: (Optional) Whether to scrape all occurrences instead of just the first. Default False.
#         :param default: (Optional) The default value for this field if none is set.
#         :param bool null: (Optional) Include in serialized output even if value is None. Default False.
#         :param bool raw: (Optional) Whether to scrape the raw HTML/XML instead of the text contents. Default False.
#         """
#         self.selection = selection
#         self.xpath = xpath
#         self.re = re
#         self.all = all
#         self.default = default
#         self.null = null
#         self.raw = raw
# 
#     def __get__(self, instance, owner):
#         """Descriptor for retrieving a value from a field in an Entity."""
#         # Check if Entity class is being called, rather than Entity instance
#         if instance is None:
#             return self
#         # Get value from Entity instance if available
#         value = instance._values.get(self.name)
#         # If value is None, empty list or empty string return the default value if set
#         if value in [None, [], ''] and self.default is not None:
#             return self.default
#         # Otherwise if value is None and all, return empty list
#         if self.all and value is None:
#             return []
#         return value
# 
#     def __set__(self, instance, value):
#         """Descriptor for assigning a value to a field in a Entity."""
#         instance._values[self.name] = value
# 
#     def _post_scrape(self, value, processor=None):
#         """Apply processing to the scraped value."""
#         # Pass each value through the field's clean method
#         value = [self.process(v) for v in value]
#         # Filter None values
#         value = [v for v in value if v is not None]
#         # Pass each value through processors defined on the entity
#         if processor:
#             value = [processor(v) for v in value]
#             value = [v for v in value if v is not None]
#         # Take first unless all is specified
#         if not self.all:
#             value = value[0] if value else None
#         log.debug('Scraped %s: %s from %s' % (self.name, value, self.selection))
#         return value
# 
#     def scrape(self, selector, cleaner=None, processor=None):
#         """Scrape the value for this field from the selector."""
#         # Apply CSS or XPath expression to the selector
#         selected = selector.xpath(self.selection) if self.xpath else selector.css(self.selection)
#         # Extract the value and apply regular expression if specified
#         value = selected.re(self.re) if self.re else selected.extract(raw=self.raw, cleaner=cleaner)
#         return self._post_scrape(value, processor=processor)
# 
#     def serialize(self, value):
#         """Serialize this field."""
#         if hasattr(value, 'serialize'):
#             return value.serialize()
#         else:
#             return value
# 
#     def process(self, value):
#         """Override to perform custom processing of a value."""
#         return value
# 
# Path: chemdataextractor/text/processors.py
# def strip_querystring(url):
#     """Remove the querystring from the end of a URL."""
#     p = six.moves.urllib.parse.urlparse(url)
#     return p.scheme + "://" + p.netloc + p.path


# Path: chemdataextractor/scrape/fields.py
import logging
import re
import dateutil.parser
import six
from .base import BaseField
from ..text.processors import strip_querystring



class EntityField(BaseField):
    """A field that contains another Entity."""

    def __init__(self, entity, selection, **kwargs):
        """

        :param entity: The embedded entity.
        """
        self.entity = entity
        super(EntityField, self).__init__(selection, **kwargs)

    def scrape(self, selector, cleaner=None, processor=None):
        """Scrape the value for this field from the selector."""
        value = self.entity.scrape(selector, root=self.selection, xpath=self.xpath)
        return self._post_scrape(value, processor=processor)


class IntField(BaseField):
    """An integer number field."""

    def process(self, value):
        """Convert value to an int."""
        try:
            return int(value)
        except (ValueError, TypeError):
            return None



class FloatField(BaseField):