# Path: brewery/metadata.py
# def collapse_record(record, separator = '.', root = None):
#     """See :func:`brewery.expand_record` for reverse operation.
#     """
# 
#     result = {}
#     for key, value in record.items():
#         if root:
#             collapsed_key = root + separator + key
#         else:
#             collapsed_key = key
# 
#         if type(value) == dict:
#             collapsed = collapse_record(value, separator, collapsed_key)
#             result.update(collapsed)
#         else:
#             result[collapsed_key] = value
#     return result
# 
# class Field(object):
#     """Metadata - information about a field in a dataset or in a datastream.
# 
#     :Attributes:
#         * `name` - field name
#         * `label` - optional human readable field label
#         * `storage_type` - Normalized data storage type. The data storage type
#           is abstracted
#         * `concrete_storage_type` (optional, recommended) - Data store/database
#           dependent storage type - this is the real name of data type as used
#           in a database where the field comes from or where the field is going
#           to be created (this might be null if unknown)
#         * `analytical_type` - data type used in data mining algorithms
#         * `missing_values` (optional) - Array of values that represent missing
#           values in the dataset for given field
#     """
# 
#     def __init__(self, name, storage_type="unknown",
#                  analytical_type="typeless", concrete_storage_type=None,
#                  missing_values=None, label=None):
#         self.name = name
#         self.label = label
#         self.storage_type = storage_type
#         self.analytical_type = analytical_type
#         self.concrete_storage_type = concrete_storage_type
#         self.missing_values = missing_values
# 
#     def to_dict(self):
#         """Return dictionary representation of the field."""
#         d = {
#                 "name": self.name,
#                 "label": self.label,
#                 "storage_type": self.storage_type,
#                 "analytical_type": self.analytical_type,
#                 "concrete_storage_type": self.concrete_storage_type,
#                 "missing_values": self.missing_values
#             }
#         return d
# 
#     def __str__(self):
#         """Return field name as field string representation."""
# 
#         return self.name
# 
#     def __repr__(self):
#         return "<%s(%s)>" % (self.__class__, self.to_dict())
# 
#     def __eq__(self, other):
#         if self is other:
#             return True
#         if self.name != other.name or self.label != other.label:
#             return False
#         elif self.storage_type != other.storage_type or self.analytical_type != other.analytical_type:
#             return False
#         elif self.concrete_storage_type != other.concrete_storage_type:
#             return False
#         elif self.missing_values != other.missing_values:
#             return False
#         else:
#             return True
# 
#     def __ne__(self,other):
#         return not self.__eq__(other)


import urllib2
import urlparse
import brewery.dq
from brewery.metadata import collapse_record, Field
        def probe_record(record, parent = None):
            for key, value in record.items():
                full_key = parent + "." + key if parent else key

                if self.expand and type(value) == dict:
                    probe_record(value, full_key)
                    continue

                if not full_key in probes:
                    probe = brewery.dq.FieldTypeProbe(full_key)
                    probes[full_key] = probe
                    keys.append(full_key)
                else:
                    probe = probes[full_key]
                probe.probe(value)

        count = 0
        for record in self.records():
            if collapse:
                record = collapse_record(record)

            probe_record(record)
            if limit and count >= limit:
                break
            count += 1

        fields = []

        for key in keys:
            probe = probes[key]

# Path: brewery/metadata.py
# def collapse_record(record, separator = '.', root = None):
#     """See :func:`brewery.expand_record` for reverse operation.
#     """
# 
#     result = {}
#     for key, value in record.items():
#         if root:
#             collapsed_key = root + separator + key
#         else:
#             collapsed_key = key
# 
#         if type(value) == dict:
#             collapsed = collapse_record(value, separator, collapsed_key)
#             result.update(collapsed)
#         else:
#             result[collapsed_key] = value
#     return result
# 
# class Field(object):
#     """Metadata - information about a field in a dataset or in a datastream.
# 
#     :Attributes:
#         * `name` - field name
#         * `label` - optional human readable field label
#         * `storage_type` - Normalized data storage type. The data storage type
#           is abstracted
#         * `concrete_storage_type` (optional, recommended) - Data store/database
#           dependent storage type - this is the real name of data type as used
#           in a database where the field comes from or where the field is going
#           to be created (this might be null if unknown)
#         * `analytical_type` - data type used in data mining algorithms
#         * `missing_values` (optional) - Array of values that represent missing
#           values in the dataset for given field
#     """
# 
#     def __init__(self, name, storage_type="unknown",
#                  analytical_type="typeless", concrete_storage_type=None,
#                  missing_values=None, label=None):
#         self.name = name
#         self.label = label
#         self.storage_type = storage_type
#         self.analytical_type = analytical_type
#         self.concrete_storage_type = concrete_storage_type
#         self.missing_values = missing_values
# 
#     def to_dict(self):
#         """Return dictionary representation of the field."""
#         d = {
#                 "name": self.name,
#                 "label": self.label,
#                 "storage_type": self.storage_type,
#                 "analytical_type": self.analytical_type,
#                 "concrete_storage_type": self.concrete_storage_type,
#                 "missing_values": self.missing_values
#             }
#         return d
# 
#     def __str__(self):
#         """Return field name as field string representation."""
# 
#         return self.name
# 
#     def __repr__(self):
#         return "<%s(%s)>" % (self.__class__, self.to_dict())
# 
#     def __eq__(self, other):
#         if self is other:
#             return True
#         if self.name != other.name or self.label != other.label:
#             return False
#         elif self.storage_type != other.storage_type or self.analytical_type != other.analytical_type:
#             return False
#         elif self.concrete_storage_type != other.concrete_storage_type:
#             return False
#         elif self.missing_values != other.missing_values:
#             return False
#         else:
#             return True
# 
#     def __ne__(self,other):
#         return not self.__eq__(other)


# Path: brewery/ds/base.py
import urllib2
import urlparse
import brewery.dq
from brewery.metadata import collapse_record, Field

        def probe_record(record, parent = None):
            for key, value in record.items():
                full_key = parent + "." + key if parent else key

                if self.expand and type(value) == dict:
                    probe_record(value, full_key)
                    continue

                if not full_key in probes:
                    probe = brewery.dq.FieldTypeProbe(full_key)
                    probes[full_key] = probe
                    keys.append(full_key)
                else:
                    probe = probes[full_key]
                probe.probe(value)

        count = 0
        for record in self.records():
            if collapse:
                record = collapse_record(record)

            probe_record(record)
            if limit and count >= limit:
                break
            count += 1

        fields = []

        for key in keys:
            probe = probes[key]

            field = Field(probe.field)