<commit_before>import sys
import pytz
import base64

from copy import deepcopy
import datetime
from fam.extra_types.lat_long import LatLong
from fractions import Fraction
from decimal import Decimal
from google.cloud.firestore_v1beta1 import GeoPoint


if sys.version_info[0] < 3:
    PYTHON_VERSION = 2
else:
    PYTHON_VERSION = 3


class BaseDataAdapter(object):

    # a fam doc to serialise into firestore
    def serialise(self, doc):
        dup = deepcopy(doc)
        return self._serialise_walk(dup)

    def deserialise(self, doc):
        dup = deepcopy(doc)
        return self._deserialise_walk(dup)


    def is_a_string(self, node):

        if PYTHON_VERSION == 3:
            return isinstance(node, str)
        else:
            return isinstance(node, basestring)


    def is_a_number(self, node):

        if PYTHON_VERSION == 3:
            return isinstance(node, int) or isinstance(node, float)
        else:
            return isinstance(node, int) or isinstance(node, float) or isinstance(node, long)


    def _serialise_walk(self, node):

        if isinstance(node, dict):
            for k, v in node.items():
                node[k] = self._serialise_walk(v)
            return node
        if isinstance(node, list):
            return [self._serialise_walk(v) for v in node]
        if isinstance(node, LatLong):
            return self.serialise_lat_long(node)
        if isinstance(node, Fraction):
            return self.serialise_fraction(node)
        if isinstance(node, Decimal):
            return self.serialise_decimal(node)
        if isinstance(node, datetime.datetime):
            return self.serialise_date_time(node)
        if self.is_a_string(node):
            return self.serialise_string(node)
        if isinstance(node, bytes):
            return self.serialise_bytes(node)
        if isinstance(node, bool):
            return self.serialise_bool(node)
        if self.is_a_number(node):
            return self.serialise_number(node)
        if hasattr(node, "to_json"):
            return self.serialise_object(node)
        raise Exception("FirestoreDataAdapter can't serialise this value: %s", node)


    def serialise_lat_long(self, lat_long):
        return "::latlong::%s,%s" % (lat_long.latitude, lat_long.longitude)

    def serialise_fraction(self, fraction):
        return "::fraction::%s/%s" % (fraction.numerator, fraction.denominator)

    def serialise_decimal(self, decimal):
        return "::decimal::%s" % str(decimal)

    def serialise_date_time(self, date_time):
        utc = pytz.utc
        if date_time.tzinfo is None:
            dt = date_time.replace(tzinfo=utc)
        else:
            dt = date_time.astimezone(utc)
        as_iso = dt.isoformat("T") + "Z"
        fixed = as_iso.replace("+00:00", "")
        return "::datetime::%s" % str(fixed)


    def serialise_bytes(self, btes):
        encoded = base64.b64encode(btes)
        result = "::bytes::%s" % encoded.decode("utf-8")
        return result<commit_msg>fixing NOne in base adapter<commit_after>import sys
import pytz
import base64

from copy import deepcopy
import datetime
from fam.extra_types.lat_long import LatLong
from fractions import Fraction
from decimal import Decimal
from google.cloud.firestore_v1beta1 import GeoPoint


if sys.version_info[0] < 3:
    PYTHON_VERSION = 2
else:
    PYTHON_VERSION = 3


class BaseDataAdapter(object):

    # a fam doc to serialise into firestore
    def serialise(self, doc):
        dup = deepcopy(doc)
        return self._serialise_walk(dup)

    def deserialise(self, doc):
        dup = deepcopy(doc)
        return self._deserialise_walk(dup)


    def is_a_string(self, node):

        if PYTHON_VERSION == 3:
            return isinstance(node, str)
        else:
            return isinstance(node, basestring)


    def is_a_number(self, node):

        if PYTHON_VERSION == 3:
            return isinstance(node, int) or isinstance(node, float)
        else:
            return isinstance(node, int) or isinstance(node, float) or isinstance(node, long)


    def _serialise_walk(self, node):

        if isinstance(node, dict):
            for k, v in node.items():
                node[k] = self._serialise_walk(v)
            return node
        if isinstance(node, list):
            return [self._serialise_walk(v) for v in node]
        if isinstance(node, LatLong):
            return self.serialise_lat_long(node)
        if isinstance(node, Fraction):
            return self.serialise_fraction(node)
        if isinstance(node, Decimal):
            return self.serialise_decimal(node)
        if isinstance(node, datetime.datetime):
            return self.serialise_date_time(node)
        if self.is_a_string(node):
            return self.serialise_string(node)
        if isinstance(node, bytes):
            return self.serialise_bytes(node)
        if isinstance(node, bool):
            return self.serialise_bool(node)
        if self.is_a_number(node):
            return self.serialise_number(node)
        if hasattr(node, "to_json"):
            return self.serialise_object(node)
        if node is None:
            return None

        raise Exception("FirestoreDataAdapter can't deserialise this value: %s" % node)


    def serialise_lat_long(self, lat_long):
        return "::latlong::%s,%s" % (lat_long.latitude, lat_long.longitude)

    def serialise_fraction(self, fraction):
        return "::fraction::%s/%s" % (fraction.numerator, fraction.denominator)

    def serialise_decimal(self, decimal):
        return "::decimal::%s" % str(decimal)

    def serialise_date_time(self, date_time):
        utc = pytz.utc
        if date_time.tzinfo is None:
            dt = date_time.replace(tzinfo=utc)
        else:
            dt = date_time.astimezone(utc)
        as_iso = dt.isoformat("T") + "Z"
        fixed = as_iso.replace("+00:00", "")
        return "::datetime::%s" % str(fixed)


    def serialise_bytes(self, btes):
        encoded = base64.b64encode(btes)
        result = "::bytes::%s" % encoded.decode("utf-8")
        return result
{'fraction_non_alphanumeric': 0.09899852241011328, 'fraction_numerical': 0.006238712855032014, 'mean_word_length': 2.5480489225393126, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '342553', 'n_tokens_mistral': 1913, 'n_tokens_neox': 1831, 'n_words': 451}