<commit_before># coding=utf-8
import ee
from . import utils
import json
import csv
from .. import tools


def fromShapefile(filename, crs=None, start=None, end=None):
    """ Convert an ESRI file (.shp and .dbf must be present) to a
    ee.FeatureCollection

    At the moment only works for shapes with less than 1000 records and doesn't
    handle complex shapes.

    :param filename: the name of the filename. If the shape is not in the
        same path than the script, specify a path instead.
    :type filename: str
    :param start:
    :return: the FeatureCollection
    :rtype: ee.FeatureCollection
    """
    import shapefile

    wgs84 = ee.Projection('EPSG:4326')
    # read the filename
    reader = shapefile.Reader(filename)
    fields = reader.fields[1:]
    field_names = [field[0] for field in fields]
    field_types = [field[1] for field in fields]
    types = dict(zip(field_names, field_types))
    features = []

    projection = utils.getProjection(filename) if not crs else crs
    # catch a string with format "EPSG:XXX"
    if isinstance(projection, str):
        if 'EPSG:' in projection:
            projection = projection.split(':')[1]
    projection = 'EPSG:{}'.format(projection)

    # filter records with start and end
    start = start if start else 0
    if not end:
        records = reader.shapeRecords()
        end = len(records)
    else:
        end = end + 1

    if (end-start)>1000:
        msg = "Can't process more than 1000 records at a time. Found {}"
        raise ValueError(msg.format(end-start))

    for i in range(start, end):
        # atr = dict(zip(field_names, sr.record))
        sr = reader.shapeRecord(i)
        atr = {}
        for fld, rec in zip(field_names, sr.record):
            fld_type = types[fld]
            if fld_type == 'D':
                value = ee.Date(rec.isoformat()).millis().getInfo()
            elif fld_type in ['C', 'N', 'F']:
                value = rec
            else:
                continue
            atr[fld] = value
        geom = sr.shape.__geo_interface__
        if projection is not None:
            geometry = ee.Geometry(geom, projection) \
                .transform(wgs84, 1)
        else:
            geometry = ee.Geometry(geom)
        feat = ee.Feature(geometry, atr)
        features.append(feat)

    return ee.FeatureCollection(features)


def fromGeoJSON(filename=None, data=None, crs=None):
    """ Create a list of Features from a GeoJSON file. Return a python tuple
    with ee.Feature inside. This is due to failing when attempting to create a
    FeatureCollection (Broken Pipe ERROR) out of the list. You can try creating
    it yourself casting the result of this function to a ee.List or using it
    directly as a FeatureCollection argument.

    :param filename: the name of the file to load
    :type filename: str
    :param crs: a coordinate reference system in EPSG format. If not specified
        it will try to get it from the geoJSON, and if not there it will rise<commit_msg>fix `fromShapefile` to accept empty records<commit_after># coding=utf-8
import ee
from . import utils
import json
import csv
from .. import tools


def fromShapefile(filename, crs=None, start=None, end=None):
    """ Convert an ESRI file (.shp and .dbf must be present) to a
    ee.FeatureCollection

    At the moment only works for shapes with less than 1000 records and doesn't
    handle complex shapes.

    :param filename: the name of the filename. If the shape is not in the
        same path than the script, specify a path instead.
    :type filename: str
    :param start:
    :return: the FeatureCollection
    :rtype: ee.FeatureCollection
    """
    import shapefile

    wgs84 = ee.Projection('EPSG:4326')
    # read the filename
    reader = shapefile.Reader(filename)
    fields = reader.fields[1:]
    field_names = [field[0] for field in fields]
    field_types = [field[1] for field in fields]
    types = dict(zip(field_names, field_types))
    features = []

    projection = utils.getProjection(filename) if not crs else crs
    # catch a string with format "EPSG:XXX"
    if isinstance(projection, str):
        if 'EPSG:' in projection:
            projection = projection.split(':')[1]
    projection = 'EPSG:{}'.format(projection)

    # filter records with start and end
    start = start if start else 0
    if not end:
        records = reader.shapeRecords()
        end = len(records)
    else:
        end = end + 1

    if (end-start)>1000:
        msg = "Can't process more than 1000 records at a time. Found {}"
        raise ValueError(msg.format(end-start))

    for i in range(start, end):
        # atr = dict(zip(field_names, sr.record))
        sr = reader.shapeRecord(i)
        atr = {}
        for fld, rec in zip(field_names, sr.record):
            if rec is None:
                atr[fld] = None
                continue
            fld_type = types[fld]
            if fld_type == 'D':
                value = ee.Date(rec.isoformat()).millis().getInfo()
            elif fld_type in ['C', 'N', 'F']:
                value = rec
            else:
                continue
            atr[fld] = value
        geom = sr.shape.__geo_interface__
        if projection is not None:
            geometry = ee.Geometry(geom, projection) \
                .transform(wgs84, 1)
        else:
            geometry = ee.Geometry(geom)
        feat = ee.Feature(geometry, atr)
        features.append(feat)

    return ee.FeatureCollection(features)


def fromGeoJSON(filename=None, data=None, crs=None):
    """ Create a list of Features from a GeoJSON file. Return a python tuple
    with ee.Feature inside. This is due to failing when attempting to create a
    FeatureCollection (Broken Pipe ERROR) out of the list. You can try creating
    it yourself casting the result of this function to a ee.List or using it
    directly as a FeatureCollection argument.

    :param filename: the name of the file to load
    :type filename: str
    :param crs: a coordinate reference system in EPSG format. If not specified
        it will try to get it from the geoJSON, and if not there it will rise
{'fraction_non_alphanumeric': 0.08719700666992028, 'fraction_numerical': 0.009110135025215552, 'mean_word_length': 2.400442477876106, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7040381', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1769, 'n_words': 669}