<commit_before>
import json
import os
from urlparse import urlparse
import zipfile 

import click

import adapters
from filters import BasicFilterer
import utils
import subprocess

def get_data(urls, folder): 
    # Create folder 
    command = 'mkdir ' + folder 
    subprocess.call(command,shell=True) 
    
    for url in urls: 
        name = url.split('/')[-1]
        string = folder + name
        # Download the data 
        command = 'curl -o ' + string + ' "' + url + '"' 
        subprocess.call(command,shell=True) 
        #Unzip the data 
        command = 'unzip ' + string + ' -d ' + folder 
        subprocess.call(command, shell=True) 
        #Delete the archive 
        subprocess.call('rm ' + string, shell=True)

def translate_to_geojson(shps, folder): 
    """ Function that create a geojson file 
    from a shp file and reproject to WSG84 using ogr2ogr tool 
    """
    projection = '-t_srs EPSG:4326'
    fmt = '-f GeoJSON'
    output = []
    for shp in shps:   
        src = folder + shp
        dst = folder + shp.split('.')[0] + '.geojson'    
        command = 'ogr2ogr ' + '-progress ' + '-overwrite ' + fmt + ' ' + projection + ' ' +  dst + ' ' + src 
        subprocess.call(command,shell=True)
        output.append(dst)
    return output 

def tiling(files, folder): 
    """ Use of Tippecanoe 
    """
    string = '' 
    for file in files: 
        string += file + ' '
    command = 'tippecanoe -f -o ' + folder + 'result' + '.mbtiles ' + string  #Default zoom 0 to 14 
    subprocess.call(command,shell=True)

def cleaning(folder): 
    """ Clean the folder except the .mbtiles 
    """
    files = [each for each in os.listdir(folder) if not each.endswith('.mbtiles')] 
    for file in files: 
        subprocess.call('rm ' + folder + '/' + file, shell=True)

# To make it a command line script 
<EMAIL>()
#@click.argument('path', type=click.Path(exists=True), required=True)
def vectorTiling(path):
    """ Function that creates vector tiles
    PATH: either a .json file containing a "url" field that contains the geojson files 
            TODO -> a directory containing more than one of this .json file
    """
    contents = []
    urls = []
    # Input is a .json file 
    if path.endswith('.json'): 
        with open(path, 'rb') as file: 
            content = json.load(file)
            urls.append(content['url'])
            contents.append(content)
    # Input is a directory    
    else: 
        for (path, dirs, files) in os.walk(path): 
            for file in files: 
                if file.endswith('.json'): 
                    uri = path + file 
                    with open(uri,'rb') as f: 
                        content = json.load(f)
                        urls.append(content['url'])
                        contents.append(content)


    # Where all the data will be downloaded - unzipped - processed 
    folder = 'VectorData/'
    urls = list(set(urls)) # A set can't contain duplicate 
    print urls 
    # Get all the data from the url(s)
    get_data(urls, folder)
    # From .shp to .geojson 
    shapefiles = [each for each in os.listdir(folder) if each.endswith('.shp')]
    geojson_files = translate_to_geojson(shapefiles, folder)
    # Tiling the .geojson 
    tiling(geojson_files, folder)
    # Cleaning the directory 
    cleaning(folder)


if __name__ == '__main__':
    example = '/Users/athissen/Documents/GaiaGps/PublicLands/sources/federal/'
    vectorTiling(example)
    <commit_msg>Better understanding of the issue<commit_after>
import json
import os
from urlparse import urlparse
import zipfile 

import click

import adapters
from filters import BasicFilterer
import utils
import subprocess
 
@click.command()
@click.argument('file', type=click.Path(exists=True), required=True)
def vectorTiling(file):
    """ Function that creates vector tiles
    INPUT: geojson file generated by process.py 
    OUTPUT: mbtiles file containing vector tiles 
    """
    # Variables to change in production 
    path = '/Users/athissen/Documents/'

    min_zoom = 0 
    max_zoom = 14   
    paths_string = ''

    with open(file, 'rb') as f: 
        geojson = json.load(f)
    features = geojson['features']
    for item in features: 
        paths_string += path + item['properties']['path'] + ' '

    command = 'tippecanoe -f -o ' + 'result.mbtiles ' + paths_string + ' -z {} -Z {}'.format(max_zoom, min_zoom)
    subprocess.call(command,shell=True)


if __name__ == '__main__':

    vectorTiling()
    
{'fraction_non_alphanumeric': 0.11271676300578035, 'fraction_numerical': 0.0035571365051133837, 'mean_word_length': 2.408333333333333, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2434098', 'n_tokens_mistral': 1434, 'n_tokens_neox': 1336, 'n_words': 409}