<commit_before>#!/usr/bin/python
# -*- coding: utf8 -*-

import os
import psycopg2
import time
import config
import argparse



def download_osm():
    import urllib
    from urllib import quote_plus
    with open('mostrans-newbuses2016.txt', 'r') as myfile:
        overpass_query=myfile.read()
    urllib.urlretrieve(overpass_query, "data.osm")



'''
  relation
    ["ref"="309"]
    ["payment:troika"="yes"]
    (55.597747184319935,37.354888916015625,55.94458588614092,38.06350708007812);
  relation
    ["ref"="346"]
    ["payment:troika"="yes"]
    (55.597747184319935,37.354888916015625,55.94458588614092,38.06350708007812);

  relation
    ["ref"="361"]
    ["payment:troika"="yes"]
    (55.597747184319935,37.354888916015625,55.94458588614092,38.06350708007812);


'''
'''
bin/osmosis \
  --read-pbf ~/Downloads/RU-SPE.osm.pbf \
  --tf accept-relations route=bus,trolleybus,tram \
  --used-way --used-node \
  --write-pbf ~/Downloads/filtered.osm.pbf

bin/osmosis \
  --read-pbf ~/Downloads/filtered.osm.pbf \
  --tf accept-relations ref=3М,3Мб \
  --used-way --used-node \
  --write-pbf ~/Downloads/filtered-numbers.osm.pbf

'''

def download_osm_dump():
    os.system('wget -N http://data.gis-lab.info/osm_dump/dump/latest/RU-MOW.osm.pbf')

    import json
    import pprint
    pp=pprint.PrettyPrinter(indent=2)

    refs=[]

    with open('cfg/moscow_newmodel_bus_routes_2016.json') as data_file:    
        data = json.load(data_file)

    for batches in data['dataset'].values():
        for batch in batches:
            for ref in batch['ref']:
                refs.append(ref)

    refsString=','.join(refs)
    

    cmd='''
~/osmosis/bin/osmosis \
  -q \
  --read-pbf RU-MOW.osm.pbf \
  --tf accept-relations route=bus \
  --used-way --used-node \
  --write-pbf routes.osm.pbf
'''
    os.system(cmd)

    cmd='''
~/osmosis/bin/osmosis \
  -q  \
  --read-pbf routes.osm.pbf \
  --tf accept-relations ref="'''+refsString.encode("UTF-8")+'''" \
  --used-way --used-node \
  --write-pbf routes2.osm.pbf
    '''
    #print cmd
    #quit()
    os.system(cmd)

    cmd='''
~/osmosis/bin/osmosis \
  -q \
  --read-pbf routes2.osm.pbf \
  --tf accept-relations "payment:troika=yes" \
  --used-way --used-node \
  --write-pbf routesFinal.osm.pbf
    '''
    os.system(cmd)


<commit_msg>gis-lab broken, оперативное переключение to overpass<commit_after>#!/usr/bin/python
# -*- coding: utf8 -*-

import os
import psycopg2
import time
import config
import argparse



def download_osm():
    import urllib
    from urllib import quote_plus
    with open('mostrans-newbuses2016.txt', 'r') as myfile:
        overpass_query=myfile.read()
    urllib.urlretrieve(overpass_query, "data.osm")



'''
  relation
    ["ref"="309"]
    ["payment:troika"="yes"]
    (55.597747184319935,37.354888916015625,55.94458588614092,38.06350708007812);
  relation
    ["ref"="346"]
    ["payment:troika"="yes"]
    (55.597747184319935,37.354888916015625,55.94458588614092,38.06350708007812);

  relation
    ["ref"="361"]
    ["payment:troika"="yes"]
    (55.597747184319935,37.354888916015625,55.94458588614092,38.06350708007812);


'''
'''
bin/osmosis \
  --read-pbf ~/Downloads/RU-SPE.osm.pbf \
  --tf accept-relations route=bus,trolleybus,tram \
  --used-way --used-node \
  --write-pbf ~/Downloads/filtered.osm.pbf

bin/osmosis \
  --read-pbf ~/Downloads/filtered.osm.pbf \
  --tf accept-relations ref=3М,3Мб \
  --used-way --used-node \
  --write-pbf ~/Downloads/filtered-numbers.osm.pbf

'''

def download_osm_dump():
    os.system('wget -N http://data.gis-lab.info/osm_dump/dump/latest/RU-MOW.osm.pbf')

    import json
    import pprint
    pp=pprint.PrettyPrinter(indent=2)

    refs=[]

    with open('cfg/moscow_newmodel_bus_routes_2016.json') as data_file:    
        data = json.load(data_file)

    for batches in data['dataset'].values():
        for batch in batches:
            for ref in batch['ref']:
                refs.append(ref)

    refsString=','.join(refs)
    

    cmd='''
~/osmosis/bin/osmosis \
  -q \
  --read-pbf RU-MOW.osm.pbf \
  --tf accept-relations route=bus \
  --used-way --used-node \
  --write-pbf routes.osm.pbf
'''
    os.system(cmd)

    cmd='''
~/osmosis/bin/osmosis \
  -q  \
  --read-pbf routes.osm.pbf \
  --tf accept-relations ref="'''+refsString.encode("UTF-8")+'''" \
  --used-way --used-node \
  --write-pbf routes2.osm.pbf
    '''
    #print cmd
    #quit()
    os.system(cmd)

    cmd='''
~/osmosis/bin/osmosis \
  -q \
  --read-pbf routes2.osm.pbf \
  --tf accept-relations "payment:troika=yes" \
  --used-way --used-node \
  --write-pbf routesFinal.osm.pbf
    '''
    os.system(cmd)

def download_osm_overpass():
    def makeOverpassQuery(currentmap):

        data=  {'data':  '''[out:xml][timeout:55];(relation["route"="bus"](55.5977,37.3548,55.9440,38.0635);out meta;>;out meta qt;'''}
        print 'new '+urllib.unquote(urllib.urlencode(data)).decode('utf8')
        #return  'http://overpass.osm.rambler.ru/cgi/interpreter?'+urllib.urlencode(data)
        return  'http://overpass-api.de/api/interpreter?'+urllib.urlencode(data)
    
    #Make overpass-api query
    overpass_query=makeOverpassQuery(currentmap)
    #print overpass_query
            
    #Do overpass query
    osmFileHandler='data.osm'

    urllib.urlretrieve(overpass_query,osmFileHandler)
    

    import json
    import pprint
    pp=pprint.PrettyPrinter(indent=2)

    refs=[]

    with open('cfg/moscow_newmodel_bus_routes_2016.json') as data_file:    
        data = json.load(data_file)

    for batches in data['dataset'].values():
        for batch in batches:
            for ref in batch['ref']:
                refs.append(ref)

    refsString=','.join(refs)
    

    cmd='''
~/osmosis/bin/osmosis \
  -q \
  --read-xml data.osm \
  --tf accept-relations route=bus \
  --used-way --used-node \
  --write-pbf routes.osm.pbf
'''
    os.system(cmd)

    cmd='''
~/osmosis/bin/osmosis \
  -q  \
  --read-pbf routes.osm.pbf \
  --tf accept-relations ref="'''+refsString.encode("UTF-8")+'''" \
  --used-way --used-node \
  --write-pbf routes2.osm.pbf
    '''
    #print cmd
    #quit()
    os.system(cmd)

    cmd='''
~/osmosis/bin/osmosis \
  -q \
  --read-pbf routes2.osm.pbf \
  --tf accept-relations "payment:troika=yes" \
  --used-way --used-node \
  --write-pbf routesFinal.osm.pbf
    '''
    os.system(cmd)


{'fraction_non_alphanumeric': 0.17312539382482672, 'fraction_numerical': 0.07577189666036548, 'mean_word_length': 2.6593659942363113, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6994423', 'n_tokens_mistral': 2902, 'n_tokens_neox': 2533, 'n_words': 369}