<jupyter_start><jupyter_text>Data Feature  osmWay<jupyter_code>sqlContext.setConf("spark.sql.parquet.binaryAsString","true")
osmWay = sqlContext.read.parquet("../data/20190531-hungary.osm.pbf.way.parquet")
osmWay = osmWay.select('id','tags','nodes')
osmWay.createOrReplaceTempView("osmWay")



osmWay.printSchema()
osmWay.head()
osmWay.show(5)
osmWay.count()<jupyter_output><empty_output><jupyter_text>osmNode<jupyter_code>osmNode = sqlContext.read.parquet("../data/20190531-hungary.osm.pbf.node.parquet")
osmNode = osmNode.select('tags', 'latitude','longitude')
osmNode.createOrReplaceTempView("osmNode")
osmNode.printSchema()
osmNode.head()
osmNode.show(5)
osmNode.count()
sqlContext.sql("SELECT tags,latitude, longitude FROM osmNode \
WHERE tags[0].key='amenity' AND tags[0].value='taxi'").show()<jupyter_output>+--------------------+------------------+------------------+
|                tags|          latitude|         longitude|
+--------------------+------------------+------------------+
|[[amenity, taxi],...|        47.5470381|         19.028375|
|   [[amenity, taxi]]|        47.4982064|        19.0704767|
|   [[amenity, taxi]]|47.496360800000005|19.070999500000003|
|   [[amenity, taxi]]|47.507562500000006|        19.0729668|
|   [[amenity, taxi]]|47.499501300000006|        19.0646431|
|   [[amenity, taxi]]|47.782211200000006|        19.1324766|
|   [[amenity, taxi]]|        47.5420646|        19.1225384|
|   [[amenity, taxi]]|        48.2466887|        20.6166425|
|   [[amenity, taxi]]|46.353023300000004|        17.7953511|
|   [[amenity, taxi]]|46.075580200000005|        18.2048794|
|[[amenity, taxi],...|47.511096200000004|19.080156300000002|
|   [[amenity, taxi]]|        47.9000818|        20.3767527|
|   [[amenity, taxi]]|46.370169600000004|18.149269800000003|
|   [[amenity, taxi]]|47[...]<jupyter_text>osmRelation<jupyter_code>osmRelation = sqlContext.read.parquet("../data/20190531-hungary.osm.pbf.relation.parquet")
osmRelation = osmRelation.select('id','tags','members')
osmRelation.createOrReplaceTempView("osmRelation")
osmRelation.printSchema()
osmRelation.head()
osmRelation.show(5)
osmRelation.count()<jupyter_output><empty_output><jupyter_text>Data Explore  Distribution of Different Type of Highway<jupyter_code>highway = sqlContext.sql("SELECT osmWay.tags[0].value as Value,COUNT(osmWay.tags[0].value) as Count FROM osmWay WHERE tags[0].key='highway' GROUP BY osmWay.tags[0].value")
highway.count()
foundDatas = []
allRows = sqlContext.sql("SELECT id,tags FROM osmWay")
for row in allRows.collect():
    for item in row["tags"]:
        if item.key == "highway":
            foundDatas.append(item.value)
counter = Counter(foundDatas)
result = [(counter[key], key) for key in counter]
highwaydf = pd.DataFrame(list(result),columns=['Count','Value'])
highway = highwaydf.sort_values(by=['Count'],ascending=False)
highway = highway.head()
highway
plt.pie(
    highway['Count'],
    labels=highway['Value'],
    shadow=True,
    explode=(0.1, 0, 0, 0, 0),
    startangle=90,
    autopct='%1.1f%%',
    pctdistance = 0.5,
)

plt.axis('equal')
fig = plt.gcf()
fig.set_size_inches(8,8)<jupyter_output><empty_output><jupyter_text>The Most Common Amenities<jupyter_code>amenities = sqlContext.sql("SELECT osmWay.tags[0].value as Value,COUNT(osmWay.tags[0].value) as Count FROM osmWay WHERE tags[0].key='amenity' GROUP BY osmWay.tags[0].value")
amenities.count()
foundDatas = []
allRows = sqlContext.sql("SELECT id,tags FROM osmWay")
for row in allRows.collect():
    for item in row["tags"]:
        if item.key == "amenity":
            foundDatas.append(item.value)
counter = Counter(foundDatas)
result = [(counter[key], key) for key in counter]
amenitydf = pd.DataFrame(list(result),columns=['Count','Value'])
amenity = amenitydf.sort_values(by=['Count'],ascending=False)
amenitiesdf = amenity.sort_values(by=['Count'],ascending=False)
amenitiesdf.head()
ax = amenitiesdf[:5].plot(kind='bar',x='Value', y='Count',title ="Amenities", figsize=(15, 10), legend=True, fontsize=12)
ax.set_xlabel("Value", fontsize=12)
ax.set_ylabel("Count", fontsize=12)
plt.show()<jupyter_output><empty_output><jupyter_text>Number of Taxi Stop and Location Each of Them<jupyter_code>taxiLocation = sqlContext.sql("SELECT latitude,longitude FROM osmNode") \
.where((array_contains(osmNode.tags.key, 'amenity') & (array_contains(osmNode.tags.value,'taxi'))))

taxiLocation.take(5)

taxiLocation = sqlContext.sql("SELECT latitude,longitude FROM osmNode") \
.where((array_contains(osmNode.tags.key, 'amenity') & (array_contains(osmNode.tags.value,'taxi')))).toPandas()
taxiLocation.count()
taxidf = pd.DataFrame(data=taxiLocation)
taxidf.head()
taxiMap = folium.Map([taxiLocation.latitude[0], taxiLocation.longitude[0]], zoom_start=7)
for index, row in taxidf.iterrows():
    folium.CircleMarker([row['latitude'], row['longitude']],
                        radius=15,
                        popup=row[['latitude','longitude']],
                        fill_color="#e4d93d",
                       ).add_to(taxiMap)
taxiMap<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18634030793525463, 'fraction_numerical': 0.07777339123568891, 'mean_word_length': 4.390425531914894, 'pattern_counts': {'":': 2, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 11, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '171269', 'license': 'MIT', 'n_tokens_mistral': 2099, 'n_tokens_neox': 1813, 'n_words': 264, 'path': 'notebooks/OsmAnalysis.ipynb', 'repo_name': 'bulutenesemre/OsmAnalysis', 'size': '5066'}