<jupyter_start><jupyter_text>parse according to "system_name":"Métro de Paris"asked help on StackOverflow :ttps://stackoverflow.com/questions/65328899/is-there-a-way-to-sort-a-dataframe-using-pandas-or-other-library-in-python<jupyter_code>data = pd.read_json(r'paris_lines_systems_and_modes.json')
data = data[data.system_name == "Métro de Paris"]
data = data[data.color != "#000"]
data.columns = ['id','Ligne Horizon','url_name','color','system_id','systeme_name','transport_mode_id','transport_mode_name']
dataDrop = data.drop(20)<jupyter_output><empty_output><jupyter_text>parse wiki table<jupyter_code>data_wiki = pd.read_html('https://fr.wikipedia.org/wiki/M%C3%A9tro_de_Paris')
dataFrame_wiki = data_wiki[1]<jupyter_output><empty_output><jupyter_text>looking for a way to replace Ligne by Ligne Horizon. (header)<jupyter_code>dataFrame_wiki.columns = ['Ligne Horizon','Parcours','Mise en service','Dernier prolongement','Longueur en km (en surface)','Nombre de stations','Materiel','Voitures par rame','Nb de Rames','Nb total de rames','Millions de voyageurs (2017)']<jupyter_output><empty_output><jupyter_text>looking for a way to replace NAN by M1, M2 etc. (content)<jupyter_code>dataFrame_wiki.loc[0,'Ligne Horizon']='M 1'
dataFrame_wiki.loc[1,'Ligne Horizon']='M 2'
dataFrame_wiki.loc[2,'Ligne Horizon']='M 3'
dataFrame_wiki.loc[3,'Ligne Horizon']='M 3bis'
dataFrame_wiki.loc[4,'Ligne Horizon']='M 4'
dataFrame_wiki.loc[5,'Ligne Horizon']='M 5'
dataFrame_wiki.loc[6,'Ligne Horizon']='M 6'
dataFrame_wiki.loc[7,'Ligne Horizon']='M 7'
dataFrame_wiki.loc[8,'Ligne Horizon']='M 7bis'
dataFrame_wiki.loc[9,'Ligne Horizon']='M 8'
dataFrame_wiki.loc[10,'Ligne Horizon']='M 9'
dataFrame_wiki.loc[11,'Ligne Horizon']='M 10'
dataFrame_wiki.loc[12,'Ligne Horizon']='M 11'
dataFrame_wiki.loc[13,'Ligne Horizon']='M 12'
dataFrame_wiki.loc[14,'Ligne Horizon']='M 13'
dataFrame_wiki.loc[15,'Ligne Horizon']='M 14'
# looking for a way to delete duplicate
Finaldata = dataFrame_wiki.drop(16)<jupyter_output><empty_output><jupyter_text>arse goejson file<jupyter_code>geodata = pd.read_json(r'paris_sections.geojson')<jupyter_output><empty_output><jupyter_text>oining the data like in SQL for use in Webhttps://pandas.pydata.org/pandas-docs/stable/user_guide/merging.htmldatabase-style-dataframe-or-named-series-joining-merging<jupyter_code>DataForViz = pd.merge(dataDrop, Finaldata, how='inner', on='Ligne Horizon')
print(DataForViz)<jupyter_output>id Ligne Horizon        url_name    color  system_id    systeme_name  \
0   352           M 1     352-ligne-1  #ffcd00         83  Métro de Paris   
1   353           M 2     353-ligne-2  #006db8         83  Métro de Paris   
2   354           M 3     354-ligne-3  #9b993b         83  Métro de Paris   
3   355        M 3bis  355-ligne-3bis  #88d3df         83  Métro de Paris   
4   356           M 4     356-ligne-4  #ba4a9a         83  Métro de Paris   
5   357           M 5     357-ligne-5  #f78e4b         83  Métro de Paris   
6   358           M 6     358-ligne-6  #77c596         83  Métro de Paris   
7   303           M 7     303-ligne-7  #f49eb2         83  Métro de Paris   
8   359        M 7bis  359-ligne-7bis  #77c596         83  Métro de Paris   
9   360           M 8     360-ligne-8  #c5a3cc         83  Métro de Paris   
10  361           M 9     361-ligne-9  #cdc92b         83  Métro de Paris   
11  362          M 10    362-ligne-10  #dfb03a         83  Métro de Paris   [...]<jupyter_text>utting the new dataset (from join in json format)<jupyter_code>DataForViz = DataForViz.to_json()
print(DataForViz)<jupyter_output>{"id":{"0":352,"1":353,"2":354,"3":355,"4":356,"5":357,"6":358,"7":303,"8":359,"9":360,"10":361,"11":362,"12":363,"13":364,"14":365,"15":366},"Ligne Horizon":{"0":"M 1","1":"M 2","2":"M 3","3":"M 3bis","4":"M 4","5":"M 5","6":"M 6","7":"M 7","8":"M 7bis","9":"M 8","10":"M 9","11":"M 10","12":"M 11","13":"M 12","14":"M 13","15":"M 14"},"url_name":{"0":"352-ligne-1","1":"353-ligne-2","2":"354-ligne-3","3":"355-ligne-3bis","4":"356-ligne-4","5":"357-ligne-5","6":"358-ligne-6","7":"303-ligne-7","8":"359-ligne-7bis","9":"360-ligne-8","10":"361-ligne-9","11":"362-ligne-10","12":"363-ligne-11","13":"364-ligne-12","14":"365-ligne-13","15":"366-ligne-14"},"color":{"0":"#ffcd00","1":"#006db8","2":"#9b993b","3":"#88d3df","4":"#ba4a9a","5":"#f78e4b","6":"#77c596","7":"#f49eb2","8":"#77c596","9":"#c5a3cc","10":"#cdc92b","11":"#dfb03a","12":"#8c6539","13":"#008c59","14":"#87d2df","15":"#662c91"},"system_id":{"0":83,"1":83,"2":83,"3":83,"4":83,"5":83,"6":83,"7":83,"8":83,"9":83,"10":83,"11":83,"12":8[...]<jupyter_text>trying to visulalize datahttps://scitools.org.uk/cartopy/docs/latest/crs/projections.htmlgeostationary parse wiki table<jupyter_code>dem_paris = pd.read_html('https://fr.wikipedia.org/wiki/Mod%C3%A8le:Tableau_D%C3%A9mographie_Paris#Carte')
df_dem_paris_250_1600 = dem_paris[0]
df_dem_paris_1637_1811 = dem_paris[1]
df_dem_paris_1817_1866 = dem_paris[2]
df_dem_paris_1872_1911 = dem_paris[3]
df_dem_paris_1921_1975 = dem_paris[4]
df_dem_paris_1982_2016 = dem_paris[5]<jupyter_output><empty_output><jupyter_text>se p2j to convert Python source code to Jupyter Notebook.ip install p2jun p2j myscript.py  clean the data<jupyter_code>df_dem_paris_250_1600.columns = ["250", "1180", "1220", "1328", "1422", "1500", "1565", "1590", "1600"]
df_dem_paris_1637_1811.columns= ["1637", "1680", "1766", "1793", "1795", "1796", "1801", "1807", "1811"]
df_dem_paris_1817_1866.columns= ["1817", "1831", "1836", "1841", "1846", "1851", "1856", "1861", "1866"]
df_dem_paris_1872_1911.columns= ["1872", "1876", "1881", "1886", "1891", "1896", "1901", "1906", "1911"]
df_dem_paris_1921_1975.columns= ["1921", "1926", "1931", "1936", "1946", "1954", "1962", "1968", "1975"]
df_dem_paris_1982_2016.columns= ["1982", "1990", "1999", "2008", "2013", "2019", " - ", " - ", " - "]
df_dem_paris_1982_2016.at[0,'1999']='2 125 246'
df_dem_paris_1982_2016.at[0,'2008']='2 211 297'
df_dem_paris_1982_2016.at[0,'2013']='2 229 621'
df_dem_paris_1982_2016.at[0,'2019']='2 169 423'
df_dem_paris_1921_1975.at[0,'1921']='2 906 472'
df_dem_paris_1921_1975.at[0,'1931']='2 891 020'
df_dem_paris_250_1600.at[0,'250']='6 000'
df_dem_paris_250_1600.at[0,'1180']='25 000'
df_dem_paris_250_1600.at[0,'1220']='50 000'
df_dem_paris_250_1600.at[0,'1328']='200 000'
df_dem_paris_250_1600.at[0,'1422']='100 000'
df_dem_paris_250_1600.at[0,'1500']='150 000'
df_dem_paris_250_1600.at[0,'1565']='294 000'
df_dem_paris_250_1600.at[0,'1590']='200 000'
df_dem_paris_250_1600.at[0,'1600']='300 000'
df_dem_paris_1637_1811.at[0,'1637']='415 000'
df_dem_paris_1637_1811.at[0,'1680']='500 000'
df_dem_paris_1637_1811.at[0,'1766']='576 639'
df_dem_paris_1637_1811.at[0,'1793']='640 504'
df_dem_paris_1637_1811.at[0,'1795']='636 722'
df_dem_paris_1637_1811.at[0,'1796']='551 347'
df_dem_paris_1637_1811.at[0,'1801']='547 756'
df_dem_paris_1637_1811.at[0,'1807']='580 609'
df_dem_paris_1637_1811.at[0,'1811']='622 636'
df_dem_paris_1817_1866.at[0,'1817']='713 966'
df_dem_paris_1817_1866.at[0,'1831']='785 862'
df_dem_paris_1817_1866.at[0,'1836']='909 126'
df_dem_paris_1817_1866.at[0,'1841']='935 261'
df_dem_paris_1817_1866.at[0,'1846']='1 053 897'
df_dem_paris_1817_1866.at[0,'1851']='1 053 262'
df_dem_paris_1817_1866.at[0,'1856']='1 174 346'
df_dem_paris_1872_1911.at[0,'1891']='2 447 957'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19793646483844693, 'fraction_numerical': 0.20038012489818083, 'mean_word_length': 6.076849183477425, 'pattern_counts': {'":': 83, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '334224', 'license': 'MIT', 'n_tokens_mistral': 4080, 'n_tokens_neox': 3205, 'n_words': 464, 'path': 'script.ipynb', 'repo_name': 'textSolver34761/cityline', 'size': '7366'}