<jupyter_start><jupyter_text>Baixa e lista arquivos INPUT<jupyter_code>drive.mount('/content/gdrive')
#pastas
!mkdir -p "/content/data"
!mkdir -p "/content/data/input"
!mkdir -p "/content/data/input/br"
!mkdir -p "/content/data/input/rm"
!mkdir -p "/content/data/input/mun"
!mkdir -p "/content/data/input/mes"
!mkdir -p "/content/data/output"
!mkdir -p "/content/data/output/ip15"
!mkdir -p "/content/data/output/ipca"
!mkdir -p "/content/data/output/inpc"

#!rm -rf "/content/data/input"

indices = ["ipca","ip15", "inpc"]

link = {"br/ipca_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n1/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "br/inpc_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n1/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "br/ip15_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n1/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "br/ipca_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n1/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "br/inpc_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n1/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "br/ip15_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n1/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "br/ipca_item":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n1/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "br/inpc_item":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n1/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "br/ip15_item":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n1/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "br/ipca_subitem":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/947075621",
        "br/inpc_subitem":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/1084704443",
        "br/ip15_subitem":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-783617233",
        
        "br/ipca_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n1/all/v/all/p/all/c315/7169/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "br/inpc_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n1/all/v/all/p/all/c315/7169/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "br/ip15_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n1/all/v/all/p/all/c315/7169/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "rm/ipca_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n7/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "rm/inpc_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n7/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "rm/ip15_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n7/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "rm/ipca_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n7/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "rm/inpc_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n7/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "rm/ip15_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n7/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "rm/ipca_item":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n7/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "rm/inpc_item":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n7/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "rm/ip15_item": "https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n7/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "rm/ipca_subitem_1":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/947075621",
        "rm/ipca_subitem_2":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/847634158",
        "rm/inpc_subitem_1":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-884745035",
        "rm/inpc_subitem_2":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-1265010694",
        "rm/ip15_subitem_1":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-1750716307",
        "rm/ip15_subitem_2":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-1258570016",
        
        "rm/ipca_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n7/all/v/all/p/all/c315/7169/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "rm/inpc_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n7/all/v/all/p/all/c315/7169/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "rm/ip15_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n7/all/v/all/p/all/c315/7169/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        
        "mun/ipca_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n6/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "mun/inpc_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n6/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "mun/ip15_grupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n6/all/v/all/p/all/c315/7170,7445,7486,7558,7625,7660,7712,7766,7786/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "mun/ipca_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n6/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "mun/inpc_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n6/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "mun/ip15_subgrupo":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n6/all/v/all/p/all/c315/7171,7432,7446,7479,7487,7521,7548,7559,7604,7615,7620,7626,7661,7683,7697,7713,7767,7787,47656/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "mun/ipca_item":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n6/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "mun/inpc_item":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n6/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "mun/ip15_item":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n6/all/v/all/p/all/c315/7172,7184,7200,7219,7241,7254,7283,7303,7335,7349,7356,7372,7384,7389,7401,7415,7433,7447,7454,7461,7480,7484,7488,7495,7517,7522,7541,7549,7560,7572,7587,7605,7616,7621,7627,7640,7656,7662,7684,7690,7695,7698,7714,7730,7758,7777,7782,7788,12427,107678,109464/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",

        "mun/ipca_subitem_1":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/866963382",
        "mun/ipca_subitem_2":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-113176757",
        "mun/inpc_subitem_1":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/1139761886",
        "mun/inpc_subitem_2":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-1289673003",
        "mun/ip15_subitem_1":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-260564956",
        "mun/ip15_subitem_2":"https://sidra.ibge.gov.br/geratabela/DownloadSelecaoComplexa/-317614754",


        "mun/ipca_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7060.csv&terr=NC&rank=-&query=t/7060/n6/all/v/all/p/all/c315/7169/d/v63%202,v66%204,v69%202,v2265%202/l/,v,t%2Bp%2Bc315",
        "mun/inpc_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7063.csv&terr=NC&rank=-&query=t/7063/n6/all/v/all/p/all/c315/7169/d/v44%202,v45%204,v68%202,v2292%202/l/,v,t%2Bp%2Bc315",
        "mun/ip15_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela7062.csv&terr=NC&rank=-&query=t/7062/n6/all/v/all/p/all/c315/7169/d/v355%202,v356%202,v357%204,v1120%202/l/,v,t%2Bp%2Bc315",
        
        "mes/ipca_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela1737.csv&terr=N&rank=-&query=t/1737/n1/all/v/all/p/all/d/v63%202,v69%202,v2263%202,v2264%202,v2265%202,v2266%2013/l/,v,t%2Bp&abreviarRotulos=True&exibirNotas=False",
        "mes/ip15_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela3065.csv&terr=N&rank=-&query=t/3065/n1/all/v/all/p/all/d/v355%202,v356%202,v1117%2013,v1118%202,v1119%202,v1120%202/l/,v,t%2Bp&abreviarRotulos=True&exibirNotas=False",
        "mes/inpc_geral":"https://sidra.ibge.gov.br/geratabela?format=br.csv&name=tabela1736.csv&terr=N&rank=-&query=t/1736/n1/all/v/all/p/all/d/v44%202,v68%202,v2289%2013,v2290%202,v2291%202,v2292%202/l/,v,t%2Bp&abreviarRotulos=True&exibirNotas=False"
        }

for key in link:
  wget.download(link[key],
                  out="/content/data/input/"+ key +".csv")
  print("Baixado", key)<jupyter_output><empty_output><jupyter_text>MES_CATEGORIA_BRASIL<jupyter_code>rename = {"Mês":"ano", 
          "Geral, grupo, subgrupo, item e subitem":"categoria",
          "IPCA - Variação mensal (%)":"variacao_mensal",
          "IPCA - Variação acumulada no ano (%)":"variacao_anual",
          "IPCA - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
          "IPCA - Peso mensal (%)":"peso_mensal",
          "INPC - Variação mensal (%)":"variacao_mensal",
          "INPC - Variação acumulada no ano (%)":"variacao_anual",
          "INPC - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
          "INPC - Peso mensal (%)":"peso_mensal",
          "IPCA15 - Variação mensal (%)":"variacao_mensal",
          "IPCA15 - Variação acumulada no ano (%)":"variacao_anual",
          "IPCA15 - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
          "IPCA15 - Peso mensal (%)":"peso_mensal"
          }

ordem = ["ano","mes","id_categoria","id_categoria_bd","categoria","peso_mensal",
         "variacao_mensal","variacao_anual","variacao_doze_meses"
        ]

n_mes = {'janeiro':'1',
        'fevereiro':'2',
        'março':'3',
        'abril':'4',
        'maio':'5',
        'junho':'6',
        'julho':'7',
        'agosto':'8',
        'setembro':'9',
        'outubro':'10',
        'novembro':'11',
        'dezembro':'12'
        }

for indice in indices:
  
  arquivos = [arquivo for arquivo in glob.iglob('data/input/br/*') if arquivo[14:18] == indice]
  for arq in arquivos:
    df = pd.read_csv(arq, skipfooter=14, skiprows=2, sep=';', dtype="str")
    #renomear colunas
    df.rename(columns=rename,inplace=True)
    #substituir "..." por vazio
    df  = df.replace("...","")
    df = df.replace('-','')
    
    #Normalizando float
    df = df.replace(',','.',regex=True)
    
    #Split coluna data e substituir mes
    df[['mes','ano']] = df['ano'].str.split(' ',1,expand=True)
    df['mes'] = df['mes'].map(n_mes)

    #Split coluna categoria e add id_categoria_bd
    if arq[19:-4] != "geral":
      df[['id_categoria','categoria']] = df['categoria'].str.split('.',1,expand=True)

    if arq[19:-4] == "grupo":
      df['id_categoria_bd']  = df['id_categoria'].apply(lambda x: x+".0.00.000")
      df = df[ordem]
      grupo = pd.DataFrame(df)
    elif arq[19:-4] == "subgrupo":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ ".00.000")
      df = df[ordem]
      subgrupo = pd.DataFrame(df)
    elif arq[19:-4] == "item":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ "." + x[2:4]+ ".000")
      df = df[ordem]
      item = pd.DataFrame(df)
    elif arq[19:-4] == "subitem":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ "." + x[2:4] +"."+x[4:7])
      df = df[ordem]
      subitem = pd.DataFrame(df)
    elif arq[19:-4] == "geral":
      df['id_categoria'] = ""
      df['id_categoria_bd'] = "0.0.00.000"
      df = df[ordem]
      geral = pd.DataFrame(df)

  df = pd.concat([grupo,subgrupo,item,subitem,geral])
  filepath='/content/data/output/{}/categoria_brasil.csv'.format(indice)
  df.to_csv(filepath,index=False)
  print("Feito BR " + indice)
    
  del df
  del item
  del subitem
  del geral
  del grupo
  del subgrupo<jupyter_output><empty_output><jupyter_text>MES_CATEGORIA_RM<jupyter_code>rename = {"Cód.":"id_regiao_metropolitana",
               "Unnamed: 1":"rm",
               "Mês":"ano", 
               "Geral, grupo, subgrupo, item e subitem":"categoria",
               "IPCA - Variação mensal (%)":"variacao_mensal",
               "IPCA - Variação acumulada no ano (%)":"variacao_anual",
               "IPCA - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
               "IPCA - Peso mensal (%)":"peso_mensal",               
                "INPC - Variação mensal (%)":"variacao_mensal",
               "INPC - Variação acumulada no ano (%)":"variacao_anual",
               "INPC - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
               "INPC - Peso mensal (%)":"peso_mensal",
                "IPCA15 - Variação mensal (%)":"variacao_mensal",
               "IPCA15 - Variação acumulada no ano (%)":"variacao_anual",
               "IPCA15 - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
               "IPCA15 - Peso mensal (%)":"peso_mensal"
               }

ordem = ["ano","mes","id_regiao_metropolitana","id_categoria","id_categoria_bd","categoria",
         "peso_mensal","variacao_mensal","variacao_anual","variacao_doze_meses"
         ]

n_mes = {'janeiro':'1',
        'fevereiro':'2',
        'março':'3',
        'abril':'4',
        'maio':'5',
        'junho':'6',
        'julho':'7',
        'agosto':'8',
        'setembro':'9',
        'outubro':'10',
        'novembro':'11',
        'dezembro':'12'
        }


for indice in indices:
  
  arquivos = [arquivo for arquivo in glob.iglob('data/input/rm/*') if arquivo[14:18] == indice]

  for arq in arquivos:
    df = pd.read_csv(arq, skipfooter=14, skiprows=2, sep=';', dtype="str")
    #renomear colunas
    df.rename(columns=rename,inplace=True)
    #substituir "..." por vazio
    df  = df.replace("...","")
    df = df.replace('-','')

    #Normalizando float
    df = df.replace(',','.',regex=True)
    
    #Split coluna data e substituir mes
    df[['mes','ano']] = df['ano'].str.split(' ',1,expand=True)
    df['mes'] = df['mes'].map(n_mes)

    #Split coluna categoria e add id_categoria_bd
    if arq[19:-4] != "geral":
      df[['id_categoria','categoria']] = df['categoria'].str.split('.',1,expand=True)

    if arq[19:-4] == "grupo":
      df['id_categoria_bd']  = df['id_categoria'].apply(lambda x: x+".0.00.000")
      df = df[ordem]
      grupo = pd.DataFrame(df)
    elif arq[19:-4] == "subgrupo":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ ".00.000")
      df = df[ordem]
      subgrupo = pd.DataFrame(df)
    elif arq[19:-4] == "item":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ "." + x[2:4]+ ".000")
      df = df[ordem]
      item = pd.DataFrame(df)
    elif arq[19:-4] == "subitem_1":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ "." + x[2:4] +"."+x[4:7])
      df = df[ordem]
      subitem_1 = pd.DataFrame(df)
    elif arq[19:-4] == "subitem_2":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ "." + x[2:4] +"."+x[4:7])
      df = df[ordem]
      subitem_2 = pd.DataFrame(df)
    elif arq[19:-4] == "geral":
      df['id_categoria'] = ""
      df['id_categoria_bd'] = "0.0.00.000"
      df = df[ordem]
      geral = pd.DataFrame(df)

  df = pd.concat([grupo,subgrupo,item,subitem_1,subitem_2,geral])
  filepath='/content/data/output/{}/categoria_rm.csv'.format(indice)
  df.to_csv(filepath,index=False)
  print("Feito RM " + indice)

  del item
  del subitem_1
  del subitem_2
  del geral
  del grupo
  del subgrupo<jupyter_output><empty_output><jupyter_text>MES_CATEGORIA_MUNICIPIO<jupyter_code>rename = {"Cód.":"id_municipio",
               "Unnamed: 1":"municipio",
               "Mês":"ano", 
               "Geral, grupo, subgrupo, item e subitem":"categoria",
               "IPCA - Variação mensal (%)":"variacao_mensal",
               "IPCA - Variação acumulada no ano (%)":"variacao_anual",
               "IPCA - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
               "IPCA - Peso mensal (%)":"peso_mensal",               
                "INPC - Variação mensal (%)":"variacao_mensal",
               "INPC - Variação acumulada no ano (%)":"variacao_anual",
               "INPC - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
               "INPC - Peso mensal (%)":"peso_mensal",
                "IPCA15 - Variação mensal (%)":"variacao_mensal",
               "IPCA15 - Variação acumulada no ano (%)":"variacao_anual",
               "IPCA15 - Variação acumulada em 12 meses (%)":"variacao_doze_meses",
               "IPCA15 - Peso mensal (%)":"peso_mensal"
               }

ordem = ["ano","mes","id_municipio","id_categoria","id_categoria_bd","categoria","peso_mensal",
         "variacao_mensal","variacao_anual","variacao_doze_meses"
         ]

n_mes = {'janeiro':'1',
        'fevereiro':'2',
        'março':'3',
        'abril':'4',
        'maio':'5',
        'junho':'6',
        'julho':'7',
        'agosto':'8',
        'setembro':'9',
        'outubro':'10',
        'novembro':'11',
        'dezembro':'12'
        }

for indice in indices:
  
  arquivos = [arquivo for arquivo in glob.iglob('data/input/mun/*') if arquivo[15:19] == indice]

  for arq in arquivos:
    df = pd.read_csv(arq, skipfooter=14, skiprows=2, sep=';', dtype="str")
    #renomear colunas
    df.rename(columns=rename,inplace=True)
    #substituir "..." por vazio
    df  = df.replace("...","")
    df = df.replace('-','')

    #Normalizando float
    df = df.replace(',','.',regex=True)
    
    #Split coluna data e substituir mes
    df[['mes','ano']] = df['ano'].str.split(' ',1,expand=True)
    df['mes'] = df['mes'].map(n_mes)

    #Split coluna categoria e add id_categoria_bd
    if arq[20:-4] != "geral":
      df[['id_categoria','categoria']] = df['categoria'].str.split('.',1,expand=True)

    if arq[20:-4] == "grupo":
      df['id_categoria_bd']  = df['id_categoria'].apply(lambda x: x+".0.00.000")
      df = df[ordem]
      grupo = pd.DataFrame(df)
    elif arq[20:-4] == "subgrupo":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ ".00.000")
      df = df[ordem]
      subgrupo = pd.DataFrame(df)
    elif arq[20:-4] == "item":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ "." + x[2:4]+ ".000")
      df = df[ordem]
      item = pd.DataFrame(df)
    elif arq[20:-4] == "subitem_1":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ "." + x[2:4] +"."+x[4:7])
      df = df[ordem]
      subitem_1 = pd.DataFrame(df)
    elif arq[20:-4] == "subitem_2":
      df['id_categoria_bd'] = df['id_categoria'].apply(lambda x: x[0] + "."+ x[1]+ "." + x[2:4] +"."+x[4:7])
      df = df[ordem]
      subitem_2 = pd.DataFrame(df)
    elif arq[20:-4] == "geral":
      df['id_categoria'] = ""
      df['id_categoria_bd'] = "0.0.00.000"
      df = df[ordem]
      geral = pd.DataFrame(df)
  
  df = pd.concat([grupo,subgrupo,item,subitem_1,subitem_2,geral])
  
  filepath='/content/data/output/{}/categoria_municipio.csv'.format(indice)
  df.to_csv(filepath,index=False)
  print("Feito MUN " + indice)
  
  del df
  del item
  del subitem_1
  del subitem_2
  del geral
  del grupo
  del subgrupo<jupyter_output><empty_output><jupyter_text>MES_BRASIL<jupyter_code>rename = {'IPCA - Número-índice (base: dezembro de 1993 = 100) (Número-índice)':'indice',
          'IPCA - Variação mensal (%)':'variacao_mensal',
          'IPCA - Variação acumulada em 3 meses (%)':'variacao_trimestral',
          'IPCA - Variação acumulada em 6 meses (%)':'variacao_semestral',
          'IPCA - Variação acumulada no ano (%)':'variacao_anual',
          'IPCA - Variação acumulada em 12 meses (%)':'variacao_doze_meses',
          'IPCA15 - Número-índice (base: dezembro de 1993 = 100) (Número-índice)':'indice',
          'IPCA15 - Variação mensal (%)':'variacao_mensal',
          'IPCA15 - Variação acumulada em 3 meses (%)':'variacao_trimestral',
          'IPCA15 - Variação acumulada em 6 meses (%)':'variacao_semestral',
          'IPCA15 - Variação acumulada no ano (%)':'variacao_anual',
          'IPCA15 - Variação acumulada em 12 meses (%)':'variacao_doze_meses',
          'INPC - Número-índice (base: dezembro de 1993 = 100) (Número-índice)':'indice',
          'INPC - Variação mensal (%)':'variacao_mensal',
          'INPC - Variação acumulada em 3 meses (%)':'variacao_trimestral',
          'INPC - Variação acumulada em 6 meses (%)':'variacao_semestral',
          'INPC - Variação acumulada no ano (%)':'variacao_anual',
          'INPC - Variação acumulada em 12 meses (%)':'variacao_doze_meses'}

ordem = ['ano','mes','indice','variacao_mensal','variacao_trimestral','variacao_semestral',
         'variacao_anual','variacao_doze_meses']

n_mes = {'janeiro':'1',
        'fevereiro':'2',
        'março':'3',
        'abril':'4',
        'maio':'5',
        'junho':'6',
        'julho':'7',
        'agosto':'8',
        'setembro':'9',
        'outubro':'10',
        'novembro':'11',
        'dezembro':'12'
        }

for indice in indices:
  
  arquivos = [arquivo for arquivo in glob.iglob('data/input/mes/*') if arquivo[15:19] == indice]
  
  for arq in arquivos:
    df = pd.read_csv(arq,skiprows=2,skipfooter=13,sep=';')
    
    df['mes'] , df['ano'] = df['Mês'].str.split(' ',1).str

    df['mes'] = df['mes'].map(n_mes)
    
    #renomear colunas
    df.rename(columns=rename,inplace=True)
    df  = df.replace("...","")
    df = df.replace('-','')

    #Normalizando float
    df = df.replace(',','.',regex=True)
    
    #Renomeando colunas e ordenando
    df = df[ordem]
  
  filepath='/content/data/output/{}/mes_geral.csv'.format(indice)
  df.to_csv(filepath,index=False)
  print("Feito MES " + indice)
  
  del df
#### Baixa PASTA ZIP
!zip -r /content/output.Zip /content/data/output<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2044596165020337, 'fraction_numerical': 0.18473997675769901, 'mean_word_length': 4.1878296910324035, 'pattern_counts': {'":': 120, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 54, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '326042', 'license': 'MIT', 'n_tokens_mistral': 15691, 'n_tokens_neox': 12926, 'n_words': 1011, 'path': 'bases/br_ibge_ipca15/code/ipca_ibge_etl.ipynb', 'repo_name': 'isadorabugarin/mais', 'size': '27536'}