<jupyter_start><jupyter_text><jupyter_code>import requests
from zipfile import ZipFile
import os
url = "https://www.transtats.bts.gov/DownLoad_Table.asp"

params = {
    "Table_ID": 258,
    "Has_Group": 3,
    "Is_Zipped": 0
}



year=2016

data = {
    "UserTableName": "T_100_Domestic_Market__U.S._Carriers",
    "RawDataTable": "T_T100D_MARKET_US_CARRIER_ONLY",
    "sqlstr": "SELECT PASSENGERS,FREIGHT,MAIL,DISTANCE,UNIQUE_CARRIER,AIRLINE_ID,UNIQUE_CARRIER_NAME,UNIQUE_CARRIER_ENTITY,REGION,CARRIER,CARRIER_NAME,CARRIER_GROUP,CARRIER_GROUP_NEW,ORIGIN_AIRPORT_ID,ORIGIN_AIRPORT_SEQ_ID,ORIGIN_CITY_MARKET_ID,ORIGIN,ORIGIN_CITY_NAME,ORIGIN_STATE_ABR,ORIGIN_STATE_FIPS,ORIGIN_STATE_NM,ORIGIN_WAC,DEST_AIRPORT_ID,DEST_AIRPORT_SEQ_ID,DEST_CITY_MARKET_ID,DEST,DEST_CITY_NAME,DEST_STATE_ABR,DEST_STATE_FIPS,DEST_STATE_NM,DEST_WAC,YEAR,QUARTER,MONTH,DISTANCE_GROUP,CLASS FROM  T_T100D_MARKET_US_CARRIER_ONLY WHERE YEAR={}".format(year),
    "varlist": "PASSENGERS,FREIGHT,MAIL,DISTANCE,UNIQUE_CARRIER,AIRLINE_ID,UNIQUE_CARRIER_NAME,UNIQUE_CARRIER_ENTITY,REGION,CARRIER,CARRIER_NAME,CARRIER_GROUP,CARRIER_GROUP_NEW,ORIGIN_AIRPORT_ID,ORIGIN_AIRPORT_SEQ_ID,ORIGIN_CITY_MARKET_ID,ORIGIN,ORIGIN_CITY_NAME,ORIGIN_STATE_ABR,ORIGIN_STATE_FIPS,ORIGIN_STATE_NM,ORIGIN_WAC,DEST_AIRPORT_ID,DEST_AIRPORT_SEQ_ID,DEST_CITY_MARKET_ID,DEST,DEST_CITY_NAME,DEST_STATE_ABR,DEST_STATE_FIPS,DEST_STATE_NM,DEST_WAC,YEAR,QUARTER,MONTH,DISTANCE_GROUP,CLASS",
    "filter1": "title=",
    "filter2": "title=",
    "timename": "Month",
    "GEOGRAPHY": "All",
    "XYEAR": "{}".format(year),
    "FREQUENCY": "All",
    "AllVars": "All",
    "VarName": "PASSENGERS",
    "VarDesc": "Passengers",
    "VarType": "Num",
    "VarName": "FREIGHT",
    "VarDesc": "Freight",
    "VarType": "Num",
    "VarName": "MAIL",
    "VarDesc": "Mail",
    "VarType": "Num",
    "VarName": "DISTANCE",
    "VarDesc": "Distance",
    "VarType": "Num",
    "VarName": "UNIQUE_CARRIER",
    "VarDesc": "UniqueCarrier",
    "VarType": "Char",
    "VarName": "AIRLINE_ID",
    "VarDesc": "AirlineID",
    "VarType": "Num",
    "VarName": "UNIQUE_CARRIER_NAME",
    "VarDesc": "UniqueCarrierName",
    "VarType": "Char",
    "VarName": "UNIQUE_CARRIER_ENTITY",
    "VarDesc": "UniqCarrierEntity",
    "VarType": "Char",
    "VarName": "REGION",
    "VarDesc": "CarrierRegion",
    "VarType": "Char",
    "VarName": "CARRIER",
    "VarDesc": "Carrier",
    "VarType": "Char",
    "VarName": "CARRIER_NAME",
    "VarDesc": "CarrierName",
    "VarType": "Char",
    "VarName": "CARRIER_GROUP",
    "VarDesc": "CarrierGroup",
    "VarType": "Num",
    "VarName": "CARRIER_GROUP_NEW",
    "VarDesc": "CarrierGroupNew",
    "VarType": "Num",
    "VarName": "ORIGIN_AIRPORT_ID",
    "VarDesc": "OriginAirportID",
    "VarType": "Num",
    "VarName": "ORIGIN_AIRPORT_SEQ_ID",
    "VarDesc": "OriginAirportSeqID",
    "VarType": "Num",
    "VarName": "ORIGIN_CITY_MARKET_ID",
    "VarDesc": "OriginCityMarketID",
    "VarType": "Num",
    "VarName": "ORIGIN",
    "VarDesc": "Origin",
    "VarType": "Char",
    "VarName": "ORIGIN_CITY_NAME",
    "VarDesc": "OriginCityName",
    "VarType": "Char",
    "VarName": "ORIGIN_STATE_ABR",
    "VarDesc": "OriginState",
    "VarType": "Char",
    "VarName": "ORIGIN_STATE_FIPS",
    "VarDesc": "OriginStateFips",
    "VarType": "Char",
    "VarName": "ORIGIN_STATE_NM",
    "VarDesc": "OriginStateName",
    "VarType": "Char",
    "VarName": "ORIGIN_WAC",
    "VarDesc": "OriginWac",
    "VarType": "Num",
    "VarName": "DEST_AIRPORT_ID",
    "VarDesc": "DestAirportID",
    "VarType": "Num",
    "VarName": "DEST_AIRPORT_SEQ_ID",
    "VarDesc": "DestAirportSeqID",
    "VarType": "Num",
    "VarName": "DEST_CITY_MARKET_ID",
    "VarDesc": "DestCityMarketID",
    "VarType": "Num",
    "VarName": "DEST",
    "VarDesc": "Dest",
    "VarType": "Char",
    "VarName": "DEST_CITY_NAME",
    "VarDesc": "DestCityName",
    "VarType": "Char",
    "VarName": "DEST_STATE_ABR",
    "VarDesc": "DestState",
    "VarType": "Char",
    "VarName": "DEST_STATE_FIPS",
    "VarDesc": "DestStateFips",
    "VarType": "Char",
    "VarName": "DEST_STATE_NM",
    "VarDesc": "DestStateName",
    "VarType": "Char",
    "VarName": "DEST_WAC",
    "VarDesc": "DestWac",
    "VarType": "Num",
    "VarName": "YEAR",
    "VarDesc": "Year",
    "VarType": "Num",
    "VarName": "QUARTER",
    "VarDesc": "Quarter",
    "VarType": "Num",
    "VarName": "MONTH",
    "VarDesc": "Month",
    "VarType": "Num",
    "VarName": "DISTANCE_GROUP",
    "VarDesc": "DistanceGroup",
    "VarType": "Num",
    "VarName": "CLASS",
    "VarDesc": "Class",
    "VarType": "Char"
}
response = requests.post(url, params=params, json=data)
response.content
response.headers
first_response_cookie = response.cookies
first_response_cookie = list(first_response_cookie)[0]
first_response_cookie
s = requests.session()
# Note that domain keyword parameter is the only optional parameter here
s.cookies.set_cookie(first_response_cookie)

url = "https://transtats.bts.gov/ftproot/TranStatsData/879947130_T_T100D_MARKET_US_CARRIER_ONLY.zip"

res = s.get(url)

data_zip = res.content

with open("./data{}.zip".format(year), 'wb') as zip_file:
    zip_file.write(data_zip)
s.cookies<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1716844512195122, 'fraction_numerical': 0.006288109756097561, 'mean_word_length': 5.1680376028202115, 'pattern_counts': {'":': 122, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '69652', 'license': 'MIT', 'n_tokens_mistral': 2245, 'n_tokens_neox': 2185, 'n_words': 300, 'path': 'module1-define-ml-problems/HADI_MODARES_LS_DS_231_assignment.ipynb', 'repo_name': 'hadi-M/DS-Unit-2-Applied-Modeling', 'size': '5248'}