<jupyter_start><jupyter_text>Travel Dataset GeneratorThis code generate a dataset of travel operations, _e.g._ , plane tickets and lodging.   This notebook just ilustrate the steps to generate the dataset, for extensive generators use the python code inside.<jupyter_code># Import packages
import names
import tqdm

import random
import pandas as pd
from datetime import datetime as dt
from datetime import timedelta as td<jupyter_output><empty_output><jupyter_text>--- DefinitionsPredefine variables.<jupyter_code>#- Companies and Users
defGenders = ['male', 'female', 'none']
defAgesInterval = {'min': 23, 'max': 50}
defFlightsInterval = {'min': 0, 'max': 3}
defCompanies = {
    'HHD': {'usersCount': 2},
    '4You': {'usersCount': 3},
}

#- Flight Agencies
defFlightTypes = {
    'economic': {'price': 1.0},
    'premium': {'price': 1.5},
}
defAgenciesName = ['FlyingDrops', 'Rainbow', 'CloudFy']
defAgencies = dict()

#- Places
defPlacesName = ['Sao Paulo (SP)', 'Rio de Janeiro (RJ)', 'Santa Catarina (SC)']
defPlaces = {name: dict() for name in defPlacesName}
defDistancesInterval = {'min': 200.0, 'max': 850.0}
defPlaceTravelKmPerHour = 400.0 

#- Lodge (Accommodation)
defLodgesInterval = {'min': 1, 'max': 3}
defLodgesPrices   = {'min': 60.0, 'max': 200.0}
defLodgesPrex = 'Hotel'
defLodges = {name: list() for name in defPlacesName}

#- Travel
defTravels = list()
defTravelsDays = {'min': 1, 'max': 3}
defTravelsFlightPrices = {'init': 300.0, 'interval': 100.0}
defTravelWithLodge = 0.3
defTravelDate = {'init': dt.now(), 'interval':{'min': 10, 'max': 60}}<jupyter_output><empty_output><jupyter_text>--- Companies and Users - Generator<jupyter_code>#- Functions
def funcUserGenerator(genders, agesInterval, flightsInterval, code):
    '''
    Generate random user, based on predefinitions.
    - genders: list
    - agesInterval {min, max}: user age
    - flightsInterval {min, max}: number of flights
    - code: user ID
    '''
    user = dict()
    user['code'] = code
    user['gender'] = genders[random.randint(0, len(genders)-1)]
    gender = user['gender'] if (user['gender'] != 'none') else False
    user['name'] = names.get_full_name(gender=gender)
    user['age'] = random.randint(agesInterval['min'], agesInterval['max'])
    user['flights'] = random.randint(flightsInterval['min'], flightsInterval['max'])
    return user
#- Fill Companies data
userId = 0
for company, data in defCompanies.items():
    users = list()
    for idx in range(data['usersCount']):
        user = funcUserGenerator(defGenders, defAgesInterval, defFlightsInterval, userId)
        users.append(user)
        userId += 1
    defCompanies[company]['users'] = users<jupyter_output><empty_output><jupyter_text>Example - Users from a Company<jupyter_code>defCompanies['HHD']['users']<jupyter_output><empty_output><jupyter_text>Flight Agencies - Generator<jupyter_code>#- Functions
def funcAgencyGenerator(flightTypes):
    '''
    Generate random agency services, based on predefinitions.
    - flightTypes: types of flight
    '''
    agency = dict()
    types = list(flightTypes.copy().keys())
    random.shuffle(types)
    typesMany = random.randint(1, len(types))
    agency['types'] = [types[i] for i in range(typesMany)]
    return agency
for agency in defAgenciesName:
    defAgencies[agency] = funcAgencyGenerator(defFlightTypes)<jupyter_output><empty_output><jupyter_text>Example - Flight Types of Agencies<jupyter_code>defAgencies<jupyter_output><empty_output><jupyter_text>Places - Generator<jupyter_code>#- Functions
def funcPlaceGenerator(i, j, distInterval, kmPerHour):
    '''
    Generate random place distances, based on predefinitions.
    - i: number of place
    - j: number of place
    - distInterval {min, max} values: distance range
    - kmPerHour: km per hour of the plain
    '''
    if i == j:
        return False, False, False
    distance = round(random.uniform(distInterval['min'], distInterval['max']), 2)
    time = round(distance/kmPerHour, 2)
    hours = int(time)
    minutes = (time*60) % 60
    timeMsg = '%d:%dh' % (hours, minutes)
    return (distance, time, timeMsg)
n = len(defPlacesName)
for i in range(n):
    for j in range(i, n):
        fromPlace = defPlacesName[i]
        toPlace = defPlacesName[j]
        distance, time, msg = funcPlaceGenerator(i, j, defDistancesInterval, defPlaceTravelKmPerHour)
        if distance and time:
            place = {'distance': distance, 'time': time, 'timeMsg': msg}
            defPlaces[fromPlace][toPlace] = place
            defPlaces[toPlace][fromPlace] = place<jupyter_output><empty_output><jupyter_text>Example - Distances from a Place<jupyter_code>defPlaces['Sao Paulo (SP)']<jupyter_output><empty_output><jupyter_text>Lodges - Generator<jupyter_code>#- Definitions
defName = 'A'


#- Functions
def getNextChar(text):
    '''
    Generate order alphabetic.
    - text: input text
    '''
    if len(text) == 0:
        return 'A'
    nextChar = chr(ord(text[-1]) + 1)
    if nextChar <= 'Z':
        text = text[:-1] + nextChar
    else:
        text = getNextChar(text[:-1]) + 'A'
    return text


def funcLodgesGenerator(lodgesInterval, lodgesPrices):
    '''
    Generate random lodges, based on predefinitions.
    - lodgesInterval {min, max} values: number of hotels
    - lodgesPrices {min, max} values: hotel range
    '''
    global defName
    lodges = list()
    n = random.randint(lodgesInterval['min'], lodgesInterval['max'])
    for i in range(n):
        lodgeName = '%s %s' % (defLodgesPrex, defName)
        price = round(random.uniform(lodgesPrices['min'], lodgesPrices['max']), 2)
        lodge = {'code': defName, 'name': lodgeName, 'price': price}
        lodges.append(lodge)
        defName = getNextChar(defName)
    return lodges
for name in defPlacesName:
    lodges = funcLodgesGenerator(defLodgesInterval, defLodgesPrices)
    defLodges[name] = lodges<jupyter_output><empty_output><jupyter_text>Example - Hotels from a Place<jupyter_code>defLodges['Sao Paulo (SP)']<jupyter_output><empty_output><jupyter_text>Travel Possibilities - Generator<jupyter_code>#- Functions
def funcCalculatePrice(priceMin, priceMax, weight):
    '''
    Calculate a random price for a travel.
    - priceMin: min price
    - priceMax: max price
    - weight: weight the price range
    '''
    priceMin = priceMin * weight
    priceMax = priceMax * weight
    price = round(random.uniform(priceMin, priceMax), 2)
    return price


def funcElaborateflight(fromPlace, toPlace, distance, agency, flightType, price, \
                        time, timeMsg):
    '''
    Elaborate a possible flight.
    - fromPlace: from
    - toPlace: to
    - distance: distance
    - agency: agency name
    - flightType: flight type
    - price: flight price
    - time: time in hours
    - timeMsg: time calculated
    '''
    flight = {'from': fromPlace, 'to': toPlace, 'distance': distance,
              'agency': agency, 'flightType': flightType, 'price': price,
              'time': time, 'timeMsg': timeMsg}
    return flight


def funcFlightsPossibilities(places, flightPrices, flightTypes, agencies):
    '''
    Elaborate a list of possible flights.
    - places: places data
    - flightPrices: flight prices
    - flightTypes: flight types
    - agencies: agencies data
    '''
    flightsPossibilities = list()
    for fromPlace, toPlaces in places.items():
        toPlacesSorted = sorted(toPlaces.items(), key=lambda x:x[1]['distance'], reverse=False)
        priceA, priceB = flightPrices['init'], \
                         flightPrices['init'] + flightPrices['interval']
        for (toPlace, placeData) in toPlacesSorted:
            for (agencyName, agencyData) in agencies.items():
                if len(agencyData['types']) > 1: # has more than 1 element
                    for typeA in agencyData['types']:
                        weight = flightTypes[typeA]['price']
                        price = funcCalculatePrice(priceA, priceB, weight)
                        flight = funcElaborateflight(fromPlace, toPlace, placeData['distance'], \
                                                     agencyName, typeA, price, placeData['time'], placeData['timeMsg'])
                        flightsPossibilities.append(flight)
                else:
                    typeA = agencyData['types'][0]
                    weight = flightTypes[typeA]['price']
                    price = funcCalculatePrice(priceA, priceB, weight)
                    flight = funcElaborateflight(fromPlace, toPlace, placeData['distance'], agencyName, \
                                                 typeA, price, placeData['time'], placeData['timeMsg'])
                    flightsPossibilities.append(flight)
            # Update prices for bigger distances
            priceA, priceB = priceB, priceB + flightPrices['interval']
    return flightsPossibilities


def funcLodgesPossibilities(placesName, lodges):
    '''
    Elaborate a list of possible hotels.
    - placesName: places names
    - lodges: lodges data
    '''
    lodgesPossibilities = list()
    for place in placesName:
        for lodge in lodges[place]:
            lodge = lodge.copy()
            lodge['place'] = place
            lodgesPossibilities.append(lodge)
    return lodgesPossibilities<jupyter_output><empty_output><jupyter_text>Example - Fligts Possibilities (for each Place)<jupyter_code>flightsPossibilities = funcFlightsPossibilities(defPlaces, defTravelsFlightPrices, defFlightTypes, defAgencies)
pd.DataFrame(flightsPossibilities).head(5)<jupyter_output><empty_output><jupyter_text>Example - Hotel Possibilities (for each Place)<jupyter_code>lodgesPossibilities = funcLodgesPossibilities(defPlacesName, defLodges)
pd.DataFrame(lodgesPossibilities).head(5)<jupyter_output><empty_output><jupyter_text>--- Travel Dataset - Generator<jupyter_code>#- Definitions
travelCode = 0


#- Functions
def df2Dict(df):
    '''
    Convert dataframe into dict
    '''
    procDict = dict()
    tmp = df.to_dict('split')
    data = tmp['data'][0]
    for (i, column) in enumerate(tmp['columns']):
        procDict[column] = data[i]
    return procDict


def funcTravelsSimulated(companies, flightsPossibilities, lodgesPossibilities, travelDate, travelsDays, \
                         travelWithLodge, placesName):
    '''
    Elaborate random travels with flights and lodges, based on possibilities.
    - flightsPossibilities: possible flights
    - lodgesPossibilities: possible hotels
    '''
    global travelCode
    dfFlightsPos = pd.DataFrame(flightsPossibilities)
    dfLodgesPos = pd.DataFrame(lodgesPossibilities)
    flightsSimulated, lodgesSimulated = list(), list()
    for (companyName, companyData) in companies.items():
        for user in companyData['users']:
            date = travelDate['init']
            for i in range(user['flights']):
                # random - days, places, hotel?
                daysFlight = random.randint(travelsDays['min'], travelsDays['max'])
                daysNextTravel = random.randint(travelDate['interval']['min'], travelDate['interval']['min'])
                fromPlace, toPlace = random.sample(placesName, 2)
                chanceTravelWithLodge = (random.randrange(100) < travelWithLodge*100)
                # travels
                fromConditions = (dfFlightsPos['from']==fromPlace) & (dfFlightsPos['to']==toPlace)
                tmpFlightFrom  = df2Dict(dfFlightsPos[fromConditions].sample(n=1))
                toConditions = (dfFlightsPos['from']==toPlace) & (dfFlightsPos['to']==fromPlace) & \
                               (dfFlightsPos['agency']==tmpFlightFrom['agency']) & \
                               (dfFlightsPos['flightType']==tmpFlightFrom['flightType'])
                tmpFlightTo  = df2Dict(dfFlightsPos[toConditions])
                tmpFlightFrom['userCode'] = tmpFlightTo['userCode'] = user['code']
                tmpFlightFrom['travelCode'] = tmpFlightTo['travelCode'] = travelCode
                tmpFlightFrom['date'] = date
                tmpFlightTo['date']   = date + td(days=daysFlight)
                # lodge
                if chanceTravelWithLodge:
                    lodgeConditions = (dfLodgesPos['place']==toPlace)
                    tmpLodge = df2Dict(dfLodgesPos[lodgeConditions])
                    tmpLodge['userCode'] = user['code']
                    tmpLodge['date'] = date
                    tmpLodge['days'] = daysFlight
                    tmpLodge['total'] = round(tmpLodge['price'] * daysFlight, 2)
                    tmpLodge['travelCode'] = travelCode
                    lodgesSimulated.append(tmpLodge)
                # save and update data
                flightsSimulated.append(tmpFlightFrom)
                flightsSimulated.append(tmpFlightTo)
                travelCode += 1
                date = dt.now() + td(days=daysNextTravel)
    return flightsSimulated, lodgesSimulated
flightsSimulated, lodgesSimulated = \
        funcTravelsSimulated(defCompanies, flightsPossibilities, lodgesPossibilities, 
        defTravelDate, defTravelsDays, defTravelWithLodge, defPlacesName)<jupyter_output><empty_output><jupyter_text>Example - Travel (From->To + To->From)<jupyter_code>flightsSimulated[0:2]
lodgesSimulated[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1193714113025083, 'fraction_numerical': 0.007026291931097008, 'mean_word_length': 3.1691338582677164, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '144903', 'license': 'CC-BY-4.0', 'n_tokens_mistral': 4177, 'n_tokens_neox': 3963, 'n_words': 917, 'path': 'jupyter/generator-example.ipynb', 'repo_name': 'leomaurodesenv/travel-dataset-generator', 'size': '13236'}