<jupyter_start><jupyter_text>Chapter Three - Data Wrangling<jupyter_code>from optimus import Optimus 
op = Optimus("pandas")
df = op.create.dataframe({
    "name": ["optimus", "bumblebee", "eject", 2],
    "job": ["Leader", "Espionage", 1, 3],
    "id": [1, 2, 3, 4],
    "birth": ["22/10/10", "22/08/08", "23/07/07", "22/10/10"]
})
df.display()
print(df.data)
print(type(df.data))<jupyter_output><empty_output><jupyter_text>Converting datatypes<jupyter_code>try:
    df.data["name"].astype(int)
except Exception as e:
    print(e)
df.cols.to_integer("name")
df.cols.to_float("name")
df.cols.to_string("id")
df.cols.to_datetime("birth", "YY/mm/dd")<jupyter_output><empty_output><jupyter_text>Selecting columns<jupyter_code>df.cols.select("job")
df.cols.select(["name", "job"])
df.cols.select(regex="n.*")
df.cols.select(data_type="int")<jupyter_output><empty_output><jupyter_text>Moving columns<jupyter_code>df.cols.move(["name", "job"], "after", "id")
df.cols.move(["name", "job"], "end")<jupyter_output><empty_output><jupyter_text>Renaming columns<jupyter_code>df2 = df.cols.rename(["name", "job"], ["string_name", "string_job"])
df2 = df2.cols.rename("id", "int_id")<jupyter_output><empty_output><jupyter_text>Removing columns<jupyter_code>df.cols.drop("job")
df.cols.select(["job", "name"])<jupyter_output><empty_output><jupyter_text>Input and output columns<jupyter_code>df.cols.upper("name")
df.cols.upper(["name", "job"])
df.cols.upper("name", output_cols="name_upper")
df.cols.upper(["name", "job"], output_cols=["name_upper", "job_upper"])
df.cols.upper(["name", "job"], "upper")
df.cols.upper("*")
df.cols.upper("name").cols.lower("job")<jupyter_output><empty_output><jupyter_text>More managing function<jupyter_code>df.cols.drop("job")
df.cols.keep("id")
df.cols.rename("name", "first_name")
df.cols.duplicate(["name", "id"], ["first_name", "id_number"])
df.cols.set("last_name", None)
df.cols.set("last_name", None).cols.fill_na("last_name", "Placeholder Value")<jupyter_output><empty_output><jupyter_text>String functions<jupyter_code>df.cols.upper("name")
df.cols.pad("id", 3, "0")<jupyter_output><empty_output><jupyter_text>Merge and split<jupyter_code>df = op.create.dataframe({ 
    "xy_position": ["42.8, 7.7", "23.3, 25.1", "35.6, 50.5", "52.7, 67.4"],
    "depth": [10.5, 50.1, 20.2, 97.0]
})
df.cols.unnest("xy_position", separator=", ", output_cols=["x", "y"], drop=True)
df.cols.nest(["xy_position", "depth"], separator=", ", output_col="xyz_position", drop=True)<jupyter_output><empty_output><jupyter_text>Search and replace<jupyter_code>df = op.create.dataframe({
    "name": ["<NAME>", " <NAME>",  "Bumblebee/Maggiolino"]
})

df.cols.replace("name", [", ", "/"], " ", search_by="chars")<jupyter_output><empty_output><jupyter_text>Numeric functions<jupyter_code>df = op.create.dataframe({ "values": [0.5, 2, "3.14"] })
df.cols.sin("values")
df.cols.log("values", 5)<jupyter_output><empty_output><jupyter_text>Date and time Functions<jupyter_code>df = op.create.dataframe({ "date": ["09/01/2021", "30/09/2020", "03/07/2020", "21/01/2030"] })
df.cols.year("date")
df.cols.date_format("date", output_format="%m-%d-%Y")<jupyter_output><empty_output><jupyter_text>URL Functions<jupyter_code>from optimus import Optimus
op = Optimus("pandas") 
 
data = {"A": ["https://www.hi-optimus.com:8080/index.php?a=1"]}  
df = op.create.dataframe(data)
 
df["port"] = df.cols.port("A")
df["subdomain"] = df.cols.sub_domain("A")
df["domain"] = df.cols.domain("A")

df.display()<jupyter_output><empty_output><jupyter_text>Email functions<jupyter_code>df = op.create.dataframe({"A": ["<EMAIL>"]})
 
df["username"] = df.cols.email_username("A")
df["domain"] = df.cols.email_domain("A")

df.display()<jupyter_output><empty_output><jupyter_text>UDF<jupyter_code>data = {"A":[0,1,2,3,4,5],"B":[6,7,8,9,10,11]} 
df = op.create.dataframe(data)
df.display()
def add_two(pandas_series):
    print(type(pandas_series))
    return pandas_series + 2
 
df.cols.apply("A", add_two)

def add_two(single_value):    
    print(type(single_value))
    return single_value + 2
 
df.cols.apply("A", add_two, mode="map").print()
def add_two(value):
    return value + 2

print(df.cols.apply("*", add_two))
df = op.load.file("DCIGNP2AYL.txt")
df.display()
from optimus.functions import F

def haversine(lat1, lon1, lat2, lon2):    
    MILES = 3959
    lat1, lon1, lat2, lon2 = map(F.radians, [lat1, lon1, lat2, lon2])
    dlat = lat2 - lat1 
    dlon = lon2 - lon1         
    a = F.sin(dlat/2)**2 + F.cos(lat1) * F.cos(lat2) * F.sin(dlon/2)**2
    c = 2 * F.asin(F.sqrt(a))  
    total_miles = MILES * c
    print(total_miles)
    return total_miles
%%time
df["distance"] = haversine(40.671, -73.985, df['latitude'], df['longitude'])
import pandas as pd
import numpy as np
pd.Series(np.radians(df.data['latitude'])) - pd.Series(np.radians(df.data['longitude']))
(df.cols.select("distance"))
df
def udf(df):
    return df["id"].cols.cos()+ df["id"].cols.sin()

udf(df)
# df.cols.assign({"a":udf})<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.21863155794933173, 'fraction_numerical': 0.03391182924396569, 'mean_word_length': 6.450222882615156, 'pattern_counts': {'":': 14, '<': 68, '<?xml version=': 0, '>': 68, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '426980', 'license': 'MIT', 'n_tokens_mistral': 2168, 'n_tokens_neox': 2069, 'n_words': 283, 'path': 'Chapter03/DataProcessingwithOptimus_Chapter03.ipynb', 'repo_name': 'arifmudi/Data-Processing-with-Optimus', 'size': '5041'}