<jupyter_start><jupyter_text>1. Bucketing for one-week (at least) assumption<jupyter_code>from datetime import timedelta

def getCampaignOneWeekRevenue(df):
    df["Bucket Index"] = 0
    minDate = df["Date"].min() - df["Date"].min().dayofweek * timedelta(days=1)
    # Get the number of weeks difference between each date and first week in the dataset
    df["Bucket Weeks"] = (df["Date"] - minDate).apply(lambda x : int(x.days / 7))
    uniqueDeltas = np.sort(np.unique(df["Bucket Weeks"]))

    bucketIndex = 0
    for i in range(len(uniqueDeltas)):
        weekIx = uniqueDeltas[i]
        where = df["Bucket Weeks"] == weekIx
        items = df[where]
        # Both conditions must apply. Unless we get both some costs and some revenues, we can't complete
        #  the bucket.
        if items["Cost"].sum() > 0 and items["Revenue"].sum() > 0:
            df.loc[where, "Bucket Index"] = bucketIndex
            bucketIndex += 1
            continue
        # We must be careful at "end" buckets, where diff between weeks is larger than 1.
        if i > 0 and uniqueDeltas[i] != uniqueDeltas[i - 1] + 1:
            bucketIndex += 1
        df.loc[where, "Bucket Index"] = bucketIndex
    return df

# Return a new column called "bucket index", which represents on what (Cost, Revenue) bucket each entry goes
# This is unique for all campaigns, so bucket index == 1 for Campaign "A", is a different bucket from bucket
#  index == 1 for Campaign "B". This column is used to train on the same logical data (Bucket, Revenue), and
#  we don't get Costs from same bucket both in train and validation set.
# Other assumptions (like weekly assumption) can update this indexes as they will to provide other logic of
#  implementation for the dataset split.
def getOneWeekRevenue(df):
    newDf = pd.DataFrame(columns=df.columns)
    campaigns = df["Campaign"].unique()
    for campaign in campaigns:
        dfCampaign = df[df["Campaign"] == campaign].reset_index(drop=True)
        newDfCampaign = getCampaignOneWeekRevenue(dfCampaign)
        numBuckets = len(newDfCampaign["Bucket Index"].unique())
        print("Campaign '%s'" % (campaign))
        print(" - Unique weeks: %d" % (len(newDfCampaign["Bucket Weeks"].unique())))
        print(" - (Cost, Revenue) buckets: %d" % (numBuckets))
        if numBuckets < 20:
            print(" - Below 20 buckets, skipping.")
            continue

        revenues = newDfCampaign["Revenue"]
        # print(campaign, np.percentile(revenues, [0.1, 1, 10, 25, 50, 75, 90, 99, 99.5, 99.9, 100]))
        # Clip the revenues to top 99.5%, to eliminate outliers.
        Max = newDfCampaign["Revenue"].max()
        Top99_9 = np.percentile(newDfCampaign["Revenue"], 99.9)
        print(" - Pruning max revenue from %2.3f to %2.3f" % (Max, Top99_9))
        newDfCampaign["Label Revenue"] = np.clip(newDfCampaign["Revenue"], 0, Top99_9)
        newDf = pd.concat([newDf, newDfCampaign], sort=False)      
    return newDf

oneWeekRevenueDf = getOneWeekRevenue(validDf)
print("___________________________________")
print("Original df: %s. Training df: %s" % (validDf.shape, oneWeekRevenueDf.shape))
trainDf = getPseudoRevenues(oneWeekRevenueDf)
trainDf["Bucket Index"] = trainDf["Bucket Index"].astype(int)<jupyter_output><empty_output><jupyter_text>1. Just Cost<jupyter_code>Stats = getMinMax(trainDf)
from sklearn.linear_model import LinearRegression
models, errors = trainAllCampaignsLR(trainDf, Stats, LinearRegression, features=["Cost"])
display(pd.DataFrame(errors.items(), columns=["Campaign", "Error (L1)"]))
predictedDf = createPseudoRevenues(models, trainDf, Stats, features=["Cost"])
print(predictedDf.shape)
sumPredictedDf = createSumPredictedPseudoRevenue(predictedDf)
print(sumPredictedDf.shape)
scatterRevenues(sumPredictedDf)
plotSummedRevenues(sumPredictedDf)
LROverallError = getOverallError(sumPredictedDf)
display(LROverallError)<jupyter_output><empty_output><jupyter_text>2. One extra feature<jupyter_code>trainDfTwoFeatures = pd.get_dummies(trainDf, columns=["Device Category"])

Stats = getMinMax(trainDfTwoFeatures)
features = ["Cost", "Device Category_desktop", "Device Category_mobile", "Device Category_tablet"]
models, errors = trainAllCampaignsLR(trainDfTwoFeatures, Stats, LinearRegression, features=features)
display(pd.DataFrame(errors.items(), columns=["Campaign", "Error (L1)"]))

predictedDf2F = createPseudoRevenues(models, trainDfTwoFeatures, Stats, features=features)
sumPredictedDf2F = createSumPredictedPseudoRevenue(predictedDf2F)
scatterRevenues(sumPredictedDf2F)
plotSummedRevenues(sumPredictedDf2F)
display(getOverallError(sumPredictedDf2F))
trainDfManyFeatures = pd.get_dummies(trainDf, columns=["Device Category"])
for col in trainDfManyFeatures:
    if col == "Bucket Index":
        continue
    try:
        trainDfManyFeatures[col] = trainDfManyFeatures[col].astype(np.float32)
    except Exception:
        pass
Stats = getMinMax(trainDfManyFeatures)

features = ["Cost", "Device Category_desktop", "Device Category_mobile", "Device Category_tablet", \
            "Users", "Sessions", "Bounces", "Transactions", "Product Detail Views", \
            "Product Adds To Cart", "Product Checkouts", "Total Unique Searches", \
            "New Users", "Session Duration (s)"]
models, errors = trainAllCampaignsLR(trainDfManyFeatures.reset_index(drop=True).copy(), Stats, LinearRegression, features=features)
display(pd.DataFrame(errors.items(), columns=["Campaign", "Error (L1)"]))

predictedDfManyF = createPseudoRevenues(models, trainDfManyFeatures, Stats, features=features)
sumPredictedDfManyF = createSumPredictedPseudoRevenue(predictedDfManyF)
scatterRevenues(sumPredictedDfManyF)
plotSummedRevenues(sumPredictedDfManyF)
display(getOverallError(sumPredictedDfManyF))
trainDfAllFeatures = pd.get_dummies(trainDf, columns=["Device Category"])
for col in trainDfAllFeatures:
    if col == "Bucket Index":
        continue
    try:
        trainDfAllFeatures[col] = trainDfAllFeatures[col].astype(np.float32)
    except Exception:
        pass
Stats = getMinMax(trainDfAllFeatures)

features = ["Cost", "Device Category_desktop", "Device Category_mobile", "Device Category_tablet", \
            "Users", "Sessions", "Bounces", "Transactions", "Product Detail Views", \
            "Product Adds To Cart", "Product Checkouts", "Total Unique Searches", \
            "New Users", "Session Duration (s)", "Impressions", "Clicks"]
models, errors = trainAllCampaignsLR(trainDfAllFeatures.reset_index(drop=True).copy(), Stats, LinearRegression, features=features)
display(pd.DataFrame(errors.items(), columns=["Campaign", "Error (L1)"]))

predictedDfAllF = createPseudoRevenues(models, trainDfAllFeatures, Stats, features=features)
sumPredictedDfAllF = createSumPredictedPseudoRevenue(predictedDfAllF)
scatterRevenues(sumPredictedDfAllF)
plotSummedRevenues(sumPredictedDfAllF)
display(getOverallError(sumPredictedDfAllF))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11862208129143846, 'fraction_numerical': 0.011386566733929087, 'mean_word_length': 4.875529212531752, 'pattern_counts': {'":': 2, '<': 14, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '626618', 'license': 'MIT', 'n_tokens_mistral': 2312, 'n_tokens_neox': 2179, 'n_words': 528, 'path': 'Notebooks/5. Budget allocation - pseudo-revenue - one-week assumption - regressions .ipynb', 'repo_name': 'Morphl-AI/Marketing-Spend-Optimization-for-Online-Retailers', 'size': '6938'}