<jupyter_start><jupyter_text>Purpose: Final Formatting and column creation for our dataset!<jupyter_code>import pandas as pd
df = pd.read_csv('data/polity_cultivar_pomo_usda.csv')
df.head()
def afn(_afn):
    _afn = _afn.artist_full_name
    if isinstance(_afn, str):
        _afn = _afn.split(',')
        _afn = [x.strip() for x in _afn]
        _afn = _afn[0:2]
        _afn = ''.join(_afn)
    return _afn
df['artist_full_name'] = df.apply(lambda x: afn(x), axis=1)
df.head()
def foc(_x):
    _x = _x['geographic_origin']
    if isinstance(_x, str):
        _x = _x.split(',')[-1]
        _x = _x.strip()
    return _x
df['fruit_origin_country'] = df.apply(lambda x: foc(x), axis=1)
df.head()
from collections import Counter
Counter(list(df['fruit_origin_country']))

def fop(_x):
    _x = _x['geographic_origin']
    if isinstance(_x, str):
        try:
            _x = _x.split(',')[-2]
        except:
            _x = None
    return _x
df['fruit_origin_polity'] = df.apply(lambda x: fop(x), axis=1)
df.head()
def iurl(_x):
    _x = _x['pomological_Id']
    _x = f'/{_x}.jpg'
    return _x
df['image_url'] = df.apply(lambda x: iurl(x), axis=1)
df.head()
df['is_usda_recognized_plant_family']=df['is_usda_recognized']
final = ['artist_full_name', 'cultivar_name', 'fruit_common_name', 'fruit_origin_country',
        'fruit_origin_polity', 'fruit_scientific_name', 'fruit_variety', 'image_created_year',
        'image_url', 'is_originpolity_largest_producer', 'is_usda_recognized_plant_family',
        'pomological_Id']
final_df = df[final]
final_df.index.name = '_index'
final_df
final_df.to_csv('data/final.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15416916616676665, 'fraction_numerical': 0.004799040191961607, 'mean_word_length': 4.024096385542169, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '295405', 'license': 'MIT', 'n_tokens_mistral': 676, 'n_tokens_neox': 653, 'n_words': 122, 'path': 'final_df.ipynb', 'repo_name': 'aloutfi/usda-dl', 'size': '1667'}