<jupyter_start><jupyter_text>Load The Data<jupyter_code>import json
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
%matplotlib inline
# Settings
data_path = '../adil-crawler/export/peraturan_2021-05-09'
data_path_out = 'peraturan_fixed.json'
df = pd.read_json(data_path)
df.head()
# Get Data Frame Information
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 269 entries, 0 to 268
Data columns (total 29 columns):
 #   Column                                 Non-Null Count  Dtype  
---  ------                                 --------------  -----  
 0   id                                     269 non-null    object 
 1   source                                 269 non-null    object 
 2   document                               269 non-null    object 
 3   document_status                        269 non-null    object 
 4   jenis_peraturan                        269 non-null    object 
 5   nomor_peraturan                        269 non-null    object 
 6   tahun_peraturan                        269 non-null    int64  
 7   tentang                                269 non-null    object 
 8   tgl_ditetapkan                         196 non-null    object 
 9   nomor_ln                               151 non-null    float64
 10  nomor_tln                              131 non-null    float64
 11  tgl_di[...]<jupyter_text>Convert JSON Data<jupyter_code># Load json data and turn into dictionaries

with open(data_path, 'r', encoding='utf-8') as f:
    data = json.loads(f.read())
# Normalized all
normal_df = pd.json_normalize(data)
# Enlarge column width
normal_df.head().style.set_properties(subset=['source'], **{'width': '100px'})
normal_df.info()
# Set Max Column Display
pd.set_option("display.max_columns", 100)
# 
normal_df.head().style.set_properties(subset=['source'], **{'width': '100px'})<jupyter_output><empty_output><jupyter_text>Exploratory Analysis  Document Column<jupyter_code>multi_docs = normal_df['document'].apply(lambda x: True if len(x) > 1 else False)

# Jumlah peraturan dengan >1 dokumen
multi_docs[multi_docs].count()
normal_df.info()
# Find if peraturan have no document
no_doc = normal_df['document'].apply(lambda x: True if len(x) == 0 else False)
no_doc[no_doc].count()<jupyter_output><empty_output><jupyter_text>Visualize Null Data<jupyter_code>sns.heatmap(normal_df.isnull(), yticklabels=False, cbar=False, cmap='viridis')<jupyter_output><empty_output><jupyter_text>Category Columns<jupyter_code>normal_df['category']
empty_category = normal_df['category'].apply(lambda x: True if len(x) == 0 else False)
empty_category[empty_category]
# Number of no category
empty_category[empty_category].sum()
# Fill the empty category
def fill_empty_cat(x):
    if len(x) == 0:
        x.append('Uncategorized')
    return x

normal_df['category'] = normal_df['category'].apply(lambda x: fill_empty_cat(x))
normal_df['category']
category_list = []
for c in normal_df['category']:
    for x in c:
        category_list.append(x)
cat_series = pd.Series(category_list)
cat_series
cat_series.nunique()
pd.set_option("display.max_rows", 38)
cat_series.value_counts().to_frame()
cat_series_w_cat = cat_series[cat_series != 'Uncategorized']
plt.figure(figsize=(12, 15))
plot = sns.countplot(y=cat_series_w_cat).set_title('Categories Count')
plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Jenis Peraturan Column<jupyter_code>normal_df['jenis_peraturan']
# Find Null Values
normal_df['jenis_peraturan'].isna().sum()
# Find unique values
normal_df['jenis_peraturan'].nunique()
# Count each unique values on how many appearances
normal_df['jenis_peraturan'].value_counts()
# Plot the count for jenis peraturan column
plt.figure(figsize=(10, 12))
sns.countplot(y=normal_df['jenis_peraturan'])<jupyter_output><empty_output><jupyter_text>Tanggal Diterapkan Column<jupyter_code># initiate the series
tanggal = normal_df['tgl_ditetapkan']
tanggal
# find NaN values
tanggal[tanggal.isna()]
# Fill in missing data
normal_df['tgl_ditetapkan'] = tanggal.fillna(normal_df['tahun_peraturan'])
normal_df['tgl_ditetapkan']
normal_df.tail()<jupyter_output><empty_output><jupyter_text>Nomor Peraturan Column<jupyter_code>nomor = normal_df['nomor_peraturan']
nomor
nomor.nunique()
# Find Missing Values
nomor[nomor.isna()]
# Fill missing Values
normal_df['nomor_peraturan'] = nomor.fillna('Tidak ada nomor peraturan')
# Check if the data only contains number
nomor = normal_df['nomor_peraturan']
num_only = nomor.apply(lambda x: True if "/" not in x else False)
sns.countplot(num_only)<jupyter_output>/Users/raisazka/opt/anaconda3/lib/python3.8/site-packages/seaborn/_decorators.py:36: FutureWarning: Pass the following variable as a keyword arg: x. From version 0.12, the only valid positional argument will be `data`, and passing other arguments without an explicit keyword will result in an error or misinterpretation.
  warnings.warn(<jupyter_text>Drop columns that have a lot of nulls<jupyter_code>normal_df.info()
# Drop it
normal_df.drop(['nomor_ln', 'nomor_tln', 'nomor_ld', 'nomor_tld', 'nomor_bn', 'nomor_tbn'], axis=1, inplace=True)
normal_df.head()
normal_df.info()
sns.heatmap(normal_df.isnull(), yticklabels=False, cbar=False,cmap='viridis')<jupyter_output><empty_output><jupyter_text>Document Status Column<jupyter_code># Check if first document have 404 status
null_doc = df['document_status'].apply(lambda x: True if x[0] == 404 else False)
null_doc.sum()
null_doc[null_doc]
# Validate the data if there is really no document
normal_df[normal_df.index == 9]
# Drop column that have 404 in the first document and create new dataframe
non_blank_doc_df = normal_df.drop(normal_df[null_doc].index)
non_blank_doc_df<jupyter_output><empty_output><jupyter_text>Convert DataFrame to JSON<jupyter_code># Normalize more data with the original dataframe
ids = normal_df['id']
normal_df.drop(axis=1, columns='id', inplace=True)
normal_df.set_index(ids, inplace=True)
normal_df.info()
new_json = normal_df.to_json(path_or_buf=data_path_out, orient='index', indent=2, )
with open(data_path_out, 'r') as f:
    json_out = json.loads(f.read())
json_out<jupyter_output><empty_output><jupyter_text>Download The PDF<jupyter_code>!pip install requests
# Download the document files using wget and the non_blank_doc dataframe
download = True # Add download toggler to prevent accidental document download
DATASET_DIR = "../../adil-dataset/"

if download:
    import requests
    headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"}
    series_df = pd.concat([non_blank_doc_df['id'], non_blank_doc_df['document']], axis=1)
    for idx, docs in series_df.iterrows():
        doc_id = docs['id']
        res = requests.get(docs['document'][0], stream=True, headers=headers)
        try:
            with open(DATASET_DIR+doc_id+".0.pdf", "wb") as pdf:
                pdf.write(res.content)
            print("Downloaded File {}.pdf".format(doc_id))
        except Exception as e:
            print(e)
            print("Unable to download file {} with status code {}".format(doc_id, res.status_code))<jupyter_output>Downloaded File 11e44c4ed09124909b9c313231363531.pdf
Downloaded File 11e44c4ea421ce60bfd2313231353337.pdf
Downloaded File 11e44c4e535c1b70a21e313231333231.pdf
Downloaded File 11e44c4e9a83706087ca313231353231.pdf
Downloaded File 11e44c4ecfb12b10a863313231363530.pdf
Downloaded File 11e44c4f108c0cc0adec313231383339.pdf
Downloaded File 11e44c4e532e88e0bdc6313231333231.pdf
Downloaded File 11e44c4e8cdb8ea0b662313231343538.pdf
Downloaded File 11e44c4ee0e61110b06e313231373139.pdf
Downloaded File 11e44c4e869f6ba0a085313231343437.pdf
Downloaded File 11e44c4f0e9849508816313231383335.pdf
Downloaded File 11e44c4f45359700b4f7313232303037.pdf
Downloaded File 11e44c4e2bd7a2b0b18f313231323135.pdf
Downloaded File 11e44c4e34bd91809c19313231323330.pdf
Downloaded File 11e44c4ea7a1fd009b91313231353433.pdf
Downloaded File 11e47c637035b46ca598313634353235.pdf
Downloaded File 11e44c4f468072f0b0d6313232303039.pdf
Downloaded File 11e44c4ef5221bf0938c313231373533.pdf
Downloaded File 11e44c4e52e4cb50b2723132313332[...]
{'fraction_non_alphanumeric': 0.10521164668460974, 'fraction_numerical': 0.0758502569121605, 'mean_word_length': 4.5423728813559325, 'pattern_counts': {'":': 1, '<': 47, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '238179', 'license': 'MIT', 'n_tokens_mistral': 3260, 'n_tokens_neox': 2872, 'n_words': 600, 'path': 'adil-data-exploration/exploration.ipynb', 'repo_name': 'patriciafiona/adil-monorepo', 'size': '8174'}