<jupyter_start><jupyter_text><jupyter_code>import time
import urllib
import json
import pandas as pd

import warnings
warnings.filterwarnings('ignore')
alldf = []

def filter(word):
  # Gak boleh lebih dari 1 kata
  if len(word.split())>1:
    return False
  # Gak boleh kosong, ''
  if len(word)==0:
    return False
  # Gak boleh selain huruf dan strip
  for letter in word:
    if letter not in 'abcdefghijklmnopqrstuvwxyz-':
      return False
  # Gak boleh strip di awal atau di akhir kata
  if word[0] == '-' or word[-1] == '-':
    return False
  # Lulus filter
  return True

def pipeline(dataframe):
  dataframe = dataframe.dropna()
  dataframe['kata'] = dataframe['kata'].apply(lambda word: word.lower().strip())
  dataframe = dataframe[dataframe['kata'].apply(filter)]
  dataframe = dataframe.drop_duplicates()
  alldf.append(dataframe)
  return dataframe
# Source 1: https://github.com/bgli/kbbi-qt

df1 = pd.read_csv('https://raw.githubusercontent.com/bgli/kbbi-qt/master/data-src/kbbi_data.csv', sep='\t')
df1 = df1[['katakunci']]
df1 = df1.rename(columns={'katakunci':'kata'})

df1 = pipeline(df1)
df1
# Source 2: https://github.com/rizaumami/kbbi
# dari KBBI edisi 4 http://bse.mahoni.com/data/umum/Kamus_Bahasa_Indonesia_2008.pdf

kosakata = []
for i in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ':
  url = "https://raw.githubusercontent.com/rizaumami/kbbi/master/teks/hasil_akhir/{}.txt".format(i)
  txt_file = urllib.request.urlopen(url)
  time.sleep(1)

  for line in txt_file:
    decoded_line = line.decode("utf-8")
    try:
      kata = decoded_line.split()[0]
    except:
      continue
    kosakata.append(kata)

df2 = pd.DataFrame()
df2['kata'] = kosakata

df2 = pipeline(df2)
df2
# Source 3: https://github.com/damzaky/kumpulan-kata-bahasa-indonesia-KBBI
# reverse engineering database file from the official KBBI apk

df3 = pd.read_csv('https://raw.githubusercontent.com/damzaky/kumpulan-kata-bahasa-indonesia-KBBI/master/indonesian-words.txt', sep='\n', names=['kata'])

df3 = pipeline(df3)
df3
# Source 4: https://github.com/muzavan/KBBI-Crawling
# dari http://kbbi.web.id

with urllib.request.urlopen("https://raw.githubusercontent.com/muzavan/KBBI-Crawling/master/kamus.json") as url:
  data = json.loads(url.read())

df4 = pd.DataFrame()
df4['kata']=data.keys()

df4 = pipeline(df4)
df4
# Source 5: https://github.com/bbn-bernard/kbbi_dataset
# dari http://kbbi.web.id

df5 = pd.read_csv('https://raw.githubusercontent.com/bbn-bernard/kbbi_dataset/master/kbbi_dataset.csv')
df5 = df5[['kata_dasar']]
df5 = df5.rename(columns={'kata_dasar':'kata'})

df5 = pipeline(df5)
df5
# Source 6: https://github.com/bekicot/indonesian_words
# Scrapped from the official KBBI website

kosakata = []
for i in 'abcdefghijklmnopqrstuvwxyz':
  url = "https://raw.githubusercontent.com/bekicot/indonesian_words/master/words/{}".format(i)
  txt_file = urllib.request.urlopen(url)
  time.sleep(1)

  for line in txt_file:
    kata = line.decode("utf-8")
    kosakata.append(kata)

df6 = pd.DataFrame()
df6['kata'] = kosakata

df6 = pipeline(df6)
df6
# Source 7: https://github.com/lufias69/KBBI2

with urllib.request.urlopen("https://raw.githubusercontent.com/lufias69/KBBI2/master/cek_KBBI/dataKBBI.json") as url:
  data = json.loads(url.read())

df7 = pd.DataFrame()
df7['kata'] = data

df7 = pipeline(df7)
df7
# combine source

text = 'huruf'
for i in range(len(alldf)):
  text += f'\tdf{i+1}'
text += '\tunion'
print(text)

kosakata = []
for letter in 'abcdefghijklmnopqrstuvwxyz':
  union = set([])

  def first_letter(word):
    if word[0] == letter:
      return True
    else:
      return False
    
  text = f'{letter}'
  for df in alldf:
    s = set(df[df['kata'].apply(first_letter)]['kata'])
    union = union.union(s)
    text += f'\t{len(s)}'
  text += f'\t{len(union)}'

  print(text)
  kosakata += list(union)
df = pd.DataFrame()
df['kata'] = kosakata
df
df.to_csv('kbbi.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1522967329894375, 'fraction_numerical': 0.01719479243429133, 'mean_word_length': 3.611551528878822, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 14, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '351575', 'license': 'MIT', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1629, 'n_words': 287, 'path': 'KBBI_Github.ipynb', 'repo_name': 'Hidayathamir/kbbi-github', 'size': '4071'}