<jupyter_start><jupyter_text>spark init<jupyter_code>Z="ok"
H={'/home/user/HADOOP': 'C:\\HADOOP',
  '/home/user/HADOOP/spark-2.3.0-bin-hadoop2.7': 'C:\\HADOOP\\spark-2.3.0-bin-hadoop2.7',
  '/home/user/Program Files/java/8.0.172-zulu': 'C:\\Program Files\\Java\\jdk1.8.0_181',
  '/home/user/TMP/SPARK': 'C:\\TMP\\SPARK'}
H=dict([(k,k) for k in H.keys()])
WARE=H['/home/user/TMP/SPARK']

def start():
    import time
    from pyspark.sql import SparkSession
    res = SparkSession.builder.master('local').appName('test').getOrCreate()
    return res

def stop(sc):
    print("bye")
    sc.stop()

def preview(df):
    df.show(n=3, truncate=False)

import os
os.environ['HADOOP_HOME'] = H['/home/user/HADOOP']
os.environ['SPARK_HOME'] = H['/home/user/HADOOP/spark-2.3.0-bin-hadoop2.7']
os.environ['JAVA_HOME'] = H['/home/user/Program Files/java/8.0.172-zulu']

WARE=H['/home/user/TMP/SPARK']
os.makedirs(WARE, exist_ok=True)
os.chmod(WARE, 0o777)
#spark=start()
#print(spark.version)
print(Z)
import sys
MODULES_PATH = '../code/'
if MODULES_PATH not in sys.path:
    sys.path.append(MODULES_PATH)
import mfuncs
import pandas as pd
import numpy as np
from tqdm import tqdm
tqdm.pandas()
df_train = pd.read_csv('../data/train_set.csv')
df_test = pd.read_csv('../data/test_set.csv')
rnm = {
    'atm_address_lat': 'atm_lat',
    'atm_address_lon': 'atm_lon',
    'pos_adress_lat': 'pos_lat',
    'pos_adress_lon': 'pos_lon',
    'home_add_lat': 'home_lat',
    'home_add_lon': 'home_lon',
    'work_add_lat': 'work_lat',
    'work_add_lon': 'work_lon',
}
df_train.rename(columns=rnm, inplace=True)
df_test.rename(columns=rnm, inplace=True)
pd.options.display.max_columns = 1000
import lightgbm as lgb

# %pylab inline<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1928817451205511, 'fraction_numerical': 0.022962112514351322, 'mean_word_length': 4.481132075471698, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '360463', 'license': 'Apache-2.0', 'n_tokens_mistral': 776, 'n_tokens_neox': 765, 'n_words': 113, 'path': 'Raif/pyspark/01Prepare.ipynb', 'repo_name': 'musicnova/7a_task', 'size': '1742'}