<jupyter_start><jupyter_text>Мерджер<jupyter_code>from tqdm.notebook import tqdm
import glob
import json
import pandas as pd
def merge_jsons(sourcepath='../data/vk_*_0*/*.json', savepath='../data/vk-api-saved-all-30-04.json'):
    files = glob.glob(sourcepath)

    def fillFalse(x):
        return None if isinstance(x, bool) else x
    
    print('Number of crawled user pages: {}'.format(len(files)))

    raw = []

    for fpath in tqdm(files):
        with open(fpath) as f:
            content = json.load(f)
            if 'execute_errors' not in content:
                content['execute_errors'] = [dict(method='users.get', error_code=None, error_msg=None),
                                             dict(method='friends.get', error_code=None, error_msg=None),
                                             dict(method='users.getFollowers', error_code=None, error_msg=None),
                                             dict(method='groups.get', error_code=None, error_msg=None),
                                             dict(method='wall.get', error_code=None, error_msg=None)]
            content['response'] = dict(user=content['response'][0], 
                                       friends=fillFalse(content['response'][1]),
                                       followers=fillFalse(content['response'][2]),
                                       groups=fillFalse(content['response'][3]),
                                       wall=fillFalse(content['response'][4]))
            raw.append(content)
            
    df = pd.DataFrame(raw)
#     df.to_json(savepath, force_ascii=False, orient='records', lines=True)
    return df
data = merge_jsons()<jupyter_output>Number of crawled user pages: 72012<jupyter_text>__________________________<jupyter_code># from pyspark.sql import SparkSession
# from pyspark.sql import *
# from pyspark.sql.functions import *

# spark = (SparkSession.builder
#          .config('spark.executor.memory', '4g')
#          .config('spark.driver.memory','4g')
#          .appName('local_spark')
#          .getOrCreate()         
#         )
# sc = spark.sparkContext
errors = pd.DataFrame(data.execute_errors.explode().to_list())
sum(~data.execute_errors.apply(lambda x : any([_['error_code']==29 for _ in x])))
sum(data.response.apply(lambda x : all([len(x[_])>0 if x[_] is not None else False for _ in x])))
data.response
def friends_error(x):
    if x is not None:
        return [_ for _ in x if _['method']=='friends.get']
    else:
        return []
sum((data.response.apply(lambda x : x['friends'] is not None)) | \
    (data.execute_errors.apply(friends_error).apply(lambda x : x[0]['error_code']!=29 if len(x)>0 else True)))
valid = (data.response.apply(lambda x : x['friends'] is not None)) | \
    (data.execute_errors.apply(friends_error).apply(lambda x : x[0]['error_code']!=29 if len(x)>0 else True))
savepath='../data/vk-api-saved-all-30-04.json'
data[valid].to_json(savepath, force_ascii=False, orient='records', lines=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14394440767703506, 'fraction_numerical': 0.010589013898080741, 'mean_word_length': 2.577514792899408, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '604229', 'license': 'MIT', 'n_tokens_mistral': 938, 'n_tokens_neox': 902, 'n_words': 174, 'path': 'notebooks/0.0.-json-parsing-draft.ipynb', 'repo_name': 'kimaril/leadership-practice-practice', 'size': '3022'}