<jupyter_start><jupyter_text>Here we want to see how much each file size would be if we eliminate NDVI<jupyter_code>import csv
import numpy as np
import pandas as pd
from math import factorial

import datetime
from datetime import date
import datetime
import time

import scipy
import os, os.path
from os import listdir
from os.path import isfile, join

import re
# from pprint import pprint
import matplotlib.pyplot as plt
import matplotlib.dates as mdates
import seaborn as sb

import sys


# search path for modules
# look @ https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
sys.path.append('/Users/hn/Documents/00_GitHub/Ag/NASA/Python_codes/')
import NASA_core as nc
import NASA_plot_core as npc<jupyter_output><empty_output><jupyter_text>Set up directories<jupyter_code>data_dir = "/Users/hn/Documents/01_research_data/NASA/VI_TS/sixth_investig_intersected/"
L5 = pd.read_csv(data_dir + "L5_T1C2L2_Scaled_intGrant_2008-01-01_2012-05-05.csv")
L7 = pd.read_csv(data_dir + "L7_T1C2L2_Scaled_intGrant_2008-01-01_2021-09-23.csv")
L8 = pd.read_csv(data_dir + "L8_T1C2L2_Scaled_intGrant_2008-01-01_2021-10-14.csv")
L5.drop(['NDVI'], axis=1, inplace=True)


L7.drop(['NDVI'], axis=1, inplace=True)


L8.drop(['NDVI'], axis=1, inplace=True)

L8.head(2)
L578 = pd.concat([L5, L7, L8])

L578.head(2)
output_dir = data_dir + "/noNDVI/"
os.makedirs(output_dir, exist_ok=True)

out_name = output_dir + "GrantL578.csv"
L578.to_csv(out_name, index = False)

out_name = output_dir + "GrantL5.csv"
L5.to_csv(out_name, index = False)

out_name = output_dir + "GrantL7.csv"
L7.to_csv(out_name, index = False)


out_name = output_dir + "GrantL8.csv"
L8.to_csv(out_name, index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11649365628604383, 'fraction_numerical': 0.058823529411764705, 'mean_word_length': 5.724806201550388, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '284473', 'license': 'MIT', 'n_tokens_mistral': 765, 'n_tokens_neox': 698, 'n_words': 144, 'path': 'NASA/Python_codes/06_sixth_investigation_testFileSize.ipynb', 'repo_name': 'HNoorazar/Kirti', 'size': '1734'}