<jupyter_start><jupyter_text>Analytics server<jupyter_code>from os import listdir
# Data to serve with our API
datasets = listdir("data")
datasets
import json
json.dumps(datasets)<jupyter_output><empty_output><jupyter_text>Working With Files in Pythonhttps://realpython.com/working-with-files-in-python/<jupyter_code>entries = os.listdir('data/')
for entry in entries:
    print(entry)
import os

with os.scandir('data/') as entries:
    for entry in entries:
        print(entry.name)
import os

with os.scandir("data") as dir_entries:
    for entry in dir_entries:
        info = entry.stat()
        print(info)<jupyter_output>os.stat_result(st_mode=33206, st_ino=0, st_dev=0, st_nlink=0, st_uid=0, st_gid=0, st_size=14937, st_atime=1590573945, st_mtime=1530305963, st_ctime=1590573945)
os.stat_result(st_mode=33206, st_ino=0, st_dev=0, st_nlink=0, st_uid=0, st_gid=0, st_size=25830, st_atime=1590573945, st_mtime=1408008491, st_ctime=1590573945)<jupyter_text>Same results with *pathlib*<jupyter_code>from pathlib import Path
current_dir = Path('data')
for path in current_dir.iterdir():
    info = path.stat()
    print(info.st_mtime)
from datetime import datetime
from os import scandir

def convert_date(timestamp):
    d = datetime.utcfromtimestamp(timestamp)
    formated_date = d.strftime('%d %b %Y')
    return formated_date

def get_files():
    dir_entries = scandir('data/')
    for entry in dir_entries:
        if entry.is_file():
            info = entry.stat()
            print(f'{entry.name}\t Last Modified: {convert_date(info.st_mtime)}')
 get_files()<jupyter_output>cars.csv	 Last Modified: 29 Jun 2018
data.csv	 Last Modified: 14 Aug 2014<jupyter_text>Retrieve file propertieshttps://www.w3resource.com/python-exercises/python-basic-exercise-107.php<jupyter_code>import os.path
import time

print('File         :', "data/cars.csv")
print('Access time  :', time.ctime(os.path.getatime("data/cars.csv")))
print('Modified time:', time.ctime(os.path.getmtime("data/cars.csv")))
print('Change time  :', time.ctime(os.path.getctime("data/cars.csv")))
print('Size         :', os.path.getsize("data/cars.csv"))
from hurry.filesize import size

def file_info():
    dir_entries = scandir('data/')
    for entry in dir_entries:
        name = entry.name
        lastModified = entry.stat().st_mtime
        filesize = size(entry.stat().st_size)
        print(name)
        print(lastModified)
        print(filesize)
file_info()
dir_entries = scandir('data/')
for entry in dir_entries:
    if entry.is_file():
        info = entry.stat()
        print(info)
 import pandas as pd

 df = pd.DataFrame(columns= ['name','lastModified','filesize'])

 df
dir_entries = scandir('data/')
for entry in dir_entries:
     df = df.append({'name' : entry.name, 'lastModified' : entry.stat().st_mtime, 'filesize' : size(entry.stat().st_size)}, ignore_index=True)    

# df.to_json("dataset.json",orient='records')
df.to_json(orient='records')
df
df = pd.DataFrame(columns=['A'])
for i in range(5):
    df = df.append({'A': i}, ignore_index=True)

df<jupyter_output><empty_output><jupyter_text>Append DataFramehttps://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.append.html  Better way to create json file from multiple lists?https://stackoverflow.com/questions/50519818/better-way-to-create-json-file-from-multiple-lists/50520116<jupyter_code>import json

devices = ['iphone', 'ipad', 'ipod', 'watch']
cities = ['NY', 'SFO', 'LA', 'NJ']
companies = ['Apple', 'Samsung', 'Walmart']

lists = ['devices', 'cities', 'companies']

data = {listname: globals()[listname] for listname in lists}
with open('abc.json', 'w') as outfile:
    json.dump(data, outfile, indent=4)

import pandas as pd

devices = ['iphone', 'ipad', 'ipod', 'watch']
cities = ['NY', 'SFO', 'LA', 'NJ']
companies = ['Apple', 'Samsung', 'Walmart']

df = pd.DataFrame(list(zip(devices,cities,companies)),columns =['devices', 'cities','companies'])
df
df.to_json("df.json",orient='records')<jupyter_output><empty_output><jupyter_text>Reading JSON from a Filehttps://stackabuse.com/reading-and-writing-json-to-a-file-in-python/<jupyter_code>import json
from datetime import datetime

def get_timestamp():
    return datetime.now().strftime(("%Y-%m-%d %H:%M:%S"))


data = {}
data['people'] = []
data['people'].append({
    "fname": "Doug",
    "lname": "Farrell",
    "timestamp": get_timestamp()
})
data['people'].append({
    "fname": "Kent",
    "lname": "Brockman",
    "timestamp": get_timestamp()
})
data['people'].append({
    "fname": "Bunny",
    "lname": "Easter",
    "timestamp": get_timestamp()
})

with open('people.txt', 'w') as outfile:
    json.dump(data, outfile)

with open('people.txt') as json_file:
    data = json.load(json_file)
    for p in data['people']:
        print('Name: ' + p['fname'])
        print('Website: ' + p['lname'])
        print('From: ' + p['timestamp'])
        print('')
import dataset

data = dataset.read()
data
json.loads(data)<jupyter_output><empty_output><jupyter_text>Request analytics server<jupyter_code>import requests as rq
res = rq.get("http://127.0.0.1:8000/api/dataset")
res.json()
# files = {'data_classification.csv': open('data_classification.csv', 'rb')}
files = {'file': ('data_classification.csv', open('data_classification.csv', 'rb'))}
r = rq.post("http://127.0.0.1:8000/api/dataset", files=files)
r.content<jupyter_output><empty_output><jupyter_text>Example of post request:https://requests.readthedocs.io/en/v0.8.2/user/quickstart/post-a-multipart-encoded-file<jupyter_code>url = 'http://httpbin.org/post'
files = {'data_classification.csv': open('data_classification.csv', 'r')}
r = rq.post(url, files=files)
r.json()
f = open('data_classification.csv', 'r')
f.seek(0,os.SEEK_END)
fsize = f.tell()
fsize
from hurry.filesize import size
size(fsize)
files = {'file': ('data_classification.csv', open('data_classification.csv', 'rb'))}
r = rq.post("http://127.0.0.1:8000/api/dataset", files=files)
r.content<jupyter_output><empty_output><jupyter_text>Get insights into the datasets<jupyter_code>import json
dataset = "cars.csv"
r = rq.get("http://127.0.0.1:8000/api/dataset/"+dataset)
cars = json.loads(r.json())
cars[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1689777491131893, 'fraction_numerical': 0.027249274427603998, 'mean_word_length': 5.039922103213242, 'pattern_counts': {'":': 9, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '444817', 'license': 'MIT', 'n_tokens_mistral': 2395, 'n_tokens_neox': 2249, 'n_words': 419, 'path': 'analytics.ipynb', 'repo_name': 'slevin48/analytics-server', 'size': '6202'}