<jupyter_start><jupyter_text>Processing data from multiple files The biggest advantage of using code to process data is that a method devised for one file can easily be applied to many.This workbook demonstrates reading data from a file and using functions to repeat chunks of code.  Data are read curves fitted and results plotted for multiple files.  Setting up<jupyter_code># Import modules
import os
import sys
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import linregress

# Show plots within notebooks
%matplotlib inline

# Show module versions
print('Python: {}'.format(sys.version))
print('Pandas: {}'.format(pd.__version__))
print('Numpy: {}'.format(np.__version__))
from matplotlib import __version__ as mplv
print('Matplotlib: {}'.format(mplv))<jupyter_output>Python: 3.8.10 (default, Jun  2 2021, 10:49:15) 
[GCC 9.4.0]
Pandas: 1.3.0
Numpy: 1.21.0
Matplotlib: 3.4.2<jupyter_text>The **os** module This section ensures that you are working in the correct directory and shows some of the features of the _os_ module.<jupyter_code>#  Set current directory as working directory
# (Change this if appropriate)
wdir = os.getcwd()
print(wdir)
#  Join folder and file names to create new paths, using correct separator for your operating system
datadir = os.path.join( wdir, 'data_files' )
print(datadir)
#  Extract file name from a full path
filepath = '/long/path/to/myfile.txt'
os.path.basename(filepath)
#  List directory contents
filelist = os.listdir(datadir)
filelist
#  List data files
datafiles = []
for filename in filelist:
    if filename.endswith('.data'):
        datafiles.append(filename)
datafiles.sort()
print(datafiles)

# Bonus exercise: use 'glob' module to get same results<jupyter_output>['Sample001.data', 'Sample002.data', 'Sample003.data', 'Sample004.data', 'Sample005.data', 'Sample006.data', 'Sample007.data', 'Sample008.data', 'Sample009.data', 'Sample010.data', 'Sample011.data', 'Sample012.data', 'Sample013.data', 'Sample014.data', 'Sample015.data', 'Sample016.data', 'Sample017.data', 'Sample018.data', 'Sample019.data', 'Sample020.data', 'Sample021.data', 'Sample022.data', 'Sample023.data', 'Sample024.data', 'Sample025.data', 'Sample026.data', 'Sample027.data', 'Sample028.data', 'Sample029.data', 'Sample030.data']<jupyter_text>Reading the datafiles<jupyter_code>#  A simple script to read a data file
x = []
y = []
filepath = os.path.join(datadir, 'Sample001.data')

with open(filepath, 'rt') as f:
    groupline = f.readline()  # Skip first line
    for line in f.readlines():
        cols = line.split(',')
        x.append(cols[0])
        y.append(cols[1])
        
x = np.array(x, dtype=float)
y = np.array(y, dtype=float)
print('x: {}'.format(x))
print('y: {}'.format(y))
#  Code that is often reused can be put in a function.
def read_data_file(filepath):
    """Read SampleXXX.data file, returning x and y as numpy arrays."""
    x = []
    y = []
        
    with open(filepath, 'rt') as f:
        groupline = f.readline()  # Skip first line
        for line in f.readlines():
            cols = line.split(',')
            x.append(cols[0])
            y.append(cols[1])
    
    x = np.array(x, dtype=float)
    y = np.array(y, dtype=float)
    
    return x, y
#  Read a data file using the function
filepath = os.path.join(datadir, 'Sample001.data')
x, y = read_data_file(filepath)
print(x)
print(y)
#  Plot the data - it really is this easy!!!
plt.scatter(x, y)<jupyter_output><empty_output><jupyter_text>Calculating the best fit line<jupyter_code>#  Use the linregress function from the scipy.stats module
slope, intercept, r_value, p_value, std_err = linregress(x, y)
os.path.basename(filename)

#  See https://mkaz.com/2012/10/10/python-string-format/ for string format hints
results_text = """
Filename: {filename}
Slope: {slope:.2f}
Intercept: {intercept:.2f}
R-squared: {r2:.2f}
""".format(filename=filename,slope=slope, intercept=intercept, r2=r_value**2)
print(results_text)
#  Calculate coordinates for the ends of the best fit line.
fitted_x = np.array([x.min(), x.max()])  # Take these from the original data
fitted_y = slope * fitted_x + intercept
#  Plot the best fit line
plt.scatter(x, y)
plt.plot(fitted_x, fitted_y, '-')
#  Code that is reused can be put into a function
def calculate_best_fit(x, y):
    """Use linregress to calculate best fit line, returning fitted X and Y arrays for plotting."""
    slope, intercept, r_value, p_value, std_err = linregress(x, y)
    fitted_x = np.array([x.min(), x.max()])
    fitted_y = slope * fitted_x + intercept   
    return fitted_x, fitted_y<jupyter_output><empty_output><jupyter_text>Combine functions into single plotting function<jupyter_code>#  Function to plot figure
def plot_sample_fit(filepath):
    """Read data file, calculate fit to x and y, plot and save figure."""
    x, y = read_data_file(filepath)  # Uses previous function
    fitted_x, fitted_y = calculate_best_fit(x, y)  # Uses previous function
    
    # Plot the figure
    plt.figure()
    plt.plot(x, y, 'o')
    plt.plot(fitted_x, fitted_y, '-x', label='Fitted')
    plt.ylim(0, 80)
    plt.xlabel('x')
    plt.ylabel('y')
    
    # Save the file
    filename = os.path.basename(filepath)
    plt.title(filename[:-5])
    figure_filename = filename.replace('.data', '.png')
    plt.savefig(figure_filename)
    plt.close()
    return
#  Plot all files.  Check the working directory and see all the files plotted.  Just like that!
for datafile in datafiles:
    print(datafile)
    data_file_path = os.path.join(datadir, datafile)
    plot_sample_fit(data_file_path)<jupyter_output>Sample001.data
Sample002.data
Sample003.data
Sample004.data
Sample005.data
Sample006.data
Sample007.data
Sample008.data
Sample009.data
Sample010.data
Sample011.data
Sample012.data
Sample013.data
Sample014.data
Sample015.data
Sample016.data
Sample017.data
Sample018.data
Sample019.data
Sample020.data
Sample021.data
Sample022.data
Sample023.data
Sample024.data
Sample025.data
Sample026.data
Sample027.data
Sample028.data
Sample029.data
Sample030.data
{'fraction_non_alphanumeric': 0.1225296442687747, 'fraction_numerical': 0.038866930171278, 'mean_word_length': 4.360105913503972, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '359731', 'license': 'MIT', 'n_tokens_mistral': 2219, 'n_tokens_neox': 2041, 'n_words': 593, 'path': '1_processing_multiple_files.ipynb', 'repo_name': 'volcan01010/python4geoscientists', 'size': '6072'}