<jupyter_start><jupyter_text>Generating DataFirstly, we need to generate the data by using an (exponential) equation with noise. The data mimics the actual data acquired by experiment.<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code># Percolation Threshold of Electrical Conductivity
# equation  exponential percolation theory
# sigma = sigma_not * ( p - pc  ) ^ t
# sigma the electrical conductivity of the nanocomposite and p is the weight fraction
# sigma_not, pc, t were fitting coefficients 
# t should be from 1.5 to 2.5; in 3D-system expected value = 2
# pc in percentage (0 to 100%), expected value ~ 0.5 % (or 0.005 real number)
def generateData(sigma_not, pc, t, noise, N = 1000, pmax = 25):
    """Generate the data by using an exponential equation with noise.
    
    Keyword arguments:
    sigma_not, pc, t: are fitting coefficients (depending on specific polymers and fillers)
    noise: maximum in percentage flucation from real value
    N: number of data generated
    pmax: Maximum in percent
    """
    
    p = np.random.uniform(float (pc), float (pmax), N)
    p[0] = 1.0001  * p[0]  # force (p -pc) to not be 0 when 1st value pi = pc
    sigma = sigma_not * np.float_power( (p -pc)/100.0 ,t )
    # add noise
    sigma_noise = sigma * (1+ ( 2*np.random.random(N) - 1)*(noise/100.) )

    return p,sigma,sigma_noise

# Use linear data for testing purpose which is easier for linear regression
def generateDataLinear(sigma_not, pc, t, noise, N = 1000, pmax = 25):
    """Generate the data by using an linear equation with noise.
    
    Keyword arguments:
    sigma_not, pc, t: are fitting coefficients (depending on specific polymers and fillers)
    noise: maximum in percentage flucation from real value
    N: number of data generated
    pmax: Maximum in percent
    """
    p = np.random.uniform(float (pc), float (pmax), N)
    p[0] = 1.00001  * p[0]  # force (p -pc) to not be 0 when 1st value pi = pc
    sigma = sigma_not * (np.random.randint(1,5)*p + pc)
    # add noise
    sigma_noise = sigma * (1+ ( 2*np.random.random(N) - 1)*(noise/100.) )

    return p,sigma,sigma_noise<jupyter_output><empty_output><jupyter_text>Generating a sample<jupyter_code>sigma_not = 1000 # S/m
pc        = 0.5 # %
t         = 2   
noise     = 50  # %
p,sigma,sigma_noise = generateData(sigma_not, pc, t, noise)
#print(p)
#print(sigma)
data = pd.DataFrame({'p':p,'sigma':sigma,'sigma_noise':sigma_noise})
data<jupyter_output><empty_output><jupyter_text>Plotting noised data vs original data<jupyter_code>plt.plot(p, sigma_noise, marker='+',linestyle = 'None')
plt.plot(p, sigma, marker='+',linestyle = 'None' )
plt.yscale('log')
plt.ylim(0.001,100)
plt.legend(["Noised sigma","Original sigma"])
plt.show()<jupyter_output><empty_output><jupyter_text>Generating data and Saving to files Polymers:Including* HDPE: High Density Poly Ethylene - a very popular polymer* HDPE treated: to make filler mix well, and expect to increase conductivity Fillers:Including* MWCNT: Multiwall Carbon Nanotube* SWCNT: Single Carbon Nanotube* GNP: Graphene Nanoplate Future note:* Next version of program should predict complex cases where there are multiple polymers reinforced with different fillers Columns in data table* polymer_1 = "HDPE" : First polymer* filler_1  = "MWCNT": First filler* wt_l2     = float  : Weight fraction of 1st filler* polymer_2 = ""     : Second polymer in the composite* ratio_1_2 = float  : If there is second polymer, what ratio between 2 polymers?* filler_2  = ""     : Second filler* wt_l2     = float  : Weight fraction of 2nd filler* foaming   = 1 or 0 : foaming or no foam (unfoamed or solid sample)* conductivity=float : Value of electrical conductivity* owner     = "owner": the sources or owner of data set  Data set 1: HDPE & MWCNT<jupyter_code>polymer_1 = "HDPE"
filler_1  = "MWCNT"
polymer_2 = ""
ratio_1_2 = ""
filler_2  = ""
wt_l2     = ""
foaming   = 0      # no foam
owner     = "data set 1"
filename  = "data/"
sigma_not = 1000 # S/m
pc        = 0.5 # %
t         = 2   
noise     = 50  # %
# Generate data
p,sigma,sigma_noise = generateData(sigma_not, pc, t, noise)
# Save to file
if polymer_2: filename += polymer_1+"-"+polymer_2
else: filename += polymer_1
filename += '_'
if filler_2:  filename += filler_1+"-"+filler_2
else: filename += filler_1
filename += '_' + owner.replace(" ","-") + ".csv"
#print(filename)
# Save to CSV files
if not wt_l2: wt_l2 = p.size*[""]
data_full = pd.DataFrame( {'polymer_1' : p.size*[polymer_1],
                           'polymer_p2': p.size*[polymer_2], 
                           'ratio_1_2' : p.size*[ratio_1_2],
                           'filler_1'  : p.size*[filler_1],
                           'wt_l1' : p,
                           'filler_2' : p.size*[filler_2],
                           'wt_l2' : wt_l2,
                           'foaming' : p.size*[foaming],
                           'conductivity':sigma_noise,  
                           'owner':owner,
                          } )
data_full.to_csv(filename, index=None)

#np.savetxt(filename, (p,sigma_noise) , delimiter=",")

plt.plot(p, sigma_noise, marker='+',linestyle = 'None')
plt.plot(p, sigma, marker='+',linestyle = 'None' )
plt.yscale('log')
plt.ylim(0.001,100)
plt.legend(["Noised sigma","Original sigma"])
plt.show()<jupyter_output><empty_output><jupyter_text>Data set 2: HDPE_treated & MWCNTTreated HDPE => better mix up => more uniform distribution of filler in polymer => higher conductivity<jupyter_code>polymer_1 = "HDPEtreated"
filler_1  = "MWCNT"
polymer_2 = ""
ratio_1_2 = ""
filler_2  = ""
wt_l2     = ""
foaming   = 0      # no foam
owner     = "data set 2"
filename  = "data/"

sigma_not = 5000 # S/m
pc        = 0.3 # %
t         = 2   
noise     = 20  # %
# Generate data
p,sigma,sigma_noise = generateData(sigma_not, pc, t, noise)
# Save to file
if polymer_2: filename += polymer_1+"-"+polymer_2
else: filename += polymer_1
filename += '_'
if filler_2:  filename += filler_1+"-"+filler_2
else: filename += filler_1
filename += '_' + owner.replace(" ","-") + ".csv"
#print(filename)
# Save to CSV files
if not wt_l2: wt_l2 = p.size*[""]
data_full = pd.DataFrame( {'polymer_1' : p.size*[polymer_1],
                           'polymer_p2': p.size*[polymer_2], 
                           'ratio_1_2' : p.size*[ratio_1_2],
                           'filler_1'  : p.size*[filler_1],
                           'wt_l1' : p,
                           'filler_2' : p.size*[filler_2],
                           'wt_l2' : wt_l2,
                           'foaming' : p.size*[foaming],
                           'conductivity':sigma_noise,  
                           'owner':owner,
                          } )
data_full.to_csv(filename, index=None)

#np.savetxt(filename, (p,sigma_noise) , delimiter=",")

plt.plot(p, sigma_noise, marker='+',linestyle = 'None')
plt.plot(p, sigma, marker='+',linestyle = 'None' )
plt.yscale('log')
plt.ylim(0.001,1000)
plt.legend(["Noised sigma","Original sigma"])
plt.show()<jupyter_output><empty_output><jupyter_text>Data set 3: HDPE & SWCNT<jupyter_code>polymer_1 = "HDPE"
filler_1  = "SWCNT"
polymer_2 = ""
ratio_1_2 = ""
filler_2  = ""
wt_l2     = ""
foaming   = 0      # no foam
owner     = "data set 3"
filename  = "data/"

sigma_not = 10000 # S/m
pc        = 0.2 # %
t         = 2   
noise     = 80  # %
# Generate data
p,sigma,sigma_noise = generateData(sigma_not, pc, t, noise)
# Save to file
if polymer_2: filename += polymer_1+"-"+polymer_2
else: filename += polymer_1
filename += '_'
if filler_2:  filename += filler_1+"-"+filler_2
else: filename += filler_1
filename += '_' + owner.replace(" ","-") + ".csv"
#print(filename)
# Save to CSV files
if not wt_l2: wt_l2 = p.size*[""]
data_full = pd.DataFrame( {'polymer_1' : p.size*[polymer_1],
                           'polymer_p2': p.size*[polymer_2], 
                           'ratio_1_2' : p.size*[ratio_1_2],
                           'filler_1'  : p.size*[filler_1],
                           'wt_l1' : p,
                           'filler_2' : p.size*[filler_2],
                           'wt_l2' : wt_l2,
                           'foaming' : p.size*[foaming],
                           'conductivity':sigma_noise,  
                           'owner':owner,
                          } )
data_full.to_csv(filename, index=None)

#np.savetxt(filename, (p,sigma_noise) , delimiter=",")

plt.plot(p, sigma_noise, marker='+',linestyle = 'None')
plt.plot(p, sigma, marker='+',linestyle = 'None' )
plt.yscale('log')
plt.ylim(0.001,10000)
plt.legend(["Noised sigma","Original sigma"])
plt.show()<jupyter_output><empty_output><jupyter_text>Data set 4: HDPE + GNPNote: Polymer is not treated => bad conductivity, high threshold (`pc`) to be conductive<jupyter_code>polymer_1 = "HDPE"
filler_1  = "GNP"
polymer_2 = ""
ratio_1_2 = ""
filler_2  = ""
wt_l2     = ""
foaming   = 0      # no foam
owner     = "data set 4"
filename  = "data/"

sigma_not = 10 # S/m
pc        = 3 # %
t         = 1.5   
noise     = 80  # %
# Generate data
p,sigma,sigma_noise = generateData(sigma_not, pc, t, noise)
# Save to file
if polymer_2: filename += polymer_1+"-"+polymer_2
else: filename += polymer_1
filename += '_'
if filler_2:  filename += filler_1+"-"+filler_2
else: filename += filler_1
filename += '_' + owner.replace(" ","-") + ".csv"
#print(filename)
# Save to CSV files
if not wt_l2: wt_l2 = p.size*[""]
data_full = pd.DataFrame( {'polymer_1' : p.size*[polymer_1],
                           'polymer_p2': p.size*[polymer_2], 
                           'ratio_1_2' : p.size*[ratio_1_2],
                           'filler_1'  : p.size*[filler_1],
                           'wt_l1' : p,
                           'filler_2' : p.size*[filler_2],
                           'wt_l2' : wt_l2,
                           'foaming' : p.size*[foaming],
                           'conductivity':sigma_noise,  
                           'owner':owner,
                          } )
data_full.to_csv(filename, index=None)

#np.savetxt(filename, (p,sigma_noise) , delimiter=",")

plt.plot(p, sigma_noise, marker='+',linestyle = 'None')
plt.plot(p, sigma , marker='+',linestyle = 'None')
plt.yscale('log')
plt.ylim(0.0001,100)
plt.legend(["Noised sigma","Original sigma"])
plt.show()<jupyter_output><empty_output><jupyter_text>Data set 5: HDPE_treated + GNPNow polymer is treated => better conductivity, lower threshold<jupyter_code>polymer_1 = "HDPEtreated"
filler_1  = "GNP"
polymer_2 = ""
ratio_1_2 = ""
filler_2  = ""
wt_l2     = ""
foaming   = 0      # no foam
owner     = "data set 5"
filename  = "data/"

sigma_not = 100 # S/m
pc        = 1 # %
t         = 1.5   
noise     = 50  # %
# Generate data
p,sigma,sigma_noise = generateData(sigma_not, pc, t, noise)
# Save to file
if polymer_2: filename += polymer_1+"-"+polymer_2
else: filename += polymer_1
filename += '_'
if filler_2:  filename += filler_1+"-"+filler_2
else: filename += filler_1
filename += '_' + owner.replace(" ","-") + ".csv"
#print(filename)
# Save to CSV files
if not wt_l2: wt_l2 = p.size*[""]
data_full = pd.DataFrame( {'polymer_1' : p.size*[polymer_1],
                           'polymer_p2': p.size*[polymer_2], 
                           'ratio_1_2' : p.size*[ratio_1_2],
                           'filler_1'  : p.size*[filler_1],
                           'wt_l1' : p,
                           'filler_2' : p.size*[filler_2],
                           'wt_l2' : wt_l2,
                           'foaming' : p.size*[foaming],
                           'conductivity':sigma_noise,  
                           'owner':owner,
                          } )
data_full.to_csv(filename, index=None)

#np.savetxt(filename, (p,sigma_noise) , delimiter=",")

plt.plot(p, sigma_noise, marker='+',linestyle = 'None')
plt.plot(p, sigma, marker='+',linestyle = 'None' )
plt.yscale('log')
plt.ylim(0.001,100)
plt.legend(["Noised sigma","Original sigma"])
plt.show()<jupyter_output><empty_output><jupyter_text>Import ALLs into ONE and save a new file<jupyter_code>import glob
import os

path = 'data/' # use your path
all_files = glob.glob(path + "*.csv")
filename_alldata = "_nanocomposite_data.csv"
li = []

for filename in all_files:
    head, tail = os.path.split(filename)
    if tail  == filename_alldata: continue;
    print(filename)
    df = pd.read_csv(filename, index_col=None, header=0)
    li.append(df)

alldata = pd.concat(li, axis=0, ignore_index=True)
alldata.to_csv(path + filename_alldata, index=None)<jupyter_output>data\HDPEtreated_GNP_data-set-5.csv
data\HDPEtreated_MWCNT_data-set-2.csv
data\HDPE_GNP_data-set-4.csv
data\HDPE_MWCNT_data-set-1.csv
data\HDPE_SWCNT_data-set-3.csv
{'fraction_non_alphanumeric': 0.13635651629072681, 'fraction_numerical': 0.028038847117794485, 'mean_word_length': 2.7216554940250655, 'pattern_counts': {'":': 2, '<': 40, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '48946', 'license': 'MIT', 'n_tokens_mistral': 4828, 'n_tokens_neox': 4403, 'n_words': 1066, 'path': 'Step1_Generation_random_input_data.ipynb', 'repo_name': 'linhhlp/electrical-conductivity-nanocomposite', 'size': '12768'}