<jupyter_start><jupyter_text>Load packages and files<jupyter_code>%matplotlib inline
import os
import boto3
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
# download files to SageMaker from S3
jcap_pickle = 'jcap_optical_encoding.pickle'
energy_npy = 'energy_ev.npy'
def hook(t):
  def inner(bytes_amount):
    t.update(bytes_amount)
  return inner
#session = boto3.session.Session(profile_name='hack2020')
#s3 = session.client('s3')
s3 = boto3.client('s3')
for filename in [jcap_pickle, energy_npy]:
    filesize = boto3.resource('s3').Object('hackathon2020-prod', 'data/' + filename).content_length
    with tqdm(total=filesize, unit='B', unit_scale=True, desc=jcap_pickle) as t:
        s3.download_file('hackathon2020-prod', 'data/' + filename, filename, Callback=hook(t))
# load pickle data into dataframe
energy_ev = np.load(energy_npy)
jcap_df = pd.read_pickle(jcap_pickle)
jcap_df.shape
#jcap_df.head()
#jcap_df.columns
#jcap_df.bandgap<jupyter_output><empty_output><jupyter_text>Define features<jupyter_code>def get_bandgap_features(dataframe):
    features = dataframe.drop(['vae','bandgap','spectrum','bandgap_pred'], axis=1, errors='ignore')
    
    return features<jupyter_output><empty_output><jupyter_text>Partition data into seed and candidate data<jupyter_code># Choose every other for train / test
#k_seed_data = jcap_df.iloc[::2]
#k_candidate_data = jcap_df.iloc[1::2]
#k_candidate_data.drop(['bandgap'], axis=1)

# Alternative: choose randomly
k_seed_idx = np.random.rand(len(jcap_df)) < 0.1
k_seed_data = jcap_df[k_seed_idx].copy()
k_candidate_data = jcap_df[~k_seed_idx].copy()
initial_mean_bandgap = np.mean(k_seed_data['bandgap'])
print(len(k_seed_data)/len(k_candidate_data)*100)
print(len(k_seed_data))
print(len(k_candidate_data))
k_seed_data = k_seed_data.sort_values('bandgap',ascending=True)
k_seed_data.head(5)
# test to ensure no overlap between seed and candidate data
assert not set(k_seed_data.index).intersection(k_candidate_data.index)<jupyter_output><empty_output><jupyter_text>Define agent<jupyter_code>from camd.agent.base import HypothesisAgent
from sklearn.ensemble import RandomForestRegressor<jupyter_output><empty_output><jupyter_text>class BandgapAgent(HypothesisAgent):    def get_hypotheses(self, candidate_data, seed_data):         Fit on known data        x_known = get_bandgap_features(seed_data)        y_known = seed_data['bandgap']        regressor = LinearRegression()  *** make changes to the regressor        regressor.fit(x_known, y_known)                 Predict unknown data        x_unknown = get_bandgap_features(candidate_data)        y_predicted = regressor.predict(x_unknown)                 Pick top 5 candidates *** make changes to agent below for additional exploration         Potential changes: predict areas far from explored ones,...        candidate_data['bandgap_pred'] = y_predicted        candidate_data = candidate_data[candidate_data['bandgap_pred'] > 0.0]                candidate_data = candidate_data.sort_values(            'bandgap_pred', ascending=True)        top_candidates = candidate_data.head(5)        return top_candidates<jupyter_code>class RFBandgapAgent(HypothesisAgent):
    def get_hypotheses(self, candidate_data, seed_data):
        # Fit on known data
        x_known = get_bandgap_features(seed_data)
        y_known = seed_data['bandgap']
        regressor = RandomForestRegressor(n_estimators=10)
        regressor.fit(x_known, y_known)
        
        # Predict unknown data
        x_unknown = get_bandgap_features(candidate_data)
        y_predicted = regressor.predict(x_unknown)
        
        # Pick top 5 candidates
        candidate_data['bandgap_pred'] = y_predicted
        candidate_data = candidate_data[candidate_data['bandgap_pred'] > 0.0]
        
        candidate_data = candidate_data.sort_values(
            'bandgap_pred', ascending=True)
        top_candidates = candidate_data.head(1000)
        return top_candidates
bg_agent = RFBandgapAgent()
hypotheses = bg_agent.get_hypotheses(k_candidate_data, k_seed_data)
# see how good the predictions were
hypotheses[['bandgap','bandgap_pred']].head(20)<jupyter_output><empty_output><jupyter_text>Define experiment<jupyter_code>from camd.experiment.base import ATFSampler
k_atf_experiment = ATFSampler(dataframe=jcap_df)
k_atf_experiment.submit(hypotheses)
results = k_atf_experiment.get_results()
results<jupyter_output><empty_output><jupyter_text>Define analyzer<jupyter_code>from camd.analysis import AnalyzerBase
class BandgapAnalyzer(AnalyzerBase):
    
    
    def analyze(self, new_experimental_results, seed_data):
        new_seed = pd.concat(
            [seed_data, new_experimental_results],
        axis=0)
        # Create a summary
        average_new_bandgap = new_experimental_results.bandgap.mean()
        average_dataset_bandgap = new_seed.bandgap.mean()
        new_result_ranks = new_seed.bandgap.rank(pct=True).loc[
            new_experimental_results.index
        ]
        min_new_bandgap = new_experimental_results.bandgap.min()
        min_dataset_bandgap = new_seed.bandgap.min()
        num_new_below_seed_mean_bandgap = sum(i < initial_mean_bandgap for i in new_experimental_results['bandgap'])
        
        summary = pd.DataFrame({
            "average_new_bandgap": [average_new_bandgap],
            "average_dataset_bandgap": [average_dataset_bandgap],
            "average_rank": [new_result_ranks.mean()],
            "min_new_bandgap": [min_new_bandgap],
            "min_dataset_bandgap": [min_dataset_bandgap]
        })
        return summary, new_seed
k_analyzer = BandgapAnalyzer()
summary, new_seed = k_analyzer.analyze(results, k_seed_data)
summary<jupyter_output><empty_output><jupyter_text>Define campaign<jupyter_code>import os
from monty.os import cd
from camd.campaigns.base import Campaign
# Set up folders
os.system('rm -rf test')
os.system('mkdir -p test')
# Reinitialize experiment to clear history
k_atf_experiment = ATFSampler(dataframe=jcap_df)
with cd('test'):
    campaign = Campaign(
        candidate_data=k_candidate_data, 
        seed_data=k_seed_data,
        agent=bg_agent,
        experiment=k_atf_experiment,
        analyzer=k_analyzer
    )
    campaign.auto_loop(initialize=True)
# Pull up some results
history = pd.read_pickle('test/history.pickle')
history
#visualize learning
history.plot(subplots=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09637805067620084, 'fraction_numerical': 0.0077724234416291, 'mean_word_length': 4.614310645724259, 'pattern_counts': {'":': 5, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '467738', 'license': 'Apache-2.0', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2131, 'n_words': 376, 'path': 'cobalt/patrick/jcap_prediction_rf.ipynb', 'repo_name': 'lerandc/tri-hackathon-2020-cobalt', 'size': '6433'}