<jupyter_start><jupyter_text>load raw data<jupyter_code>name = "genesis_efrc_minipipes"  # MPContribs project name
indir = Path(f"/Users/patrick/GoogleDriveLBNL/MaterialsProject/gitrepos/mpcontribs-data/{name}")
config_path = indir / "basic.mpj"
config = json.loads(config_path.read_bytes())

# adding project name and API key to config (TODO: set through minipipes UI)
config["meta"]["mpcontribs"] = {
    "project": name, "apikey": os.environ["MPCONTRIBS_API_KEY"]
}

ped_path = indir / "PED of BMG for PDF 1-29-20_0035-0070.gr"
ped = json.loads(ped_path.read_bytes())

png_path = indir / "DP_spotty_92x70.png"<jupyter_output><empty_output><jupyter_text>init client<jupyter_code># retrieve MPContribs config and init client
# using pop here to avoid saving API key in attachment
mpcontribs_config = config["meta"].pop("mpcontribs")
name = mpcontribs_config["project"]
client = Client(
    host = "lightsources-api.materialsproject.org",
    apikey = mpcontribs_config["apikey"]
)<jupyter_output><empty_output><jupyter_text>prep project<jupyter_code># TODO init MPContribs columns and its units for project (see client.init_columns())
# TODO update `other` in project with columns legend
client.get_project(name)<jupyter_output><empty_output><jupyter_text>prep contribution<jupyter_code>runs_meta = config["runs"]["meta"]
composition = runs_meta[-1]["config"]["composition"]
formula = Composition(composition).get_integer_formula_and_factor()[0]
contrib = {
    "project": name,
    "identifier": "TODO", # usually mp-id, can be custom
    "formula": formula,
    "is_public": True,  # will make this contribution public automatically when project is set to public
    # data, tables and attachments added explicitly below
}
# FYI submitting a contribution with its ID triggers update of this contribution<jupyter_output><empty_output><jupyter_text>data<jupyter_code>names_map = {
    "i_Reduce_Data.Mask_Images.Mask_f": "mask",
    "i_Reduce_Data.Image_to_IQ.Integrate_f": "integrate",
    "i_Reduce_Data.IQ_to_PDF.Transform_f": "transform"
}
keys_maps = [ # len(runs_meta) = 3
    {
        "alpha": "α",
        "edge": "edge",
        "lower_threshold": "thresholds.lower",
        "upper_threshold": "thresholds.upper",
        "smoothing function": "smoothing",
        "vmin": "v.min",
        "vmax": "v.max"
    }, {
        "wavelength (A)": "λ",  # TODO unit Angstrom
        "polarization": "polarization",
        "detector": "detector"
    }, {
        "processor": "processor",
        "mode": "mode",
        "qmax": "q.max",
        "qmin": "q.min",
        "rpoly": "r.poly",
        "rmin": "r.min",
        "rmax": "r.max",
        "step": "step",
        "shift": "shift"
    }
]

flat_data = {}

for idx, meta in enumerate(runs_meta):
    root_key = names_map[meta["name"]]
    keys_map = keys_maps[idx]

    for old_key, new_key in keys_map.items():
        key = f"{root_key}.{new_key}"
        value = meta["config"][old_key]  # TODO add units
        flat_data[key] = value

contrib["data"] = unflatten(flat_data)<jupyter_output><empty_output><jupyter_text>tables<jupyter_code>x, y = "r", "G(r)"
df = DataFrame(data={x: ped["data"][0], y: ped["data"][1]})
df.set_index(x, inplace=True)
df.index.name = f"{x} [Å]"
df.columns.name = "spectral type"
df.attrs["name"] = y
df.attrs["title"] = "Radial Distribution Function"
df.attrs["labels"] = {"value": f"{y} [Å⁻²]"} 
# df.plot(**df.attrs)
contrib["tables"] = [df]<jupyter_output><empty_output><jupyter_text>attachments<jupyter_code>config_attachment = Attachment.from_data("config", config)
contrib["attachments"] = [config_attachment, png_path]<jupyter_output><empty_output><jupyter_text>submit contributions<jupyter_code>client.submit_contributions([contrib])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15188528943175783, 'fraction_numerical': 0.005841741901221455, 'mean_word_length': 4.84031007751938, 'pattern_counts': {'":': 29, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '501830', 'license': 'MIT', 'n_tokens_mistral': 1370, 'n_tokens_neox': 1327, 'n_words': 258, 'path': 'mpcontribs-portal/notebooks/lightsources.materialsproject.org/genesis_efrc_minipipes.ipynb', 'repo_name': 'rkingsbury/MPContribs', 'size': '3766'}