<commit_before># -*- coding: utf-8 -*-
import os, tarfile, json, re, urllib
import numpy as np
from pandas import DataFrame
from mpcontribs.io.core.recdict import RecursiveDict
from mpcontribs.io.core.utils import nest_dict
from monty.json import MontyDecoder
from mpcontribs.users.jarvis_dft.rest.rester import JarvisDftRester

def clean_value(val, unit):
    return '-' if val == 'na' else '{} {}'.format(val, unit)

def run(mpfile, nmax=10, dup_check_test_site=True):

    # book-keeping
    existing_mpids = {}
    for b in [False, True]:
        with JarvisDftRester(test_site=b) as mpr:
            for doc in mpr.query_contributions(criteria=mpr.query):
                existing_mpids[doc['mp_cat_id']] = doc['_id']
        if not dup_check_test_site:
            break

    for typ in ['2d', '3d']:

        url = mpfile.hdata['_hdata']['input_url'].format(typ)
        dbfile = url.rsplit('/')[-1]

        if not os.path.exists(dbfile):
            print 'downloading', dbfile, '...'
            url_opener = urllib.URLopener()
            url_opener.retrieve(url, dbfile)

        else:
            print 'unpacking', dbfile, '...'
            with tarfile.open(dbfile, "r:gz") as tar:
                member = tar.getmembers()[0]
                d = json.load(tar.extractfile(member), cls=MontyDecoder)
                print len(d), 'materials loaded.'

                for idx,i in enumerate(d):
                    mpid = i['mpid']
                    print 'adding', mpid, '...'
                    data = RecursiveDict()
                    data['jid'] = i['jid']
                    data['formula'] = i['final_str'].composition.reduced_formula
                    data['spacegroup'] = i['final_str'].get_space_group_info()[0]
                    data['final_energy'] = clean_value(i["fin_en"], 'eV')
                    data['optB88vDW_bandgap'] = clean_value(i["op_gap"], 'eV')
                    data['mbj_bandgap'] = clean_value(i["mbj_gap"], 'eV')
                    data['bulk_modulus'] = clean_value(i["kv"], 'GPa')
                    data['shear_modulus'] = clean_value(i["gv"], 'GPa')
                    mpfile.add_hierarchical_data(nest_dict(data, ['data', typ]), identifier=mpid)
                    mpfile.add_structure(i['final_str'], name=data['formula'], identifier=mpid)

                    if mpid in existing_mpids:
                        cid = existing_mpids[mpid]
                        mpfile.insert_id(mpid, cid)
                        print cid, 'inserted to update', mpid

                    if idx >= nmax-1:
                        break

            print 'DONE with', typ<commit_msg>jarvis_dft: adapt for mgc load<commit_after># -*- coding: utf-8 -*-
import os, tarfile, json, re, urllib, certifi
import numpy as np
from pandas import DataFrame
from mpcontribs.io.core.recdict import RecursiveDict
from mpcontribs.io.core.utils import nest_dict
from monty.json import MontyDecoder
from mpcontribs.users.utils import duplicate_check

def clean_value(val, unit):
    return '-' if val == 'na' else '{} {}'.format(val, unit)

@duplicate_check
def run(mpfile):

    for typ in ['2d', '3d']:

        url = mpfile.hdata.general['input_url'].format(typ)
        dbfile = os.path.join(os.environ['HOME'], 'work', url.rsplit('/')[-1])

        if not os.path.exists(dbfile):
            print 'downloading', dbfile, '...'
            urllib.urlretrieve(url, dbfile)
            #resp = urlrq.urlopen('https://foo.com/bar/baz.html', cafile=certifi.where())

        else:
            print 'unpacking', dbfile, '...'
            with tarfile.open(dbfile, "r:gz") as tar:
                member = tar.getmembers()[0]
                d = json.load(tar.extractfile(member), cls=MontyDecoder)
                print len(d), 'materials loaded.'

                for idx,i in enumerate(d):
                    mpid = i['mpid']
                    print 'adding', mpid, '...'
                    data = RecursiveDict()
                    data['jid'] = i['jid']
                    data['formula'] = i['final_str'].composition.reduced_formula
                    data['spacegroup'] = i['final_str'].get_space_group_info()[0]
                    data['final_energy'] = clean_value(i["fin_en"], 'eV')
                    data['optB88vDW_bandgap'] = clean_value(i["op_gap"], 'eV')
                    data['mbj_bandgap'] = clean_value(i["mbj_gap"], 'eV')
                    data['bulk_modulus'] = clean_value(i["kv"], 'GPa')
                    data['shear_modulus'] = clean_value(i["gv"], 'GPa')
                    mpfile.add_hierarchical_data(nest_dict(data, ['data', typ]), identifier=mpid)
                    mpfile.add_structure(i['final_str'], name=data['formula'], identifier=mpid)

            print 'DONE with', typ
{'fraction_non_alphanumeric': 0.14180672268907563, 'fraction_numerical': 0.003991596638655462, 'mean_word_length': 2.0676546391752577, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4910905', 'n_tokens_mistral': 1547, 'n_tokens_neox': 1454, 'n_words': 299}