<commit_before>import os

from ce.common import cached, timeit


class FlopocoMissingImplementationError(Exception):
    """Unsynthesizable operator"""


we_min, we_max = 5, 16
wf_min, wf_max = 10, 113
we_range = list(range(we_min, we_max))
wf_range = list(range(wf_min, wf_max))
default_file = 'ce/semantics/area.pkl'
device = 'xc6vlx760'


def get_luts(file_name):
    from bs4 import BeautifulSoup
    with open(file_name, 'r') as f:
        f = BeautifulSoup(f.read())
        app = f.document.application
        util = app.find('section', stringid='XST_DEVICE_UTILIZATION_SUMMARY')
        luts = util.find('item', stringid='XST_NUMBER_OF_SLICE_LUTS')
        return luts.get('value')


@timeit
def _para_synth(op_we_wf):
    import sh
    op, we, wf = op_we_wf
    flo_cmd = []
    if op == 'add':
        flo_cmd += ['FPAdder', we, wf]
    elif op == 'mul':
        flo_cmd += ['FPMultiplier', we, wf, wf]
    wd = 'syn_%d' % os.getpid()
    f = 'flopoco.vhdl'
    g = 'flopoco.ngc'
    h = g + '_xst.xrpt'
    s = sh.echo('run', '-p', device,
                '-ifn', f, '-ifmt', 'VHDL', '-ofn', g, '-ofmt', 'NGC')
    try:
        print('Processing', op, we, wf)
        sh.mkdir('-p', wd)
        sh.cd(wd)
        sh.flopoco(*flo_cmd)
        print('Xilinx', op, we, wf)
        sh.xst(s)
        item = dict(op=op, we=we, wf=wf, value=get_luts(h))
        print(item)
    except sh.ErrorReturnCode:
        print('Error processing', op, we, wf)
        item = dict(op=op, we=we, wf=wf, value=-1)
    except KeyboardInterrupt:
        pass
    sh.cd('..')
    sh.rm('-rf', wd)
    return item


pool = None


@timeit
def synth(we_range, wf_range):
    import itertools
    from multiprocessing import Pool
    global pool
    if not pool:
        pool = Pool(8)
    args = itertools.product(['add', 'mul'], we_range, wf_range)
    return list(pool.imap_unordered(_para_synth, args))


def load(file_name):
    import pickle
    with open(file_name, 'rb') as f:
        return pickle.loads(f.read())


def save(file_name, results):
    import pickle
    with open(file_name + '.pkl', 'wb') as f:
        pickle.dump(results, f)


def plot(results):
    import matplotlib.pyplot as plt
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = Axes3D(fig)
    vl = []
    for i in results:
        xv, yv, zv = int(i['we']), int(i['wf']), int(i['value'])
        if zv < 0:
            continue
        vl.append((xv, yv, zv))
    ax.scatter(*zip(*vl))
    plt.show()


if not os.path.isfile(default_file):
    print('area statistics file does not exist, synthesizing...')
    save(default_file, synth(we_range, wf_range))

_add = {}
_mul = {}
for i in load(default_file):
    xv, yv, zv = int(i['we']), int(i['wf']), int(i['value'])
    if zv <= 0:
        continue
    if i['op'] == 'add':
        _add[xv, yv] = zv
    elif i['op'] == 'mul':
        _mul[xv, yv] = zv


def _impl(_dict, we, wf):
    try:<commit_msg>Refactor ce.semantics.flopoco for real area<commit_after>import os
import sh
from contextlib import contextmanager

from ce.common import cached, timeit


class FlopocoMissingImplementationError(Exception):
    """Unsynthesizable operator"""


we_min, we_max = 5, 16
wf_min, wf_max = 10, 113
we_range = list(range(we_min, we_max))
wf_range = list(range(wf_min, wf_max))
default_file = 'ce/semantics/area.pkl'
device = 'xc6vlx760'


@contextmanager
def cd(d):
    p = os.path.abspath(os.curdir)
    if d:
        sh.mkdir('-p', d)
        sh.cd(d)
    yield
    sh.cd(p)


def get_luts(file_name):
    from bs4 import BeautifulSoup
    with open(file_name, 'r') as f:
        f = BeautifulSoup(f.read())
        app = f.document.application
        util = app.find('section', stringid='XST_DEVICE_UTILIZATION_SUMMARY')
        luts = util.find('item', stringid='XST_NUMBER_OF_SLICE_LUTS')
        return luts.get('value')


def flopoco(op, we, wf, f):
    flo_fn = 'flopoco.vhdl'
    flo_cmd = []
    if op == 'add':
        flo_cmd += ['FPAdder', we, wf]
    elif op == 'mul':
        flo_cmd += ['FPMultiplier', we, wf, wf]
    wd, fn = os.path.split(f)
    with cd(wd):
        try:
            sh.flopoco(*flo_cmd)
            if fn != flo_fn:
                sh.mv(flo_fn, fn)
        except sh.ErrorReturnCode as e:
            print('Flopoco failed', op, we, wf)
    return op, we, wf, f


def xilinx(op, we, wf, f):
    wd, fn = os.path.split(f)
    g = os.path.splitext(fn)[0] + '.ngc'
    with cd(wd):
        try:
            sh.xst(sh.echo('run', '-p', device,
                           '-ifn', fn, '-ifmt', 'VHDL',
                           '-ofn', g, '-ofmt', 'NGC'))
            item = dict(op=op, we=we, wf=wf, value=get_luts(g + '_xst.xrpt'))
        except sh.ErrorReturnCode as e:
            print('Xilinx failed', op, we, wf)
            item = dict(op=op, we=we, wf=wf, value=-1)
    if wd:
        sh.rm('-rf', wd)
    return item


def synth_eval(op, we, wf):
    work_dir = 'syn_%d' % os.getpid()
    f = os.path.join(work_dir, '%s_%d_%d.vhdl' % (op, we, wf))
    print('Processing', op, we, wf)
    item = xilinx(*flopoco(op, we, wf, f=f))
    print(item)
    return item


@timeit
def _para_synth(op_we_wf):
    return synth_eval(*op_we_wf)


pool = None


@timeit
def batch_synth(we_range, wf_range):
    import itertools
    from multiprocessing import Pool
    global pool
    if not pool:
        pool = Pool(8)
    args = itertools.product(['add', 'mul'], we_range, wf_range)
    return list(pool.imap_unordered(_para_synth, args))


def load(file_name):
    import pickle
    with open(file_name, 'rb') as f:
        return pickle.loads(f.read())


def save(file_name, results):
    import pickle
    with open(file_name + '.pkl', 'wb') as f:
        pickle.dump(results, f)


def plot(results):
    import matplotlib.pyplot as plt
    from mpl_toolkits.mplot3d import Axes3D
    fig = plt.figure()
    ax = Axes3D(fig)
    vl = []
    for i in results:
        xv, yv, zv = int(i['we']), int(i['wf']), int(i['value'])
        if zv < 0:
            continue
        vl.append((xv, yv, zv))
    ax.scatter(*zip(*vl))
    plt.show()


if not os.path.isfile(default_file):
    print('area statistics file does not exist, synthesizing...')
    save(default_file, batch_synth(we_range, wf_range))

_add = {}
_mul = {}
for i in load(default_file):
    xv, yv, zv = int(i['we']), int(i['wf']), int(i['value'])
    if zv <= 0:
        continue
    if i['op'] == 'add':
        _add[xv, yv] = zv
    elif i['op'] == 'mul':
        _mul[xv, yv] = zv


def _impl(_dict, we, wf):
    try:
{'fraction_non_alphanumeric': 0.15510141833155405, 'fraction_numerical': 0.006252859539423517, 'mean_word_length': 2.558328811720022, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1322581', 'n_tokens_mistral': 2639, 'n_tokens_neox': 2548, 'n_words': 604}