<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
"""Codec for processing vulnerability curves
from XML files.

A DOM version of the vulnerability model parser,
that takes into account the really small size of this input file.

"""

import json

from lxml import etree
from ordereddict import OrderedDict

from opengem import identifiers
from opengem import logs
from opengem import memcached
from opengem import shapes
from opengem.xml import NRML

logger = logs.RISK_LOG

EMPTY_CODE = 'EMPTY'

def register_vuln_curves(vuln_curves, job_id, memcache_client=None):
    """This fuction writes a set of vulnerability curves to memcache.
    It requires a dict as input, which holds the vulnerability curve codes
    as keys, and the corresponding FastCurve objects as values.
    job_id is part of the memcache key that points to the curves.
    The function returns a dict with the curves."""

    #old input values: vuln_curve_code, vuln_curve
    if memcache_client is None:
        memcache_client = memcached.get_client(binary=False)

    # get already registered curves
    vulnerability_curves = \
        load_vulnerability_curves_from_memcache(memcache_client, job_id)

    # append new curves, or update existing curves
    vulnerability_curves.update(vuln_curves)

    # write to memcache
    write_vulnerability_curves_to_memcache(memcache_client, job_id,
                                           vulnerability_curves)

    return vulnerability_curves

def delete_vuln_curves(job_id, memcache_client=None):
    """Deletes vulnerability curves for a given job_id from memcache."""

    if memcache_client is None:
        memcache_client = memcached.get_client(binary=False)

    memcache_client.delete(generate_memcache_key(job_id))

def load_vulnerability_model(job_id, path, memcache_client=None):
    """Loads and registers all the vulnerability functions defined."""

    vulnerability_curves = {}

    # register empty curve
    vulnerability_curves[EMPTY_CODE] = shapes.EMPTY_CURVE.to_json()

    vulnerability_model = etree.parse(path).getroot()

    intensity_measure_values = [float(x) for x in
            vulnerability_model.find(
            ".//%sValues" % NRML).text.strip().split()]
    
    for vulnerability_function in vulnerability_model.findall(
            ".//%sDiscreteVulnerability" % NRML):

        # parse values
        loss_ratio_values = [float(x) for x in
                vulnerability_function.find(
                '%sLossRatios' % NRML).text.strip().split()]
                
        coefficient_variation_values = [float(x) for x in
                vulnerability_function.find(
                '%sCoefficientsVariation' % NRML).text.strip().split()]

        # convert raw values into our object model
        curve_data = []
        pairs = zip(loss_ratio_values, coefficient_variation_values)
        
        for idx, IML in enumerate(intensity_measure_values):
            curve_data.append(('%s' % IML, pairs[idx]))

        vulnerability_curves[vulnerability_function.attrib['ID']] = \
            shapes.FastCurve(curve_data).to_json()<commit_msg>Refactored vulnerability parser matching the new schema<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
"""Codec for processing vulnerability curves
from XML files.

A DOM version of the vulnerability model parser,
that takes into account the really small size of this input file.

"""

import json

from lxml import etree
from ordereddict import OrderedDict

from opengem import identifiers
from opengem import logs
from opengem import memcached
from opengem import shapes
from opengem.xml import NRML

logger = logs.RISK_LOG

EMPTY_CODE = 'EMPTY'

def register_vuln_curves(vuln_curves, job_id, memcache_client=None):
    """This fuction writes a set of vulnerability curves to memcache.
    It requires a dict as input, which holds the vulnerability curve codes
    as keys, and the corresponding FastCurve objects as values.
    job_id is part of the memcache key that points to the curves.
    The function returns a dict with the curves."""

    #old input values: vuln_curve_code, vuln_curve
    if memcache_client is None:
        memcache_client = memcached.get_client(binary=False)

    # get already registered curves
    vulnerability_curves = \
        load_vulnerability_curves_from_memcache(memcache_client, job_id)

    # append new curves, or update existing curves
    vulnerability_curves.update(vuln_curves)

    # write to memcache
    write_vulnerability_curves_to_memcache(memcache_client, job_id,
                                           vulnerability_curves)

    return vulnerability_curves

def delete_vuln_curves(job_id, memcache_client=None):
    """Deletes vulnerability curves for a given job_id from memcache."""

    if memcache_client is None:
        memcache_client = memcached.get_client(binary=False)

    memcache_client.delete(generate_memcache_key(job_id))

def load_vulnerability_model(job_id, path, memcache_client=None):
    """Loads and registers all the vulnerability functions defined."""

    vulnerability_curves = {}

    # register empty curve
    vulnerability_curves[EMPTY_CODE] = shapes.EMPTY_CURVE.to_json()

    vulnerability_model = etree.parse(path).getroot()

    intensity_measure_values = [float(x) for x in
            vulnerability_model.find(
            ".//%sValues" % NRML).text.strip().split()]
    
    for vulnerability_function in vulnerability_model.findall(
            ".//%sDiscreteVulnerability" % NRML):

        loss_ratio_values = [float(x) for x in
                vulnerability_function.find(
                '%sLossRatios' % NRML).text.strip().split()]
                
        coefficient_variation_values = [float(x) for x in
                vulnerability_function.find(
                '%sCoefficientsVariation' % NRML).text.strip().split()]

        # convert raw values into our object model
        curve_data = []
        pairs = zip(loss_ratio_values, coefficient_variation_values)
        
        for idx, IML in enumerate(intensity_measure_values):
            curve_data.append(('%s' % IML, pairs[idx]))

        vulnerability_curves[vulnerability_function.attrib['ID']] = \
            shapes.FastCurve(curve_data).to_json()
{'fraction_non_alphanumeric': 0.07618896134881502, 'fraction_numerical': 0.0012724669953873072, 'mean_word_length': 3.2833787465940056, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2623511', 'n_tokens_mistral': 1863, 'n_tokens_neox': 1841, 'n_words': 503}