<commit_before># ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

import os
import re
import subprocess

from hydra_agent.utils import normalize_device, Mounts
from hydra_agent import shell
from hydra_agent.plugins import ActionPlugin


def get_local_targets():
    blkid_lines = shell.try_run(['blkid', '-s', 'UUID']).split("\n")

    lustre_devices = []

    for line in [l for l in blkid_lines if len(l)]:
        dev, uuid = re.search("(.*): UUID=\"(.*)\"", line).groups()
        dev = normalize_device(dev)

        rc, tunefs_text, stderr = shell.run(["tunefs.lustre", "--dryrun", dev])
        if rc != 0:
            # Not lustre
            continue

        # For a Lustre block device, extract name and params
        # ==================================================
        name = re.search("Target:\\s+(.*)\n", tunefs_text).group(1)
        flags = int(re.search("Flags:\\s+(0x[a-fA-F0-9]+)\n", tunefs_text).group(1), 16)
        params_re = re.search("Parameters:\\ ([^\n]+)\n", tunefs_text)
        if params_re:
            # Dictionary of parameter name to list of instance values
            params = {}
            # FIXME: naive parse: can these lines be quoted/escaped/have spaces?
            for param, value in [t.split('=') for t in params_re.group(1).split()]:
                if not param in params:
                    params[param] = []
                params[param].append(value)
        else:
            params = {}

        if name.find("ffff") != -1:
            # Do not report unregistered lustre targets
            continue

        mounted = dev in set([normalize_device(m[0]) for m in Mounts().all()])

        lustre_devices.append({
            "name": name,
            "uuid": uuid,
            "params": params,
            "device": dev,
            "mounted": mounted
            })
        if flags & 0x0005 == 0x0005:
            # For combined MGS/MDT volumes, synthesise an 'MGS'
            lustre_devices.append({
                "name": "MGS",
                "uuid": uuid,
                "params": params,
                "device": dev,
                "mounted": mounted
                })

    return lustre_devices


def get_mgs_targets(local_targets):
    """If there is an MGS in local_targets, use debugfs to
       get a list of targets.  Return a dict of filesystem->(list of targets)"""
    TARGET_NAME_REGEX = "([\w-]+)-(MDT|OST)\w+"
    mgs_target = None
    for t in local_targets:
        if t["name"] == "MGS":
            mgs_target = t
    if not mgs_target:
        return ({}, {})

    conf_params = {}
    mgs_targets = {}
    dev = mgs_target["device"]
    ls = shell.try_run(["debugfs", "-c", "-R", "ls -l CONFIGS/", dev])
    filesystems = []
    targets = []
    for line in ls.split("\n"):
        try:
            name = line.split()[8]
            size = line.split()[5]
        except IndexError:
            pass
<commit_msg>HYD-451<commit_after># ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

import os
import re
import subprocess

from hydra_agent.utils import normalize_device, Mounts
from hydra_agent import shell
from hydra_agent.plugins import ActionPlugin


def get_local_targets():
    blkid_lines = shell.try_run(['blkid', '-s', 'UUID']).split("\n")

    # Working set: accumulate device paths for each (uuid, name).  This is
    # necessary because in multipathed environments we will see the same
    # lustre target on more than one block device.  The reason we use name
    # as well as UUID is that two logical targets can have the same UUID
    # when we see a combined MGS+MDT
    uuid_name_to_target = {}

    for line in [l for l in blkid_lines if len(l)]:
        dev, uuid = re.search("(.*): UUID=\"(.*)\"", line).groups()
        dev = normalize_device(dev)

        rc, tunefs_text, stderr = shell.run(["tunefs.lustre", "--dryrun", dev])
        if rc != 0:
            # Not lustre
            continue

        # For a Lustre block device, extract name and params
        # ==================================================
        name = re.search("Target:\\s+(.*)\n", tunefs_text).group(1)
        flags = int(re.search("Flags:\\s+(0x[a-fA-F0-9]+)\n", tunefs_text).group(1), 16)
        params_re = re.search("Parameters:\\ ([^\n]+)\n", tunefs_text)
        if params_re:
            # Dictionary of parameter name to list of instance values
            params = {}
            # FIXME: naive parse: can these lines be quoted/escaped/have spaces?
            for param, value in [t.split('=') for t in params_re.group(1).split()]:
                if not param in params:
                    params[param] = []
                params[param].append(value)
        else:
            params = {}

        if name.find("ffff") != -1:
            # Do not report unregistered lustre targets
            continue

        mounted = dev in set([normalize_device(m[0]) for m in Mounts().all()])

        if flags & 0x0005 == 0x0005:
            # For combined MGS/MDT volumes, synthesise an 'MGS'
            names = ["MGS", name]
        else:
            names = [name]

        for name in names:
            try:
                target_dict = uuid_name_to_target[(uuid, name)]
                target_dict['devices'].append(dev)
            except KeyError:
                target_dict = {"name": name,
                               "uuid": uuid,
                               "params": params,
                               "devices": [dev],
                               "mounted": mounted}
                uuid_name_to_target[(uuid, name)] = target_dict

    return uuid_name_to_target.values()


def get_mgs_targets(local_targets):
    """If there is an MGS in local_targets, use debugfs to
       get a list of targets.  Return a dict of filesystem->(list of targets)"""
    TARGET_NAME_REGEX = "([\w-]+)-(MDT|OST)\w+"
    mgs_target = None
    for t in local_targets:
        if t["name"] == "MGS":
            mgs_target = t
    if not mgs_target:
        return ({}, {})

    conf_params = {}
    mgs_targets = {}
    dev = mgs_target["devices"][0]
    ls = shell.try_run(["debugfs", "-c", "-R", "ls -l CONFIGS/", dev])
    filesystems = []
    targets = []
    for line in ls.split("\n"):
        try:
            name = line.split()[8]
            size = line.split()[5]
        except IndexError:
            pass

{'fraction_non_alphanumeric': 0.16024844720496895, 'fraction_numerical': 0.009006211180124223, 'mean_word_length': 1.9986033519553073, 'pattern_counts': {'":': 17, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4542491', 'n_tokens_mistral': 2008, 'n_tokens_neox': 1873, 'n_words': 541}