<commit_before>import hedValidator from 'hed-validator'
import path from 'path'
import semver from 'semver'
import utils from '../../utils'
const Issue = utils.issues.Issue

export default function checkHedStrings(events, headers, jsonContents, dir) {
  let issues = []

  // find specified version in sidecar
  const schemaDefinition = {}
  const datasetDescription = jsonContents['/dataset_description.json']

  if (datasetDescription && datasetDescription.HEDVersion) {
    if (semver.valid(datasetDescription.HEDVersion)) {
      schemaDefinition.version = datasetDescription.HEDVersion
    } else {
      schemaDefinition.path = path.join(
        path.resolve(dir),
        'sourcedata',
        datasetDescription.HEDVersion,
      )
    }
  }

  // run HED validator
  return hedValidator.validator
    .buildSchema(schemaDefinition)
    .then(hedSchema => {
      let hedStringsFound = false
      const sidecarIssues = {}
      let sidecarIssuesFound = false
      // loop through event data files
      events.forEach(eventFile => {
        const hedStrings = []
        // get the json sidecar dictionary associated with the event data
        const potentialSidecars = utils.files.potentialLocations(
          eventFile.path.replace('.tsv', '.json'),
        )
        // validate the HED strings in the json sidecars
        for (const sidecarName of potentialSidecars) {
          if (!(sidecarName in sidecarIssues)) {
            const sidecarDictionary = jsonContents[sidecarName]
            const sidecarHedStrings = []
            for (const sidecarKey in sidecarDictionary) {
              const sidecarValue = sidecarDictionary[sidecarKey]
              if (
                sidecarValue !== null &&
                typeof sidecarValue === 'object' &&
                sidecarValue.HED !== undefined
              ) {
                sidecarHedStrings.push(sidecarValue.HED)
              }
            }
            let fileIssues = []
            for (const hedString of sidecarHedStrings) {
              const [
                isHedStringValid,
                hedIssues,
              ] = hedValidator.validator.validateHedString(
                hedString,
                hedSchema,<commit_msg>Validate string values instead of container object<commit_after>import hedValidator from 'hed-validator'
import path from 'path'
import semver from 'semver'
import utils from '../../utils'
const Issue = utils.issues.Issue

export default function checkHedStrings(events, headers, jsonContents, dir) {
  let issues = []

  // find specified version in sidecar
  const schemaDefinition = {}
  const datasetDescription = jsonContents['/dataset_description.json']

  if (datasetDescription && datasetDescription.HEDVersion) {
    if (semver.valid(datasetDescription.HEDVersion)) {
      schemaDefinition.version = datasetDescription.HEDVersion
    } else {
      schemaDefinition.path = path.join(
        path.resolve(dir),
        'sourcedata',
        datasetDescription.HEDVersion,
      )
    }
  }

  // run HED validator
  return hedValidator.validator
    .buildSchema(schemaDefinition)
    .then(hedSchema => {
      let hedStringsFound = false
      const sidecarIssues = {}
      let sidecarIssuesFound = false
      // loop through event data files
      events.forEach(eventFile => {
        const hedStrings = []
        // get the json sidecar dictionary associated with the event data
        const potentialSidecars = utils.files.potentialLocations(
          eventFile.path.replace('.tsv', '.json'),
        )
        // validate the HED strings in the json sidecars
        for (const sidecarName of potentialSidecars) {
          if (!(sidecarName in sidecarIssues)) {
            const sidecarDictionary = jsonContents[sidecarName]
            let sidecarHedStrings = []
            for (const sidecarKey in sidecarDictionary) {
              const sidecarValue = sidecarDictionary[sidecarKey]
              if (
                sidecarValue !== null &&
                typeof sidecarValue === 'object' &&
                sidecarValue.HED !== undefined
              ) {
                if (typeof sidecarValue.HED === 'string') {
                  sidecarHedStrings.push(sidecarValue.HED)
                } else {
                  sidecarHedStrings = sidecarHedStrings.concat(
                    Object.values(sidecarValue.HED),
                  )
                }
              }
            }
            let fileIssues = []
            for (const hedString of sidecarHedStrings) {
              const [
                isHedStringValid,
                hedIssues,
              ] = hedValidator.validator.validateHedString(
                hedString,
                hedSchema,
{'fraction_non_alphanumeric': 0.08271631055637825, 'fraction_numerical': 0.0, 'mean_word_length': 2.062176165803109, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2257975', 'n_tokens_mistral': 1193, 'n_tokens_neox': 1179, 'n_words': 295}