<commit_before>'use strict'

const Wappalyzer = {
  technologies: [],
  categories: [],

  slugify(string) {
    return string
      .toLowerCase()
      .replace(/[^a-z0-9-]/g, '-')
      .replace(/--+/g, '-')
      .replace(/(?:^-|-$)/, '')
  },

  getTechnology(name) {
    return Wappalyzer.technologies.find(({ name: _name }) => name === _name)
  },

  getCategory(id) {
    return Wappalyzer.categories.find(({ id: _id }) => id === _id)
  },

  /**
   * Resolve promises for implied technology.
   * @param {Array} detections
   */
  resolve(detections = []) {
    const resolved = detections.reduce((resolved, { technology }) => {
      if (
        resolved.findIndex(
          ({ technology: { name } }) => name === technology.name
        ) === -1
      ) {
        let version = ''
        let confidence = 0

        detections.forEach(
          ({ technology: { name }, pattern, version: _version = '' }) => {
            if (name === technology.name) {
              confidence = Math.min(100, confidence + pattern.confidence)
              version =
                _version.length > version.length && _version.length <= 10
                  ? _version
                  : version
            }
          }
        )

        resolved.push({ technology, confidence, version })
      }

      return resolved
    }, [])

    Wappalyzer.resolveExcludes(resolved)
    Wappalyzer.resolveImplies(resolved)

    const priority = ({ technology: { categories } }) =>
      categories.reduce((max, id) =>
        Math.max(max, Wappalyzer.getCategory(id).priority)
      )

    return resolved
      .sort((a, b) => (priority(a) > priority(b) ? 1 : -1))
      .map(
        ({
          technology: { name, slug, categories, icon, website, cpe },
          confidence,
          version
        }) => ({
          name,
          slug,
          categories: categories.map((id) => Wappalyzer.getCategory(id)),
          confidence,
          version,
          icon,
          website,
          cpe
        })
      )
  },

  /**
   * Resolve promises for version of technology.
   * @param {Promise} resolved
   */
  resolveVersion({ version, regex }, match) {
    let resolved = version

    if (version) {
      const matches = regex.exec(match)

      if (matches) {
        matches.forEach((match, index) => {
          // Parse ternary operator
          const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec(
            version
          )

          if (ternary && ternary.length === 3) {
            resolved = version.replace(
              ternary[0],
              match ? ternary[1] : ternary[2]
            )
          }

          // Replace back references
          resolved = resolved
            .trim()
            .replace(new RegExp(`\\\\${index}`, 'g'), match || '')
        })
      }
    }

    return resolved
  },

  /**
   * Resolve promises for excluded technology.
   * @param {Promise} resolved
   */
  resolveExcludes(resolved) {
    resolved.forEach(({ technology }) => {
      technology.excludes.forEach(({ name }) => {
        const excluded = Wappalyzer.getTechnology(name)

        if (!excluded) {
          throw new Error(`Excluded technology does not exist: ${name}`)
        }

        const index = resolved.findIndex(({ name }) => name === excluded.name)

        if (index !== -1) {
          resolved.splice(index, 1)
        }
      })
    })
  },

  /**
   * Resolve promises for implied technology.
   * @param {Promise} resolved
   */
  resolveImplies(resolved) {
    let done = false

    while (resolved.length && !done) {
      resolved.forEach(({ technology, confidence }) => {
        done = true

        technology.implies.forEach(({ name, confidence: _confidence }) => {
          const implied = Wappalyzer.getTechnology(name)

          if (!implied) {
            throw new Error(`Implied technology does not exist: ${name}`)
          }

          if (
            resolved.findIndex(
              ({ technology: { name } }) => name === implied.name
            ) === -1
          ) {
            resolved.push({
              technology: implied,
              confidence: Math.min(confidence, _confidence),
              version: ''
            })

            done = false
          }
        })
      })
    }
  },

  /**
   * Initialize analyzation.
   * @param {*} param0
   */
  analyze({ url, html, meta, headers, cookies, scripts }) {
    const oo = Wappalyzer.analyzeOneToOne
    const om = Wappalyzer.analyzeOneToMany
    const mm = Wappalyzer.analyzeManyToMany

    const flatten = (array) => Array.prototype.concat.apply([], array)

    try {
      const detections = flatten(
        Wappalyzer.technologies.map((technology) =>
          flatten([
            oo(technology, 'url', url),
            oo(technology, 'html', html),
            om(technology, 'scripts', scripts),
            mm(technology, 'cookies', cookies),
            mm(technology, 'meta', meta),
            mm(technology, 'headers', headers)
          ])
        )
      ).filter((technology) => technology)

      return detections
    } catch (error) {
      throw new Error(error.message || error.toString())
    }
  },

  /**
   * Extract technologies from data collected.
   * @param {*object} data
   */
  setTechnologies(data) {
    const transform = Wappalyzer.transformPatterns

    Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => {
      const {
        cats,
        url,
        html,
        meta,
        headers,
        cookies,
        script,
        js,
        implies,
        excludes,
        icon,
        website,
        cpe
      } = data[name]

      technologies.push({
        name,
        categories: cats || [],
        slug: Wappalyzer.slugify(name),
        url: transform(url),
        headers: transform(headers),
        cookies: transform(cookies),
        html: transform(html),
        meta: transform(meta),
        scripts: transform(script),
        js: transform(js, true),
        implies: transform(implies).map(({ value, confidence }) => ({
          name: value,
          confidence
        })),
        excludes: transform(excludes).map(({ value }) => ({
          name: value
        })),
        icon: icon || 'default.svg',
        website: website || '',
        cpe
      })

      return technologies
    }, [])
  },

  /**
   * Assign categories for data.
   * @param {Object} data<commit_msg>Return null if CPE is not defined<commit_after>'use strict'

const Wappalyzer = {
  technologies: [],
  categories: [],

  slugify(string) {
    return string
      .toLowerCase()
      .replace(/[^a-z0-9-]/g, '-')
      .replace(/--+/g, '-')
      .replace(/(?:^-|-$)/, '')
  },

  getTechnology(name) {
    return Wappalyzer.technologies.find(({ name: _name }) => name === _name)
  },

  getCategory(id) {
    return Wappalyzer.categories.find(({ id: _id }) => id === _id)
  },

  /**
   * Resolve promises for implied technology.
   * @param {Array} detections
   */
  resolve(detections = []) {
    const resolved = detections.reduce((resolved, { technology }) => {
      if (
        resolved.findIndex(
          ({ technology: { name } }) => name === technology.name
        ) === -1
      ) {
        let version = ''
        let confidence = 0

        detections.forEach(
          ({ technology: { name }, pattern, version: _version = '' }) => {
            if (name === technology.name) {
              confidence = Math.min(100, confidence + pattern.confidence)
              version =
                _version.length > version.length && _version.length <= 10
                  ? _version
                  : version
            }
          }
        )

        resolved.push({ technology, confidence, version })
      }

      return resolved
    }, [])

    Wappalyzer.resolveExcludes(resolved)
    Wappalyzer.resolveImplies(resolved)

    const priority = ({ technology: { categories } }) =>
      categories.reduce((max, id) =>
        Math.max(max, Wappalyzer.getCategory(id).priority)
      )

    return resolved
      .sort((a, b) => (priority(a) > priority(b) ? 1 : -1))
      .map(
        ({
          technology: { name, slug, categories, icon, website, cpe },
          confidence,
          version
        }) => ({
          name,
          slug,
          categories: categories.map((id) => Wappalyzer.getCategory(id)),
          confidence,
          version,
          icon,
          website,
          cpe
        })
      )
  },

  /**
   * Resolve promises for version of technology.
   * @param {Promise} resolved
   */
  resolveVersion({ version, regex }, match) {
    let resolved = version

    if (version) {
      const matches = regex.exec(match)

      if (matches) {
        matches.forEach((match, index) => {
          // Parse ternary operator
          const ternary = new RegExp(`\\\\${index}\\?([^:]+):(.*)$`).exec(
            version
          )

          if (ternary && ternary.length === 3) {
            resolved = version.replace(
              ternary[0],
              match ? ternary[1] : ternary[2]
            )
          }

          // Replace back references
          resolved = resolved
            .trim()
            .replace(new RegExp(`\\\\${index}`, 'g'), match || '')
        })
      }
    }

    return resolved
  },

  /**
   * Resolve promises for excluded technology.
   * @param {Promise} resolved
   */
  resolveExcludes(resolved) {
    resolved.forEach(({ technology }) => {
      technology.excludes.forEach(({ name }) => {
        const excluded = Wappalyzer.getTechnology(name)

        if (!excluded) {
          throw new Error(`Excluded technology does not exist: ${name}`)
        }

        const index = resolved.findIndex(({ name }) => name === excluded.name)

        if (index !== -1) {
          resolved.splice(index, 1)
        }
      })
    })
  },

  /**
   * Resolve promises for implied technology.
   * @param {Promise} resolved
   */
  resolveImplies(resolved) {
    let done = false

    while (resolved.length && !done) {
      resolved.forEach(({ technology, confidence }) => {
        done = true

        technology.implies.forEach(({ name, confidence: _confidence }) => {
          const implied = Wappalyzer.getTechnology(name)

          if (!implied) {
            throw new Error(`Implied technology does not exist: ${name}`)
          }

          if (
            resolved.findIndex(
              ({ technology: { name } }) => name === implied.name
            ) === -1
          ) {
            resolved.push({
              technology: implied,
              confidence: Math.min(confidence, _confidence),
              version: ''
            })

            done = false
          }
        })
      })
    }
  },

  /**
   * Initialize analyzation.
   * @param {*} param0
   */
  analyze({ url, html, meta, headers, cookies, scripts }) {
    const oo = Wappalyzer.analyzeOneToOne
    const om = Wappalyzer.analyzeOneToMany
    const mm = Wappalyzer.analyzeManyToMany

    const flatten = (array) => Array.prototype.concat.apply([], array)

    try {
      const detections = flatten(
        Wappalyzer.technologies.map((technology) =>
          flatten([
            oo(technology, 'url', url),
            oo(technology, 'html', html),
            om(technology, 'scripts', scripts),
            mm(technology, 'cookies', cookies),
            mm(technology, 'meta', meta),
            mm(technology, 'headers', headers)
          ])
        )
      ).filter((technology) => technology)

      return detections
    } catch (error) {
      throw new Error(error.message || error.toString())
    }
  },

  /**
   * Extract technologies from data collected.
   * @param {*object} data
   */
  setTechnologies(data) {
    const transform = Wappalyzer.transformPatterns

    Wappalyzer.technologies = Object.keys(data).reduce((technologies, name) => {
      const {
        cats,
        url,
        html,
        meta,
        headers,
        cookies,
        script,
        js,
        implies,
        excludes,
        icon,
        website,
        cpe
      } = data[name]

      technologies.push({
        name,
        categories: cats || [],
        slug: Wappalyzer.slugify(name),
        url: transform(url),
        headers: transform(headers),
        cookies: transform(cookies),
        html: transform(html),
        meta: transform(meta),
        scripts: transform(script),
        js: transform(js, true),
        implies: transform(implies).map(({ value, confidence }) => ({
          name: value,
          confidence
        })),
        excludes: transform(excludes).map(({ value }) => ({
          name: value
        })),
        icon: icon || 'default.svg',
        website: website || null,
        cpe: cpe || null
      })

      return technologies
    }, [])
  },

  /**
   * Assign categories for data.
   * @param {Object} data
{'fraction_non_alphanumeric': 0.1409790426107803, 'fraction_numerical': 0.0029386745031320083, 'mean_word_length': 1.7317279256442755, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3902165', 'n_tokens_mistral': 3752, 'n_tokens_neox': 3599, 'n_words': 804}