<commit_before>import sha256 from 'crypto-js/sha256'
import utmParams from '@segmentstream/utils/utmParams'
import getQueryParam from '@segmentstream/utils/getQueryParam'
import htmlGlobals from '@segmentstream/utils/htmlGlobals'
import cleanObject from '@segmentstream/utils/cleanObject'
import arrayMerge from '@segmentstream/utils/arrayMerge'
import size from '@segmentstream/utils/size'
import isCrawler from '@segmentstream/utils/isCrawler'
import each from '@segmentstream/utils/each'
import { warn } from '@segmentstream/utils/safeConsole'
import utmParamsFromReferrer from '@segmentstream/utils/utmParamsFromReferrer'
import uuid from 'uuid/v1'
import UAParser from 'ua-parser-js'
import Integration from '../Integration'
import StreamingFilters, {
  pageProps,
  listingProps,
  cartProps,
  productProps,
  transactionProps,
  campaignProps,
  listItemProps,
  websiteProps
} from './DDManagerStreaming/Filters'
import {
  DIGITALDATA_VAR,
  PRODUCT_VAR
} from '../variableTypes'
import {
  VIEWED_PAGE,
  VIEWED_CART,
  COMPLETED_TRANSACTION,
  VIEWED_PRODUCT,
  CLICKED_PRODUCT,
  VIEWED_PRODUCT_DETAIL,
  ADDED_PRODUCT,
  REMOVED_PRODUCT,
  VIEWED_PRODUCT_LISTING,
  SEARCHED_PRODUCTS,
  VIEWED_CAMPAIGN,
  CLICKED_CAMPAIGN,
  VIEWED_EXPERIMENT,
  INTEGRATION_VALIDATION_FAILED,
  EXCEPTION
} from '../events/semanticEvents'

class DDManagerStreaming extends Integration {
  constructor (digitalData, options) {
    const optionsWithDefaults = Object.assign({
      projectId: '',
      projectName: '',
      buildNumber: 1,
      trackingUrl: 'https://track.ddmanager.ru/collect',
      dimensions: {},
      metrics: {},
      internal: false,
      streamingEndpoint: ''
    }, options)
    super(digitalData, optionsWithDefaults)
    this.website = {}

    this.dimensions = {}
    this.metrics = {}
    this.productDimensions = {}
    this.productMetrics = {}
    this.customEnrichableProps = []

    this.viewedPageCounter = 0

    // TODO: refactoring
    each(optionsWithDefaults.dimensions, (key, variable) => {
      if (variable.type === PRODUCT_VAR) {
        this.productDimensions[key] = variable.value
      } else {
        if (variable.type === DIGITALDATA_VAR) {
          this.customEnrichableProps.push(variable.value)
        }
        this.dimensions[key] = variable.value
      }
    })
    each(optionsWithDefaults.metrics, (key, variable) => {
      if (variable.type === PRODUCT_VAR) {
        this.productMetrics[key] = variable.value
      } else {
        if (variable.type === DIGITALDATA_VAR) {
          this.customEnrichableProps.push(variable.value)
        }
        this.metrics[key] = variable.value
      }
    })
    this.filters = new StreamingFilters(
      this.dimensions,
      this.metrics,
      this.productDimensions,
      this.productMetrics
    )

    this.setupStreamingEndpoints()
  }

  setupStreamingEndpoints () {
    const streamingEndpoint = this.getOption('streamingEndpoint') || ''

    this.streamingEndpoints = [this.getOption('trackingUrl')]
    if (streamingEndpoint.match(/^(https?):\/\/[^ "]+$/)) this.streamingEndpoints.push(streamingEndpoint)
  }

  initialize () {
    this._isLoaded = true
    if (isCrawler(htmlGlobals.getNavigator().userAgent)) {
      return false
    }
    return true
  }

  getIgnoredEvents () {
    if (this.getOption('internal')) {
      return [VIEWED_PRODUCT, CLICKED_PRODUCT, INTEGRATION_VALIDATION_FAILED, EXCEPTION]<commit_msg>add ability for setup multiple streaming endpoints<commit_after>import sha256 from 'crypto-js/sha256'
import utmParams from '@segmentstream/utils/utmParams'
import getQueryParam from '@segmentstream/utils/getQueryParam'
import htmlGlobals from '@segmentstream/utils/htmlGlobals'
import cleanObject from '@segmentstream/utils/cleanObject'
import arrayMerge from '@segmentstream/utils/arrayMerge'
import size from '@segmentstream/utils/size'
import isCrawler from '@segmentstream/utils/isCrawler'
import each from '@segmentstream/utils/each'
import { warn } from '@segmentstream/utils/safeConsole'
import utmParamsFromReferrer from '@segmentstream/utils/utmParamsFromReferrer'
import uuid from 'uuid/v1'
import UAParser from 'ua-parser-js'
import Integration from '../Integration'
import StreamingFilters, {
  pageProps,
  listingProps,
  cartProps,
  productProps,
  transactionProps,
  campaignProps,
  listItemProps,
  websiteProps
} from './DDManagerStreaming/Filters'
import {
  DIGITALDATA_VAR,
  PRODUCT_VAR
} from '../variableTypes'
import {
  VIEWED_PAGE,
  VIEWED_CART,
  COMPLETED_TRANSACTION,
  VIEWED_PRODUCT,
  CLICKED_PRODUCT,
  VIEWED_PRODUCT_DETAIL,
  ADDED_PRODUCT,
  REMOVED_PRODUCT,
  VIEWED_PRODUCT_LISTING,
  SEARCHED_PRODUCTS,
  VIEWED_CAMPAIGN,
  CLICKED_CAMPAIGN,
  VIEWED_EXPERIMENT,
  INTEGRATION_VALIDATION_FAILED,
  EXCEPTION
} from '../events/semanticEvents'

class DDManagerStreaming extends Integration {
  constructor (digitalData, options) {
    const optionsWithDefaults = Object.assign({
      projectId: '',
      projectName: '',
      buildNumber: 1,
      trackingUrl: 'https://track.ddmanager.ru/collect',
      dimensions: {},
      metrics: {},
      internal: false,
      streamingEndpoint: '',
      streamingEndpoints: []
    }, options)
    super(digitalData, optionsWithDefaults)
    this.website = {}

    this.dimensions = {}
    this.metrics = {}
    this.productDimensions = {}
    this.productMetrics = {}
    this.customEnrichableProps = []

    this.viewedPageCounter = 0

    // TODO: refactoring
    each(optionsWithDefaults.dimensions, (key, variable) => {
      if (variable.type === PRODUCT_VAR) {
        this.productDimensions[key] = variable.value
      } else {
        if (variable.type === DIGITALDATA_VAR) {
          this.customEnrichableProps.push(variable.value)
        }
        this.dimensions[key] = variable.value
      }
    })
    each(optionsWithDefaults.metrics, (key, variable) => {
      if (variable.type === PRODUCT_VAR) {
        this.productMetrics[key] = variable.value
      } else {
        if (variable.type === DIGITALDATA_VAR) {
          this.customEnrichableProps.push(variable.value)
        }
        this.metrics[key] = variable.value
      }
    })
    this.filters = new StreamingFilters(
      this.dimensions,
      this.metrics,
      this.productDimensions,
      this.productMetrics
    )

    this.setupStreamingEndpoints()
  }

  setupStreamingEndpoints () {
    const streamingEndpoints = this.getOption('streamingEndpoints') || []

    // TODO: remove (backward compatibility)
    const streamingEndpoint = this.getOption('streamingEndpoint')
    if (streamingEndpoint) streamingEndpoints.push(streamingEndpoint)

    const trackingUrl = this.getOption('trackingUrl')
    if (trackingUrl) streamingEndpoints.unshift(trackingUrl) // the main tracking url should be the first

    this.streamingEndpoints = streamingEndpoints.filter(endpoint => endpoint.match(/^(https?):\/\/[^ "]+$/))
  }

  initialize () {
    this._isLoaded = true
    if (isCrawler(htmlGlobals.getNavigator().userAgent)) {
      return false
    }
    return true
  }

  getIgnoredEvents () {
    if (this.getOption('internal')) {
      return [VIEWED_PRODUCT, CLICKED_PRODUCT, INTEGRATION_VALIDATION_FAILED, EXCEPTION]
{'fraction_non_alphanumeric': 0.10767104353835522, 'fraction_numerical': 0.00248790601243953, 'mean_word_length': 3.6744186046511627, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1995769', 'n_tokens_mistral': 2289, 'n_tokens_neox': 2212, 'n_words': 406}