<commit_before>// @flow

import path from 'path'
import globrex from 'globrex'
import resolveFrom from 'resolve-from'
import Imurmurhash from 'imurmurhash'

import type Context from './context'
import type { Loc, Chunk, ImportResolved } from './types'

export const NEWLINE_REGEXP = /\r\n|[\n\r\u2028\u2029]/

export function getStringHash(str: string): string {
  return new Imurmurhash()
    .hash(str)
    .result()
    .toString()
}

export function getUniqueHash(item: ImportResolved | Chunk): string {
  let stringKey = ''
  if (typeof item.filePath === 'string') {
    stringKey = item.filePath
  } else if (typeof item.label === 'string') {
    stringKey = item.label
  } else if (typeof item.entry === 'string') {
    stringKey = item.entry
  } else if (Array.isArray(item.imports)) {
    stringKey = `${JSON.stringify(item.imports)}`
  }

  return `${item.format}_${getStringHash(stringKey)}`
}

export function getChunkKey(chunk: Chunk): string {
  return `chunk_${getUniqueHash(chunk)}`
}

export function getFileKey(item: ImportResolved | Chunk): string {
  return `file_${getUniqueHash(item)}`
}

export function getChunk(
  format: string,
  label: ?string = null,
  entry: ?string = null,
  imports: Array<ImportResolved> = [],<commit_msg>:bug: Include chunk.root as part of chunk key<commit_after>// @flow

import path from 'path'
import globrex from 'globrex'
import resolveFrom from 'resolve-from'
import Imurmurhash from 'imurmurhash'

import type Context from './context'
import type { Loc, Chunk, ImportResolved } from './types'

export const NEWLINE_REGEXP = /\r\n|[\n\r\u2028\u2029]/

export function getStringHash(str: string): string {
  return new Imurmurhash()
    .hash(str)
    .result()
    .toString()
}

export function getUniqueHash(item: ImportResolved | Chunk): string {
  let stringKey = ''
  if (typeof item.filePath === 'string') {
    stringKey = item.filePath
  } else if (typeof item.label === 'string') {
    stringKey = item.label
  } else if (typeof item.entry === 'string') {
    stringKey = item.entry
  } else if (Array.isArray(item.imports)) {
    stringKey = `${JSON.stringify(item.imports)}`
  }

  return `${item.format}_${getStringHash(stringKey)}`
}

export function getChunkKey(chunk: Chunk): string {
  return `chunk_${chunk.root ? '1' : '0'}$${getUniqueHash(chunk)}`
}

export function getFileKey(item: ImportResolved | Chunk): string {
  return `file_${getUniqueHash(item)}`
}

export function getChunk(
  format: string,
  label: ?string = null,
  entry: ?string = null,
  imports: Array<ImportResolved> = [],
{'fraction_non_alphanumeric': 0.1425233644859813, 'fraction_numerical': 0.007009345794392523, 'mean_word_length': 4.3078512396694215, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7568389', 'n_tokens_mistral': 849, 'n_tokens_neox': 839, 'n_words': 225}