<commit_before>shjs = require 'shelljs'
TokenizerFactory = require './tokenizer/TokenizerFactory'
crypto = require 'crypto'

Storage = require './storage/StorageMemory'

Clone = require('./clone').Clone

class Strategy

  constructor: (options) ->
    @options = options
    @languages = options.languages
    @storage = new Storage()

  detect: (map, file, @minLines, @minTokens) ->
    tokenizer = TokenizerFactory::makeTokenizer file, @languages
    unless tokenizer
      return no
    language = tokenizer.getType()

    if shjs.test '-f', file
      code = shjs.cat file
    else
      return no

    lines = code.split '\n'
    map.numberOfLines =  map.numberOfLines + lines.length

    tokenizer.skipComments = @options['skip-comments']

    {tokensPositions, currentMap} = tokenizer.tokenize(code).generateMap()

    firstLine = 0
    tokenNumber = 0
    isClone = false

    while tokenNumber <= tokensPositions.length - @minTokens
      mapFrame = currentMap.substring tokenNumber * 33, tokenNumber * 33 + @minTokens * 33
      hash = crypto.createHash('md5').update(mapFrame).digest('hex').substring 0, 8
      if @storage.hasHash hash, language
        isClone = true
        if firstLine is 0
          firstLine = tokensPositions[tokenNumber]
          firstHash = hash
          firstToken = tokenNumber
      else
        if isClone
          lastToken = tokenNumber + @minTokens - 2
          @addClone(
            map,
            file,
            firstHash,<commit_msg>fixes #49 hash collisions with large project - birthday paradox<commit_after>shjs = require 'shelljs'
TokenizerFactory = require './tokenizer/TokenizerFactory'
crypto = require 'crypto'

Storage = require './storage/StorageMemory'

Clone = require('./clone').Clone

class Strategy

  constructor: (options) ->
    @options = options
    @languages = options.languages
    @storage = new Storage()

  detect: (map, file, @minLines, @minTokens) ->
    tokenizer = TokenizerFactory::makeTokenizer file, @languages
    unless tokenizer
      return no
    language = tokenizer.getType()

    if shjs.test '-f', file
      code = shjs.cat file
    else
      return no

    lines = code.split '\n'
    map.numberOfLines =  map.numberOfLines + lines.length

    tokenizer.skipComments = @options['skip-comments']

    {tokensPositions, currentMap} = tokenizer.tokenize(code).generateMap()

    firstLine = 0
    tokenNumber = 0
    isClone = false

    while tokenNumber <= tokensPositions.length - @minTokens
      mapFrame = currentMap.substring tokenNumber * 33, tokenNumber * 33 + @minTokens * 33
      hash = crypto.createHash('md5').update(mapFrame).digest('hex')

      if @storage.hasHash hash, language
        isClone = true
        if firstLine is 0
          firstLine = tokensPositions[tokenNumber]
          firstHash = hash
          firstToken = tokenNumber
      else
        if isClone
          lastToken = tokenNumber + @minTokens - 2
          @addClone(
            map,
            file,
            firstHash,
{'fraction_non_alphanumeric': 0.09741550695825049, 'fraction_numerical': 0.008614976805831677, 'mean_word_length': 2.6329723225030084, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182656', 'n_tokens_mistral': 874, 'n_tokens_neox': 847, 'n_words': 225}