<commit_before>--[[lit-meta
  name = "creationix/sha1"
  version = "1.0.0"
  homepage = "https://github.com/luvit/lit/blob/master/deps/sha1.lua"
  description = "Pure Lua implementation of SHA1 using bitop"
  authors = {
    "<NAME>"
  }
]]

-- http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA_All.pdf

local bit = require('bit')
local band = bit.band
local bor = bit.bor
local bxor = bit.bxor
local lshift = bit.lshift
local rshift = bit.rshift
local rol = bit.rol
local tobit = bit.tobit
local tohex = bit.tohex

local byte = string.byte
local concat = table.concat
local floor = table.floor

local ffi = require('ffi')
local function newBlock()
-- TODO: add table fallback for when ffi isn't available
  return ffi.new("uint32_t[80]")
end

local shared = newBlock()

local function unsigned(n)
  return n < 0 and (n + 0x100000000) or n
end

local function create(sync)
  local h0 = 0x67452301
  local h1 = 0xEFCDAB89
  local h2 = 0x98BADCFE
  local h3 = 0x10325476
  local h4 = 0xC3D2E1F0
  -- The first 64 bytes (16 words) is the data chunk<commit_msg>sha1: fallback for when ffi is not available<commit_after>--[[lit-meta
  name = "creationix/sha1"
  version = "1.0.1"
  homepage = "https://github.com/luvit/lit/blob/master/deps/sha1.lua"
  description = "Pure Lua implementation of SHA1 using bitop"
  authors = {
    "<NAME>"
  }
]]

-- http://csrc.nist.gov/groups/ST/toolkit/documents/Examples/SHA_All.pdf

local bit = require('bit')
local band = bit.band
local bor = bit.bor
local bxor = bit.bxor
local lshift = bit.lshift
local rshift = bit.rshift
local rol = bit.rol
local tobit = bit.tobit
local tohex = bit.tohex

local byte = string.byte
local concat = table.concat
local floor = table.floor

local ffi = pcall(require, "ffi")
local newBlock = ffi and function ()
  return ffi.new("uint32_t[80]")
end or function ()
  local t = {}
  for i = 0, 79 do
    t[i] = 0
  end
  return t
end

local shared = newBlock()

local function unsigned(n)
  return n < 0 and (n + 0x100000000) or n
end

local function create(sync)
  local h0 = 0x67452301
  local h1 = 0xEFCDAB89
  local h2 = 0x98BADCFE
  local h3 = 0x10325476
  local h4 = 0xC3D2E1F0
  -- The first 64 bytes (16 words) is the data chunk
{'fraction_non_alphanumeric': 0.11432444645277903, 'fraction_numerical': 0.05558065973791233, 'mean_word_length': 3.4908722109533468, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2168529', 'n_tokens_mistral': 905, 'n_tokens_neox': 831, 'n_words': 245}