<commit_before>-- Copyright 2007-2012 Mitchell mitchell.<EMAIL>. See LICENSE.

local M = {}

--[[ This comment is for LuaDoc.
---
-- Handles file type detection for Textadept.
-- @field _G.events.LANGUAGE_MODULE_LOADED (string)
--   Called after loading a language-specific module.
--   This is useful for overriding a language-specific module's key bindings
--   or other properties since the module is not loaded when Textadept starts.
--   Arguments:
--
--   * *`lang`*: The language lexer name.
module('_M.textadept.mime_types')]]

-- Events.
local events, events_connect = events, events.connect
events.LANGUAGE_MODULE_LOADED = 'language_module_loaded'

---
-- Map of file extensions (excluding the leading '.') to their associated
-- lexers.
-- If the file type is not recognized by shebang words or first-line patterns,
-- each file extension is matched against the file's extension.
-- @class table
-- @name extensions
M.extensions = {}

---
-- Map of shebang words to their associated lexers.
-- If the file has a shebang line, a line that starts with "#!" and is the first
-- line in the file, each shebang word is matched against that line.
-- @class table
-- @name shebangs
M.shebangs = {}

---
-- Map of first-line patterns to their associated lexers.
-- If a file type is not recognized by shebang words, each pattern is matched
-- against the first line in the file.
-- @class table
-- @name patterns
M.patterns = {}

-- Load mime-types from *modules/textadept/mime_types.conf*.
local mime_types
local f = io.open(_HOME..'/modules/textadept/mime_types.conf', 'rb')
if f then
  mime_types = f:read('*all')
  f:close()
end
f = io.open(_USERHOME..'/mime_types.conf', 'rb')
if f then
  mime_types = mime_types..'\n'..f:read('*all')
  f:close()
end
for line in mime_types:gmatch('[^\r\n]+') do
  if not line:find('^%s*%%') then
    if line:find('^%s*[^#/]') then -- extension definition
      local ext, lexer_name = line:match('^%s*(.+)%s+(%S+)$')
      if ext and lexer_name then M.extensions[ext] = lexer_name end
    else -- shebang or pattern
      local ch, text, lexer_name = line:match('^%s*([#/])(.+)%s+(%S+)$')
      if ch and text and lexer_name then
        (ch == '#' and M.shebangs or M.patterns)[text] = lexer_name
      end
    end
  end
end

---
-- List of detected lexers are read from *lexers/* and *~/.textadept/lexers/*.
-- @class table
-- @name lexers
M.lexers = {}

-- Generate lexer list
local lexers_found = {}
for lexer in lfs.dir(_HOME..'/lexers') do
  if lexer:find('%.lua$') and lexer ~= 'lexer.lua' then
    lexers_found[lexer:match('^(.+)%.lua$')] = true
  end
end
if lfs.attributes(_USERHOME..'/lexers') then
  for lexer in lfs.dir(_USERHOME..'/lexers') do
    if lexer:find('%.lua$') and lexer ~= 'lexer.lua' then
      lexers_found[lexer:match('^(.+)%.lua$')] = true
    end
  end
end
for lexer in pairs(lexers_found) do M.lexers[#M.lexers + 1] = lexer end
table.sort(M.lexers)

---
-- Prompts the user to select a lexer for the current buffer.
-- @see buffer.set_lexer
-- @name select_lexer
function M.select_lexer()
  local lexer = gui.filteredlist(_L['Select Lexer'], _L['Name'], M.lexers)
  if lexer then buffer:set_lexer(lexer) end
end

-- LuaDoc is in core/.buffer.luadoc.
local function get_style_name(buffer, style_num)
  buffer:check_global()
  if style_num < 0 or style_num > 255 then error('0 <= style_num < 256') end
  return buffer:private_lexer_call(style_num)
end

-- Contains the whitespace styles for lexers.
-- These whitespace styles are used to determine the lexer at the caret position
-- since the styles have the name "[lang]_whitespace".
-- @class table
-- @name ws_styles
local ws_styles = {}
local SETDIRECTPOINTER = _SCINTILLA.properties.doc_pointer[2]
local SETLEXERLANGUAGE = _SCINTILLA.properties.lexer_language[2]
-- LuaDoc is in core/.buffer.luadoc.
local function set_lexer(buffer, lang)
  buffer:check_global()
  buffer._lexer = lang
  buffer:private_lexer_call(SETDIRECTPOINTER, buffer.direct_pointer)
  buffer:private_lexer_call(SETLEXERLANGUAGE, lang)
  if package.searchpath(lang, package.path) then
    _M[lang] = require(lang)
    local post_init = lang..'.post_init'
    if package.searchpath(post_init, package.path) then require(post_init) end
    if _M[lang].set_buffer_properties then _M[lang].set_buffer_properties() end
    events.emit(events.LANGUAGE_MODULE_LOADED, lang)
  end
  buffer:colourise(0, -1)
  -- Create the ws_styles[lexer] lookup table for `get_lexer()`.
  if ws_styles[lang] then return end
  local ws = {}
  for i = 0, 255 do
    ws[i] = buffer:private_lexer_call(i):find('whitespace') ~= nil
  end
  ws_styles[lang] = ws
end

local GETLEXERLANGUAGE = _SCINTILLA.properties.lexer_language[1]
-- LuaDoc is in core/.buffer.luadoc.
local function get_lexer(buffer, current)
  buffer:check_global()
  local lexer = buffer:private_lexer_call(GETLEXERLANGUAGE)
  if not current then return lexer end
  local i, ws, style_at = buffer.current_pos, ws_styles[lexer], buffer.style_at
  if ws then while i > 0 and not ws[style_at[i]] do i = i - 1 end end
  return get_style_name(buffer, style_at[i]):match('^(.+)_whitespace$') or lexer
end

events_connect(events.BUFFER_NEW, function()
  buffer.set_lexer, buffer.get_lexer = set_lexer, get_lexer
  buffer.get_style_name = get_style_name<commit_msg>Greatly improve speed when loading large files; modules/textadept/mime_types.lua<commit_after>-- Copyright 2007-2012 Mitchell mitchell.att.<EMAIL>ica.<EMAIL>. See LICENSE.

local M = {}

--[[ This comment is for LuaDoc.
---
-- Handles file type detection for Textadept.
-- @field _G.events.LANGUAGE_MODULE_LOADED (string)
--   Called after loading a language-specific module.
--   This is useful for overriding a language-specific module's key bindings
--   or other properties since the module is not loaded when Textadept starts.
--   Arguments:
--
--   * *`lang`*: The language lexer name.
module('_M.textadept.mime_types')]]

-- Events.
local events, events_connect = events, events.connect
events.LANGUAGE_MODULE_LOADED = 'language_module_loaded'

---
-- Map of file extensions (excluding the leading '.') to their associated
-- lexers.
-- If the file type is not recognized by shebang words or first-line patterns,
-- each file extension is matched against the file's extension.
-- @class table
-- @name extensions
M.extensions = {}

---
-- Map of shebang words to their associated lexers.
-- If the file has a shebang line, a line that starts with "#!" and is the first
-- line in the file, each shebang word is matched against that line.
-- @class table
-- @name shebangs
M.shebangs = {}

---
-- Map of first-line patterns to their associated lexers.
-- If a file type is not recognized by shebang words, each pattern is matched
-- against the first line in the file.
-- @class table
-- @name patterns
M.patterns = {}

-- Load mime-types from *modules/textadept/mime_types.conf*.
local mime_types
local f = io.open(_HOME..'/modules/textadept/mime_types.conf', 'rb')
if f then
  mime_types = f:read('*all')
  f:close()
end
f = io.open(_USERHOME..'/mime_types.conf', 'rb')
if f then
  mime_types = mime_types..'\n'..f:read('*all')
  f:close()
end
for line in mime_types:gmatch('[^\r\n]+') do
  if not line:find('^%s*%%') then
    if line:find('^%s*[^#/]') then -- extension definition
      local ext, lexer_name = line:match('^%s*(.+)%s+(%S+)$')
      if ext and lexer_name then M.extensions[ext] = lexer_name end
    else -- shebang or pattern
      local ch, text, lexer_name = line:match('^%s*([#/])(.+)%s+(%S+)$')
      if ch and text and lexer_name then
        (ch == '#' and M.shebangs or M.patterns)[text] = lexer_name
      end
    end
  end
end

---
-- List of detected lexers are read from *lexers/* and *~/.textadept/lexers/*.
-- @class table
-- @name lexers
M.lexers = {}

-- Generate lexer list
local lexers_found = {}
for lexer in lfs.dir(_HOME..'/lexers') do
  if lexer:find('%.lua$') and lexer ~= 'lexer.lua' then
    lexers_found[lexer:match('^(.+)%.lua$')] = true
  end
end
if lfs.attributes(_USERHOME..'/lexers') then
  for lexer in lfs.dir(_USERHOME..'/lexers') do
    if lexer:find('%.lua$') and lexer ~= 'lexer.lua' then
      lexers_found[lexer:match('^(.+)%.lua$')] = true
    end
  end
end
for lexer in pairs(lexers_found) do M.lexers[#M.lexers + 1] = lexer end
table.sort(M.lexers)

---
-- Prompts the user to select a lexer for the current buffer.
-- @see buffer.set_lexer
-- @name select_lexer
function M.select_lexer()
  local lexer = gui.filteredlist(_L['Select Lexer'], _L['Name'], M.lexers)
  if lexer then buffer:set_lexer(lexer) end
end

-- LuaDoc is in core/.buffer.luadoc.
local function get_style_name(buffer, style_num)
  buffer:check_global()
  if style_num < 0 or style_num > 255 then error('0 <= style_num < 256') end
  return buffer:private_lexer_call(style_num)
end

-- Contains the whitespace styles for lexers.
-- These whitespace styles are used to determine the lexer at the caret position
-- since the styles have the name "[lang]_whitespace".
-- @class table
-- @name ws_styles
local ws_styles = {}
local SETDIRECTPOINTER = _SCINTILLA.properties.doc_pointer[2]
local SETLEXERLANGUAGE = _SCINTILLA.properties.lexer_language[2]
-- LuaDoc is in core/.buffer.luadoc.
local function set_lexer(buffer, lang)
  buffer:check_global()
  buffer._lexer = lang
  buffer:private_lexer_call(SETDIRECTPOINTER, buffer.direct_pointer)
  buffer:private_lexer_call(SETLEXERLANGUAGE, lang)
  if package.searchpath(lang, package.path) then
    _M[lang] = require(lang)
    local post_init = lang..'.post_init'
    if package.searchpath(post_init, package.path) then require(post_init) end
    if _M[lang].set_buffer_properties then _M[lang].set_buffer_properties() end
    events.emit(events.LANGUAGE_MODULE_LOADED, lang)
  end
  local last_line = buffer.first_visible_line + buffer.lines_on_screen
  buffer:colourise(0, buffer:position_from_line(last_line + 1))
  -- Create the ws_styles[lexer] lookup table for `get_lexer()`.
  if ws_styles[lang] then return end
  local ws = {}
  for i = 0, 255 do
    ws[i] = buffer:private_lexer_call(i):find('whitespace') ~= nil
  end
  ws_styles[lang] = ws
end

local GETLEXERLANGUAGE = _SCINTILLA.properties.lexer_language[1]
-- LuaDoc is in core/.buffer.luadoc.
local function get_lexer(buffer, current)
  buffer:check_global()
  local lexer = buffer:private_lexer_call(GETLEXERLANGUAGE)
  if not current then return lexer end
  local i, ws, style_at = buffer.current_pos, ws_styles[lexer], buffer.style_at
  if ws then while i > 0 and not ws[style_at[i]] do i = i - 1 end end
  return get_style_name(buffer, style_at[i]):match('^(.+)_whitespace$') or lexer
end

events_connect(events.BUFFER_NEW, function()
  buffer.set_lexer, buffer.get_lexer = set_lexer, get_lexer
  buffer.get_style_name = get_style_name
{'fraction_non_alphanumeric': 0.1312494202764122, 'fraction_numerical': 0.005194323346628327, 'mean_word_length': 3.9232876712328766, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985997', 'n_tokens_mistral': 3902, 'n_tokens_neox': 3693, 'n_words': 1083}