<commit_before>(*
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.

Copyright (c) <NAME>
*)
unit proc_str;

{$mode objfpc}{$H+}

interface

uses
  SysUtils, Classes,
  StrUtils,
  LazFileUtils,
  ATStringProc,
  ATStringProc_Separator,
  ATSynEdit_RegExpr,
  ec_RegExpr;

function STextWholeWordSelection(const S: UnicodeString; OffsetBegin, OffsetEnd: integer;
  const ANonWordChars: UnicodeString): boolean;
function SFindFuzzyPositions(SText, SFind: UnicodeString): TATIntArray;
procedure SDeleteDuplicateSpaces(var S: string);
function SDeleteCurlyBrackets(const S: string): string;
function STextListsAllWords(SText, SWords: string): boolean;
function STextListsFuzzyInput(const AText, AFind: string; out ASimpleMatch: boolean): boolean;
function SRegexReplaceSubstring(const AStr, AStrFind, AStrReplace: string; AUseSubstitute: boolean): string;
function SRegexMatchesString(const ASubject, ARegex: string; ACaseSensitive: boolean): boolean;

function IsLexerListed(const AItem, AItemList: string): boolean;
function IsFilenameListedInExtensionList(const AFilename, AExtList: string): boolean;

type
  TRegexParts = array[0..8] of
    record
      Pos, Len: integer;
      Str: string;
    end;
function SRegexFindParts(const ARegex, AStr: string; out AParts: TRegexParts): boolean;
function SStringToPythonString(const Str: string; AndQuote: boolean=true): string;

function SMaskFilenameSlashes(const fn: string): string;
procedure SParseFilenameWithTwoNumbers(var fn: string; out NLine, NColumn: integer);
function IsPythonExpression(const S: string): boolean;


implementation

function SStringToPythonString(const Str: string; AndQuote: boolean=true): string;
var
  i: integer;
begin
  Result:= Str;
  UniqueString(Result);
  for i:= Length(Result) downto 1 do
    case Result[i] of
      '\', '"':
        Insert('\', Result, i);
      #10:
        begin
          Result[i]:= 'n';
          Insert('\', Result, i);
        end;
      #13:
        begin
          Result[i]:= 'r';
          Insert('\', Result, i);
        end;
    end;
  if AndQuote then
    Result:= '"'+Result+'"';
end;

function SMaskFilenameSlashes(const fn: string): string;
begin
  result:= fn;
  result:= StringReplace(result, '/', '|', [rfReplaceAll]);
  result:= StringReplace(result, '\', '|', [rfReplaceAll]);
end;

function SFindFuzzyPositions(SText, SFind: UnicodeString): TATIntArray;
var
  i, N: integer;
begin
  Result:= nil;

  SText:= UnicodeLowerCase(SText);
  SFind:= UnicodeLowerCase(SFind);

  //if simple match is found, don't calculate complex fuzzy matches
  N:= Pos(SFind, SText);
  if N>0 then
  begin
    SetLength(Result, Length(SFind));
    Result[0]:= N;
    for i:= 1 to High(Result) do
      Result[i]:= Result[i-1]+1;
    exit;
  end;

  //calculate complex matches
  N:= 0;
  for i:= 1 to Length(SFind) do
  begin
    N:= PosEx(SFind[i], SText, N+1);
    if N=0 then Exit(nil);
    SetLength(Result, Length(Result)+1);
    Result[High(Result)]:= N;
  end;
end;


function SRegexFindParts(const ARegex, AStr: string; out AParts: TRegexParts): boolean;
var
  Obj: TRegExpr;
  i: integer;
begin
  Result:= false;
  for i:= Low(AParts) to High(AParts) do
  begin
    AParts[i].Pos:= -1;
    AParts[i].Len:= 0;
    AParts[i].Str:= '';
  end;

  if ARegex='' then exit;
  if AStr='' then exit;

  Obj:= TRegExpr.Create;
  try<commit_msg>optim fuzzy search<commit_after>(*
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.

Copyright (c) <NAME>
*)
unit proc_str;

{$mode objfpc}{$H+}

interface

uses
  SysUtils, Classes,
  StrUtils,
  LazFileUtils,
  ATStringProc,
  ATStringProc_Separator,
  ATSynEdit_RegExpr,
  ec_RegExpr;

function STextWholeWordSelection(const S: UnicodeString; OffsetBegin, OffsetEnd: integer;
  const ANonWordChars: UnicodeString): boolean;
function SFindFuzzyPositions(SText, SFind: UnicodeString): TATIntArray;
procedure SDeleteDuplicateSpaces(var S: string);
function SDeleteCurlyBrackets(const S: string): string;
function STextListsAllWords(SText, SWords: string): boolean;
function STextListsFuzzyInput(const AText, AFind: string; out ASimpleMatch: boolean): boolean;
function SRegexReplaceSubstring(const AStr, AStrFind, AStrReplace: string; AUseSubstitute: boolean): string;
function SRegexMatchesString(const ASubject, ARegex: string; ACaseSensitive: boolean): boolean;

function IsLexerListed(const AItem, AItemList: string): boolean;
function IsFilenameListedInExtensionList(const AFilename, AExtList: string): boolean;

type
  TRegexParts = array[0..8] of
    record
      Pos, Len: integer;
      Str: string;
    end;
function SRegexFindParts(const ARegex, AStr: string; out AParts: TRegexParts): boolean;
function SStringToPythonString(const Str: string; AndQuote: boolean=true): string;

function SMaskFilenameSlashes(const fn: string): string;
procedure SParseFilenameWithTwoNumbers(var fn: string; out NLine, NColumn: integer);
function IsPythonExpression(const S: string): boolean;


implementation

function SStringToPythonString(const Str: string; AndQuote: boolean=true): string;
var
  i: integer;
begin
  Result:= Str;
  UniqueString(Result);
  for i:= Length(Result) downto 1 do
    case Result[i] of
      '\', '"':
        Insert('\', Result, i);
      #10:
        begin
          Result[i]:= 'n';
          Insert('\', Result, i);
        end;
      #13:
        begin
          Result[i]:= 'r';
          Insert('\', Result, i);
        end;
    end;
  if AndQuote then
    Result:= '"'+Result+'"';
end;

function SMaskFilenameSlashes(const fn: string): string;
begin
  result:= fn;
  result:= StringReplace(result, '/', '|', [rfReplaceAll]);
  result:= StringReplace(result, '\', '|', [rfReplaceAll]);
end;

function SFindFuzzyPositions(SText, SFind: UnicodeString): TATIntArray;
var
  i, N: integer;
begin
  Result:= nil;

  SText:= UnicodeLowerCase(SText);
  SFind:= UnicodeLowerCase(SFind);

  //if simple match is found, don't calculate complex fuzzy matches
  N:= Pos(SFind, SText);
  if N>0 then
  begin
    SetLength(Result, Length(SFind));
    Result[0]:= N;
    for i:= 1 to High(Result) do
      Result[i]:= Result[i-1]+1;
    exit;
  end;

  //calculate complex matches
  N:= 0;
  SetLength(Result, Length(SFind));
  for i:= 1 to Length(SFind) do
  begin
    N:= PosEx(SFind[i], SText, N+1);
    if N=0 then Exit(nil);
    Result[i-1]:= N;
  end;
end;


function SRegexFindParts(const ARegex, AStr: string; out AParts: TRegexParts): boolean;
var
  Obj: TRegExpr;
  i: integer;
begin
  Result:= false;
  for i:= Low(AParts) to High(AParts) do
  begin
    AParts[i].Pos:= -1;
    AParts[i].Len:= 0;
    AParts[i].Str:= '';
  end;

  if ARegex='' then exit;
  if AStr='' then exit;

  Obj:= TRegExpr.Create;
  try
{'fraction_non_alphanumeric': 0.12422894850093244, 'fraction_numerical': 0.006598766317601492, 'mean_word_length': 4.25, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5648624', 'n_tokens_mistral': 2451, 'n_tokens_neox': 2302, 'n_words': 689}