<commit_before>// This is part of the Mommon Library
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed without any warranty.
//
// @author <NAME> (<EMAIL> - www.mammola.net)

unit mConcatenationDataProvider;

{$IFDEF FPC}
  {$MODE DELPHI}
{$ENDIF}

interface


uses
  Classes, contnrs,
  {$IFNDEF FPC}mInterfaces,{$ENDIF}
  mDataProviderFieldDefs, mDataProviderInterfaces, mNullables;

const
  CONCATENATION_DATA_PROVIDER_FIELD_LENGTH = 1024;

type

  { TConcatenationDatumKey }

  TConcatenationDatumKey = class(IVDDatumKey)
  strict private
    FIndex : TNullableInteger;
  public
    constructor Create;
    destructor Destroy; override;

    procedure Assign(aSource : TObject);
    function AsString : string;

    property Index : TNullableInteger read FIndex;
  end;

  TConcatenationDatum = class ({$IFNDEF FPC}TJavaInterfacedObject, {$ENDIF}IVDDatum)
  strict private
    FKey : TConcatenationDatumKey;
    FValue : TNullableString;
    FOriginalDatum : IVDDatum;
  public
    const FLD_KEY = 'KEY';
    const FLD_VALUE = 'VALUE';
  public
    constructor Create;
    destructor Destroy; override;

    function GetDatumKey : IVDDatumKey;
    function AsObject: TObject;

    class procedure FillVirtualFieldDefs (aFieldDefs : TmVirtualFieldDefs; const aPrefix : String);
    class function GetKeyField : String;
    function GetPropertyByFieldName(const aFieldName : String) : Variant;

    property Key : TConcatenationDatumKey read FKey;
    property Value : TNullableString read FValue;
    property OriginalDatum : IVDDatum read FOriginalDatum write FOriginalDatum;
  end;

  { TConcatenationDataProvider }

  TConcatenationDataProvider = class (IVDDataProvider)
  strict private
    FExternalDataProvider : IVDDataProvider;
    FFieldsToBeConcatenated : TStringList;
    FDatumList : TObjectList;
  public
    constructor Create(const aExternalDataProvider : IVDDataProvider);
    destructor Destroy; override;
    function Count : integer;
    function GetDatum(const aIndex : integer) : IVDDatum;
    function FindDatumByKey (const aKey : IVDDatumKey) : IVDDatum;
    function FindDatumByStringKey (const aStringKey : string): IVDDatum;
    procedure Clear;
    procedure FillVirtualFieldDefs (aFieldDefs : TmVirtualFieldDefs; const aPrefix : String);
    function GetKeyFieldName : String;
    procedure GetMinimumFields(aFieldsForLookup : TStringList);
    procedure Refresh;

    property FieldsToBeConcatenated : TStringList read FFieldsToBeConcatenated;
  end;



implementation

uses
  Variants;

{ TConcatenationDataProvider }

constructor TConcatenationDataProvider.Create(const aExternalDataProvider: IVDDataProvider);
begin
  FExternalDataProvider := aExternalDataProvider;
  FFieldsToBeConcatenated := TStringList.Create;
  FDatumList := TObjectList.Create(true);
end;

destructor TConcatenationDataProvider.Destroy;
begin
  FDatumList.Free;
  FFieldsToBeConcatenated.Free;
end;

function TConcatenationDataProvider.Count: integer;
begin
  Result := FDatumList.Count;
end;

function TConcatenationDataProvider.GetDatum(const aIndex: integer): IVDDatum;
begin
  Result := FDatumList.Items[aIndex] as TConcatenationDatum;
end;

function TConcatenationDataProvider.FindDatumByKey(const aKey: IVDDatumKey): IVDDatum;
begin
  Result := Self.FindDatumByStringKey(TConcatenationDatumKey(aKey).AsString);
end;

function TConcatenationDataProvider.FindDatumByStringKey(const aStringKey: string): IVDDatum;
var
  i : integer;
begin
  // eventually to be optimized..
  // useless if TConcatenationDataProvider is used together with mPivoter
  Result := nil;
  for i := 0 to FDatumList.Count - 1 do
  begin
    if (FDatumList.Items[i] as TConcatenationDatum).Key.AsString = aStringKey then
    begin
      Result := FDatumList.Items[i] as TConcatenationDatum;
      exit;<commit_msg>* warning ignored<commit_after>// This is part of the Mommon Library
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed without any warranty.
//
// @author <NAME> (<EMAIL> - www.mammola.net)

unit mConcatenationDataProvider;

{$IFDEF FPC}
  {$MODE DELPHI}
{$ENDIF}

interface


uses
  Classes, contnrs,
  {$IFNDEF FPC}mInterfaces,{$ENDIF}
  mDataProviderFieldDefs, mDataProviderInterfaces, mNullables;

const
  CONCATENATION_DATA_PROVIDER_FIELD_LENGTH = 1024;

type

  { TConcatenationDatumKey }

  TConcatenationDatumKey = class(IVDDatumKey)
  strict private
    FIndex : TNullableInteger;
  public
    constructor Create;
    destructor Destroy; override;

    procedure Assign(aSource : TObject);
    function AsString : string;

    property Index : TNullableInteger read FIndex;
  end;

  TConcatenationDatum = class ({$IFNDEF FPC}TJavaInterfacedObject, {$ENDIF}IVDDatum)
  strict private
    FKey : TConcatenationDatumKey;
    FValue : TNullableString;
    FOriginalDatum : IVDDatum;
  public
    const FLD_KEY = 'KEY';
    const FLD_VALUE = 'VALUE';
  public
    constructor Create;
    destructor Destroy; override;

    function GetDatumKey : IVDDatumKey;
    function AsObject: TObject;

    class procedure FillVirtualFieldDefs (aFieldDefs : TmVirtualFieldDefs; const aPrefix : String);
    class function GetKeyField : String;
    function GetPropertyByFieldName(const aFieldName : String) : Variant;

    property Key : TConcatenationDatumKey read FKey;
    property Value : TNullableString read FValue;
    property OriginalDatum : IVDDatum read FOriginalDatum write FOriginalDatum;
  end;

  { TConcatenationDataProvider }

  TConcatenationDataProvider = class (IVDDataProvider)
  strict private
    FExternalDataProvider : IVDDataProvider;
    FFieldsToBeConcatenated : TStringList;
    FDatumList : TObjectList;
  public
    constructor Create(const aExternalDataProvider : IVDDataProvider);
    destructor Destroy; override;
    function Count : integer;
    function GetDatum(const aIndex : integer) : IVDDatum;
    function FindDatumByKey (const aKey : IVDDatumKey) : IVDDatum;
    function FindDatumByStringKey (const aStringKey : string): IVDDatum;
    procedure Clear;
    procedure FillVirtualFieldDefs (aFieldDefs : TmVirtualFieldDefs; const aPrefix : String);
    function GetKeyFieldName : String;
    procedure GetMinimumFields(aFieldsForLookup : TStringList);
    procedure Refresh;

    property FieldsToBeConcatenated : TStringList read FFieldsToBeConcatenated;
  end;



implementation

uses
  Variants;

{ TConcatenationDataProvider }

constructor TConcatenationDataProvider.Create(const aExternalDataProvider: IVDDataProvider);
begin
  FExternalDataProvider := aExternalDataProvider;
  FFieldsToBeConcatenated := TStringList.Create;
  FDatumList := TObjectList.Create(true);
end;

destructor TConcatenationDataProvider.Destroy;
begin
  FDatumList.Free;
  FFieldsToBeConcatenated.Free;
end;

function TConcatenationDataProvider.Count: integer;
begin
  Result := FDatumList.Count;
end;

function TConcatenationDataProvider.GetDatum(const aIndex: integer): IVDDatum;
begin
  Result := FDatumList.Items[aIndex] as TConcatenationDatum;
end;

function TConcatenationDataProvider.FindDatumByKey(const aKey: IVDDatumKey): IVDDatum;
begin
  {$push}
  {$warnings off}
  Result := Self.FindDatumByStringKey(TConcatenationDatumKey(aKey).AsString);
  {$pop}
end;

function TConcatenationDataProvider.FindDatumByStringKey(const aStringKey: string): IVDDatum;
var
  i : integer;
begin
  // eventually to be optimized..
  // useless if TConcatenationDataProvider is used together with mPivoter
  Result := nil;
  for i := 0 to FDatumList.Count - 1 do
  begin
    if (FDatumList.Items[i] as TConcatenationDatum).Key.AsString = aStringKey then
    begin
      Result := FDatumList.Items[i] as TConcatenationDatum;
      exit;
{'fraction_non_alphanumeric': 0.07432348173088914, 'fraction_numerical': 0.0024940765681506424, 'mean_word_length': 4.620182200420462, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3858987', 'n_tokens_mistral': 2628, 'n_tokens_neox': 2522, 'n_words': 645}