<commit_before>module CineworldUk
  # Internal utility classes: Do not use
  # @api private
  module Internal
    # Sanitize and standardize film titles
    class TitleSanitizer
      # strings and regex to be removed
      REMOVE = [
        /\s+[23]d/i,                          # dimension
        /\(Hindi\)/i,                         # Indian Language
        /\(Malayalam\)/i,                     # Indian Language
        /\(Punjabi\)/i,                       # Indian Language
        /\(Tamil\)/i,                         # Indian Language
        /\(Turkish\)/i,                       # Foreign Language
        %r(-? \d{1,2}/\d{1,2}/\d{2,4}),       # date
        /\n/,                                 # newlines
        /\- Encore/,                          # encore
        'Autism Friendly Screening:',         # autism screening
        '- Unlimited Screening',              # unlimited screening
        /LFF Opening Night Live/,             # london film festival
        '- Special Performance',              # special performance
        /\ATake 2 -/,                         # take 2
        / - (Subtitled) Movies for Juniors/i, # movies for juniors
        '(IMAX)',                             # IMAX
        'SciScreen: ',                        # SciScreen
        ' - Live Q and A',                    # Live Q&A
        /(\s+\-\s+)?MOVIES FOR JUNIORS/i,     # movies for juniors
        /\bsing\-?a\-?long\b/i,               # singalong
      ]

      # regexes and their replacements
      REPLACE = {
        /Bolshoi Ballet: (.*)/         => 'Bolshoi: ',
        /ENO: (.*)/i                   => 'English National Opera: ',
        /Guardian Live - (.*)/         => 'The Guardian: ',
        /Met Opera - (.*)/i            => 'Met Opera: ',
        /NT Live: (.*)/                => 'National Theatre: ',
        /NT Live Encore: (.*)/         => 'National Theatre: ',
        /ROH - (.*)/                   => 'Royal Opera House: ',
        /RSC Live: (.*)/               => 'Royal Shakespeare Company: ',
        /The Royal Ballet - (.*)/      => 'The Royal Ballet: '
      }

      # @param [String] title a film title
      def initialize(title)
        @title = title
      end

      # sanitized and standardized title
      # @return [String] title
      def sanitized
        @sanitzed ||= begin
          sanitized = @title
          REMOVE.each do |pattern|
            sanitized.gsub! pattern, ''
          end
          REPLACE.each do |pattern, prefix|
            sanitized.gsub!(pattern) { |_| prefix + $1 }
          end
          sanitized.squeeze(' ').strip
        end
      end
    end
  end
end<commit_msg>TitleSanitzer inherits from cinebase<commit_after>module CineworldUk
  # Internal utility classes: Do not use
  # @api private
  module Internal
    # Sanitize and standardize film titles
    class TitleSanitizer < Cinebase::TitleSanitizer

      # @!method initialize(title)
      #   @param [String] title a film title
      #   @return [CineworldUk::Internal::TitleSanitizer]

      # @!method sanitized
      #   sanitized and standardized title
      #   @return [String] sanitised title

      private

      # Strings and regex to be removed
      def remove
        [
          /\s+[23]d/i,                          # dimension
          /\(Hindi\)/i,                         # Indian Language
          /\(Malayalam\)/i,                     # Indian Language
          /\(Punjabi\)/i,                       # Indian Language
          /\(Tamil\)/i,                         # Indian Language
          /\(Cantonese\)/i,                     # Chinese
          /\(Turkish\)/i,                       # Foreign Language
          %r(-? \d{1,2}/\d{1,2}/\d{2,4}),       # date
          /\n/,                                 # newlines
          /\- Encore/,                          # encore
          /Autism Friendly Screening\s*[:-] /,  # autism screening
          '- Unlimited Screening',              # unlimited screening
          /LFF Opening Night Live/,             # london film festival
          '- Special Performance',              # special performance
          /\ATake 2 -/,                         # take 2
          / - (Subtitled) Movies for Juniors/i, # movies for juniors
          '(IMAX)',                             # IMAX
          'SciScreen: ',                        # SciScreen
          /\s(-\s)*(With)*\sLive Q And A/i,     # Live Q&A
          /(\s+\-\s+)?MOVIES FOR JUNIORS/i,     # movies for juniors
          /\bsing\-?a\-?long\b/i,               # singalong
          %r{\s20\d{2}[\/-]20\d{2} Season\b}
        ]
      end

      # Regexes and their replacements
      def replace
        {
          /Bolshoi Ballet\s*[:-] (.*)/       => 'Bolshoi Ballet: ',
          /ENO\s*[:-] (.*)/i                 => 'English National Opera: ',
          /Guardian Live\s*[:-] (.*)/        => 'The Guardian: ',
          /Glyndebourne 20\d{2}\s*[:-] (.*)/ => 'Glyndebourne: ',
          /Met Opera\s*[:-] (.*)/i           => 'Met Opera: ',
          /NT Live\s*[:-] (.*)/              => 'National Theatre: ',
          /NT Live Encore\s*[:-] (.*)/       => 'National Theatre: ',
          /ROH\s*[:-] (.*)/                  => 'Royal Opera House: ',
          /ROH Royal Opera\s*[:-] (.*)/      => 'Royal Opera House: ',
          /RSC Live\s*[:-] (.*)/             => 'Royal Shakespeare Company: ',
          /The Royal Ballet\s*[:-] (.*)/     => 'The Royal Ballet: ',
          /Royal Ballet\s*[:-] (.*)/         => 'The Royal Ballet: ',
          /ROH The Royal Ballet\s*[:-] (.*)/ => 'The Royal Ballet: ',
          /Royal Ballet Live\s*[:-] (.*)/    => 'The Royal Ballet: ',
          /San Francisco Ballets (.*)/       => 'San Francisco Ballet: '
        }
      end

    end
  end
end
{'fraction_non_alphanumeric': 0.14399308556611928, 'fraction_numerical': 0.005185825410544511, 'mean_word_length': 1.1319086219602064, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5441173', 'n_tokens_mistral': 1758, 'n_tokens_neox': 1626, 'n_words': 404}