<commit_before>#!/usr/bin/ruby
# $Id$
# -*- ruby -*- vim:set ft=ruby et sw=2 sts=2:

require 'ldap'
require 'ldaptor/schema'
require 'ldap/filter'

module Ldaptor

  # RFC 2252.  Second column is "Human Readable"
  SYNTAX_STRING = <<-EOF unless defined? SYNTAX_STRING
ACI Item                        N  1.3.6.1.4.1.1466.172.16.17.32
Access Point                    Y  1.3.6.1.4.1.1466.172.16.58.3
Attribute Type Description      Y  1.3.6.1.4.1.1466.172.16.31.10
Audio                           N  1.3.6.1.4.1.1466.172.16.17.32
Binary                          N  1.3.6.1.4.1.1466.172.16.58.3
Bit String                      Y  1.3.6.1.4.1.1466.192.168.3.11
Boolean                         Y  1.3.6.1.4.1.1466.172.16.31.10
Certificate                     N  1.3.6.1.4.1.1466.172.16.58.3
Certificate List                N  1.3.6.1.4.1.1466.172.16.58.3
Certificate Pair                N  1.3.6.1.4.1.1466.172.16.17.32
Country String                  Y  1.3.6.1.4.1.1466.172.16.58.3
DN                              Y  1.3.6.1.4.1.1466.192.168.3.11
Data Quality Syntax             Y  1.3.6.1.4.1.1466.192.168.127.12
Delivery Method                 Y  1.3.6.1.4.1.1466.172.16.58.3
Directory String                Y  1.3.6.1.4.1.1466.192.168.127.12
DIT Content Rule Description    Y  1.3.6.1.4.1.1466.192.168.3.11
DIT Structure Rule Description  Y  1.3.6.1.4.1.1466.192.168.127.12
DL Submit Permission            Y  1.3.6.1.4.1.1466.172.16.17.32
DSA Quality Syntax              Y  1.3.6.1.4.1.1466.192.168.3.11
DSE Type                        Y  1.3.6.1.4.1.1466.192.168.127.12
Enhanced Guide                  Y  1.3.6.1.4.1.1466.172.16.58.3
Facsimile Telephone Number      Y  1.3.6.1.4.1.1466.172.16.31.10
Fax                             N  1.3.6.1.4.1.1466.192.168.127.12
Generalized Time                Y  1.3.6.1.4.1.1466.172.16.17.32
Guide                           Y  1.3.6.1.4.1.1466.192.168.3.11
IA5 String                      Y  1.3.6.1.4.1.1466.172.16.17.32
INTEGER                         Y  1.3.6.1.4.1.1466.192.168.3.11
JPEG                            N  1.3.6.1.4.1.1466.172.16.58.3
LDAP Syntax Description         Y  1.3.6.1.4.1.1466.172.16.31.10
LDAP Schema Definition          Y  1.3.6.1.4.1.1466.192.168.3.11
LDAP Schema Description         Y  1.3.6.1.4.1.1466.192.168.3.11
Master And Shadow Access Points Y  1.3.6.1.4.1.1466.172.16.17.32
Matching Rule Description       Y  1.3.6.1.4.1.1466.172.16.58.3
Matching Rule Use Description   Y  1.3.6.1.4.1.1466.192.168.127.12
Mail Preference                 Y  1.3.6.1.4.1.1466.192.168.3.11
MHS OR Address                  Y  1.3.6.1.4.1.1466.192.168.3.11
Modify Rights                   Y  1.3.6.1.4.1.1466.192.168.3.11
Name And Optional UID           Y  1.3.6.1.4.1.1466.192.168.3.11
Name Form Description           Y  1.3.6.1.4.1.1466.192.168.127.12
Numeric String                  Y  1.3.6.1.4.1.1466.192.168.127.12
Object Class Description        Y  1.3.6.1.4.1.1466.115.121.1.37
Octet String                    Y  1.3.6.1.4.1.1466.192.168.3.11
OID                             Y  1.3.6.1.4.1.1466.192.168.3.11
Other Mailbox                   Y  1.3.6.1.4.1.1466.172.16.31.10
Postal Address                  Y  1.3.6.1.4.1.1466.172.16.58.3
Protocol Information            Y  1.3.6.1.4.1.1466.172.16.17.32
Presentation Address            Y  1.3.6.1.4.1.1466.192.168.127.12
Printable String                Y  1.3.6.1.4.1.1466.192.168.3.11
Substring Assertion             Y  1.3.6.1.4.1.1466.192.168.3.11
Subtree Specification           Y  1.3.6.1.4.1.1466.172.16.31.10
Supplier Information            Y  1.3.6.1.4.1.1466.172.16.17.32
Supplier Or Consumer            Y  1.3.6.1.4.1.1466.172.16.31.10
Supplier And Consumer           Y  1.3.6.1.4.1.1466.192.168.127.12
Supported Algorithm             N  1.3.6.1.4.1.1466.172.16.31.10
Telephone Number                Y  1.3.6.1.4.1.1466.172.16.58.3
Teletex Terminal Identifier     Y  1.3.6.1.4.1.1466.172.16.58.31
Telex Number                    Y  1.3.6.1.4.1.1466.172.16.58.32
UTC Time                        Y  1.3.6.1.4.1.1466.192.168.3.11
EOF

  SYNTAXES = {} unless defined? SYNTAXES
  SYNTAX_STRING.each_line do |line|
    a, b = line.chomp.split(/ [YN]  /)
    a.rstrip!
    SYNTAXES[b] = a
  end

  module Syntaxes

    module DirectoryString
      def self.parse(string)
        string
      end
      def self.format(string)
        string.to_str
      end
    end

    module Boolean
      def self.parse(string)
        return string == "TRUE"
      end
      def self.format(boolean)
        case boolean
        when "TRUE",  true  then "TRUE"
        when "FALSE", false then "FALSE"
        else raise TypeError, "boolean expected", caller
        end
      end
    end

    module INTEGER
      def self.parse(string)
        return string.to_i
      end
      def self.format(integer)
        Integer(integer).to_s
      end
    end

    module GeneralizedTime
      def self.parse(string)
        require 'time'
        Time.parse(string)
      end
      def self.format(time)
        time.utc.strftime("%Y%m%d%H%M%S")+".%06dZ" % (time.usec/100_000)
      end
    end

  end

  class SchemaLine

    def self.attr_boolean_schema(*attrs)
      attr_ldap_reader(*attrs)
      @hash ||= {}
      attrs.each do |attr|
        @hash[attr.to_s.upcase.to_s.gsub('_','-')] = :boolean
      end
    end

    def self.attr_string_schema(*attrs)
      attr_ldap_reader(*attrs)
      @hash ||= {}
      attrs.each do |attr|
        @hash[attr.to_s.upcase.to_s.gsub('_','-')] = :string
      end
    end

    def self.attr_array_schema(*attrs)
      attr_ldap_reader(*attrs)
      @hash ||= {}
      attrs.each do |attr|
        @hash[attr.to_s.upcase.to_s.gsub('_','-')] = :array
      end
    end

    def self.attr_ldap_reader(*attrs)
      attrs.each do |attr|
        class_eval(<<-EOS)
          def #{attr.to_s.downcase.gsub('-','_')}
            @hash[#{attr.to_s.upcase.gsub('_','-').inspect}]
          end
        EOS
      end
    end

    attr_accessor :oid
    def self.attributes
      @hash ||= {}
    end

    def initialize(string)
      @string = string.dup
      string = @string.dup
      # raise string unless string =~ /^\(\s*(\d[.\d]*\d) (.*?)\s*\)\s*$/
      string.gsub!(/^\s*\(\s*(\d[\d.]*\d)\s*(.*?)\s*\)\s*$/,'\\2')
      @oid = $1
      @hash = {}
      while value = eat(string,/^\s*([A-Z-]+)\s*/)
        if self.class.attributes[value] == :string
          @hash[value] = eatstr(string)
        elsif self.class.attributes[value] == :array
          @hash[value] = eatary(string)
        elsif self.class.attributes[value] == :boolean
          @hash[value] = true
        elsif value =~ /^X-/
          eatstr(string)
        end
      end
    end

    def to_s
      name
    end

    def eat(string,regex)
      string.gsub!(regex,'')
      $1 || $&
    end
    def eatstr(string)
      if eaten = eat(string, /^\(([^)]+)\)/i)
        eaten.split("$").collect{|attr| attr.strip.sub(/^'(.*)'$/,'\1') }
      else
        eat(string,/^'([^']*)'\s*/)
      end
    end

    def eatary(string)
      if eaten = eat(string, /^\(([\w\d_\s\$-]+)\)/i)
        eaten.split("$").collect{|attr| attr.strip}
      else
        eat(string,/^([\w\d_-]+)/i)
      end
    end

  end

  class AttributeType < SchemaLine
    attr_boolean_schema :obsolete, :single_value, :collective, :no_user_modification
    attr_string_schema  :name, :desc, :sup, :equality, :ordering, :substr, :syntax, :usage
  end

  class ObjectClass < SchemaLine
    attr_boolean_schema :structural, :auxiliary, :abstract, :obsolete
    attr_string_schema  :name, :desc
    attr_array_schema   :sup, :may, :must
  end

  class Error < ::RuntimeError #:nodoc:
  end

  class RecordNotFound < Error
  end<commit_msg>Fixed name array parsing<commit_after>#!/usr/bin/ruby
# $Id$
# -*- ruby -*- vim:set ft=ruby et sw=2 sts=2:

require 'ldap'
require 'ldaptor/schema'
require 'ldap/filter'

module Ldaptor

  # RFC 2252.  Second column is "Human Readable"
  SYNTAX_STRING = <<-EOF unless defined? SYNTAX_STRING
ACI Item                        N  1.3.6.1.4.1.1466.172.16.17.32
Access Point                    Y  1.3.6.1.4.1.1466.172.16.58.3
Attribute Type Description      Y  1.3.6.1.4.1.1466.172.16.31.10
Audio                           N  1.3.6.1.4.1.1466.172.16.17.32
Binary                          N  1.3.6.1.4.1.1466.172.16.58.3
Bit String                      Y  1.3.6.1.4.1.1466.192.168.3.11
Boolean                         Y  1.3.6.1.4.1.1466.172.16.31.10
Certificate                     N  1.3.6.1.4.1.1466.172.16.58.3
Certificate List                N  1.3.6.1.4.1.1466.172.16.58.3
Certificate Pair                N  1.3.6.1.4.1.1466.172.16.17.32
Country String                  Y  1.3.6.1.4.1.1466.172.16.58.3
DN                              Y  1.3.6.1.4.1.1466.192.168.3.11
Data Quality Syntax             Y  1.3.6.1.4.1.1466.192.168.127.12
Delivery Method                 Y  1.3.6.1.4.1.1466.172.16.58.3
Directory String                Y  1.3.6.1.4.1.1466.192.168.127.12
DIT Content Rule Description    Y  1.3.6.1.4.1.1466.192.168.3.11
DIT Structure Rule Description  Y  1.3.6.1.4.1.1466.192.168.127.12
DL Submit Permission            Y  1.3.6.1.4.1.1466.172.16.17.32
DSA Quality Syntax              Y  1.3.6.1.4.1.1466.192.168.3.11
DSE Type                        Y  1.3.6.1.4.1.1466.192.168.127.12
Enhanced Guide                  Y  1.3.6.1.4.1.1466.172.16.58.3
Facsimile Telephone Number      Y  1.3.6.1.4.1.1466.172.16.58.32
Fax                             N  1.3.6.1.4.1.1466.192.168.127.12
Generalized Time                Y  1.3.6.1.4.1.1466.172.16.17.32
Guide                           Y  1.3.6.1.4.1.1466.192.168.3.11
IA5 String                      Y  1.3.6.1.4.1.1466.172.16.17.32
INTEGER                         Y  1.3.6.1.4.1.1466.192.168.3.11
JPEG                            N  1.3.6.1.4.1.1466.172.16.58.3
LDAP Syntax Description         Y  1.3.6.1.4.1.1466.172.16.31.10
LDAP Schema Definition          Y  1.3.6.1.4.1.1466.192.168.3.11
LDAP Schema Description         Y  1.3.6.1.4.1.1466.192.168.3.11
Master And Shadow Access Points Y  1.3.6.1.4.1.1466.172.16.17.32
Matching Rule Description       Y  1.3.6.1.4.1.1466.172.16.58.3
Matching Rule Use Description   Y  1.3.6.1.4.1.1466.192.168.127.12
Mail Preference                 Y  1.3.6.1.4.1.1466.192.168.3.11
MHS OR Address                  Y  1.3.6.1.4.1.1466.192.168.3.11
Modify Rights                   Y  1.3.6.1.4.1.1466.172.16.58.35
Name And Optional UID           Y  1.3.6.1.4.1.1466.172.16.31.104
Name Form Description           Y  1.3.6.1.4.1.1466.192.168.127.12
Numeric String                  Y  1.3.6.1.4.1.1466.172.16.31.106
Object Class Description        Y  1.3.6.1.4.1.1466.192.168.3.11
Octet String                    Y  1.3.6.1.4.1.1466.192.168.3.11
OID                             Y  1.3.6.1.4.1.1466.192.168.3.11
Other Mailbox                   Y  1.3.6.1.4.1.1466.172.16.31.10
Postal Address                  Y  1.3.6.1.4.1.1466.172.16.58.3
Protocol Information            Y  1.3.6.1.4.1.1466.172.16.17.32
Presentation Address            Y  1.3.6.1.4.1.1466.192.168.127.12
Printable String                Y  1.3.6.1.4.1.1466.192.168.3.11
Substring Assertion             Y  1.3.6.1.4.1.1466.192.168.3.11
Subtree Specification           Y  1.3.6.1.4.1.1466.172.16.31.10
Supplier Information            Y  1.3.6.1.4.1.1466.172.16.17.32
Supplier Or Consumer            Y  1.3.6.1.4.1.1466.172.16.17.327
Supplier And Consumer           Y  1.3.6.1.4.1.1466.172.16.17.328
Supported Algorithm             N  1.3.6.1.4.1.1466.172.16.31.10
Telephone Number                Y  1.3.6.1.4.1.1466.172.16.58.30
Teletex Terminal Identifier     Y  1.3.6.1.4.1.1466.192.168.3.11
Telex Number                    Y  1.3.6.1.4.1.1466.172.16.58.3
UTC Time                        Y  1.3.6.1.4.1.1466.192.168.3.11
EOF

  SYNTAXES = {} unless defined? SYNTAXES
  SYNTAX_STRING.each_line do |line|
    a, b = line.chomp.split(/ [YN]  /)
    a.rstrip!
    SYNTAXES[b] = a
  end

  module Syntaxes

    module DirectoryString
      def self.parse(string)
        string
      end
      def self.format(string)
        string.to_str
      end
    end

    module Boolean
      def self.parse(string)
        return string == "TRUE"
      end
      def self.format(boolean)
        case boolean
        when "TRUE",  true  then "TRUE"
        when "FALSE", false then "FALSE"
        else raise TypeError, "boolean expected", caller
        end
      end
    end

    module INTEGER
      def self.parse(string)
        return string.to_i
      end
      def self.format(integer)
        Integer(integer).to_s
      end
    end

    module GeneralizedTime
      def self.parse(string)
        require 'time'
        Time.parse(string)
      end
      def self.format(time)
        time.utc.strftime("%Y%m%d%H%M%S")+".%06dZ" % (time.usec/100_000)
      end
    end

  end

  class SchemaLine

    def self.attr_boolean_schema(*attrs)
      attr_ldap_reader(*attrs)
      @hash ||= {}
      attrs.each do |attr|
        @hash[attr.to_s.upcase.to_s.gsub('_','-')] = :boolean
      end
    end

    def self.attr_string_schema(*attrs)
      attr_ldap_reader(*attrs)
      @hash ||= {}
      attrs.each do |attr|
        @hash[attr.to_s.upcase.to_s.gsub('_','-')] = :string
      end
    end

    def self.attr_array_schema(*attrs)
      attr_ldap_reader(*attrs)
      @hash ||= {}
      attrs.each do |attr|
        @hash[attr.to_s.upcase.to_s.gsub('_','-')] = :array
      end
    end

    def self.attr_ldap_reader(*attrs)
      attrs.each do |attr|
        class_eval(<<-EOS)
          def #{attr.to_s.downcase.gsub('-','_')}
            @hash[#{attr.to_s.upcase.gsub('_','-').inspect}]
          end
        EOS
      end
    end

    attr_accessor :oid
    def self.attributes
      @hash ||= {}
    end

    def initialize(string)
      @string = string.dup
      string = @string.dup
      # raise string unless string =~ /^\(\s*(\d[.\d]*\d) (.*?)\s*\)\s*$/
      string.gsub!(/^\s*\(\s*(\d[\d.]*\d)\s*(.*?)\s*\)\s*$/,'\\2')
      @oid = $1
      @hash = {}
      while value = eat(string,/^\s*([A-Z-]+)\s*/)
        if self.class.attributes[value] == :string
          @hash[value] = eatstr(string)
        elsif self.class.attributes[value] == :array
          @hash[value] = eatary(string)
        elsif self.class.attributes[value] == :boolean
          @hash[value] = true
        elsif value =~ /^X-/
          eatstr(string)
        end
      end
    end

    def to_s
      name
    end

    def eat(string,regex)
      string.gsub!(regex,'')
      $1 || $&
    end
    def eatstr(string)
      if eaten = eat(string, /^\(\s*'([^)]+)'\s*\)/i)
        eaten.split("' '").collect{|attr| attr.strip }
      else
        eat(string,/^'([^']*)'\s*/)
      end
    end

    def eatary(string)
      if eaten = eat(string, /^\(([\w\d_\s\$-]+)\)/i)
        eaten.split("$").collect{|attr| attr.strip}
      else
        eat(string,/^([\w\d_-]+)/i)
      end
    end

  end

  class AttributeType < SchemaLine
    attr_boolean_schema :obsolete, :single_value, :collective, :no_user_modification
    attr_string_schema  :name, :desc, :sup, :equality, :ordering, :substr, :syntax, :usage
  end

  class ObjectClass < SchemaLine
    attr_boolean_schema :structural, :auxiliary, :abstract, :obsolete
    attr_string_schema  :name, :desc
    attr_array_schema   :sup, :may, :must
  end

  class Error < ::RuntimeError #:nodoc:
  end

  class RecordNotFound < Error
  end
{'fraction_non_alphanumeric': 0.15407541790851367, 'fraction_numerical': 0.14636516781132564, 'mean_word_length': 1.547870584351271, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 232, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5646294', 'n_tokens_mistral': 7167, 'n_tokens_neox': 6158, 'n_words': 1073}