<commit_before>require 'optparse'
require 'pg'
require 'set'

require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_cluster'))

indent_level = 0

OptionParser.new do |opts|
  opts.banner = "Usage: #$PROGRAM_NAME [options]"

  opts.on('-iLEVEL', '--indent=LEVEL', 'Indent level') do |indent|
    indent_level = Integer(indent)
  end
end.parse!

INDENT = (' ' * indent_level).freeze

def iputs(level, *args)
  print(INDENT)
  print(' ' * level) unless level == 0
  puts(*args)
end

TEST_CLUSTER.bottledwater_format = :json # skip schema registry
TEST_CLUSTER.start
at_exit { TEST_CLUSTER.stop }
pg = TEST_CLUSTER.postgres

types = pg.exec(<<-SQL)
  SELECT
    -- get e.g. 'timestamp with time zone' instead of 'timestamptz'
    format_type(t.oid, NULL) AS name,
    t.typname AS typname,
    t.typcategory AS typcategory
  FROM pg_type t
  WHERE
    -- exclude composite and pseudotypes
    t.typtype NOT IN ('c', 'p')
    -- exclude invisible types
    AND pg_type_is_visible(t.oid)
    -- exclude 'element' types
    AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_type el WHERE el.oid = t.typelem AND el.typarray = t.oid)
  ORDER BY name
SQL

# Add types to this list if they require a more specific way to create a
# test value than can be inferred from just their 'typcategory'
#
# If the value is a string, it will be escaped (via String#inspect) and
# included in the generated code as a string literal.  If the value is wrapped
# in a single-element Array, it will be included unescaped in the generated
# code.  See function #genvalue below for reference.
CUSTOM_VALUE_TYPES = {
  'date'    => ['TEST_DATETIME.to_date'],

  # inet types
  'cidr'    => '192.168.1.0/24',
  'inet'    => '192.168.1.1/24',

  # range types
  'int4range' => '[1,5)',
  'int8range' => '[1,5)',
  'numrange' => '[1,5)',
  'daterange' => '[1837-06-20,1901-01-22)', # the reign of Queen Victoria
  'tsrange' =>   '["1837-06-20 00:00:00","1901-01-22 00:00:00")',
  'tstzrange' => '["1837-06-20 00:00:00+00","1901-01-22 00:00:00+00")',

  # geometric types
  'point' => '(3,4)',
  'line' => '{3,4,5}',
  'lseg' => '[(0,0),(3,4)]',
  'box' => '(3,4),(0,0)',
  'path' => '((1,2),(0,0),(3,4))',
  'polygon' => '((1,2),(0,0),(3,4))',
  'circle' => '<(1,2),5>',
}
def genvalue(value)
  case value
  when Array
    raise "Bad value expression: #{value.inspect}" if value.size != 1
    value.first
  else
    value.inspect
  end
end

# add types to this list if the test table needs to explicitly specify the
# length of the type
BOUNDED_LENGTH_TYPES = Set[*%w(
  bit
  character
)]

# add types to this list if they're too obscure to bother filing an issue for
# not supporting them (e.g. Postgres internals)
INTERNAL_TYPES = Set[*%w(
  abstime
  aclitem
  "char"
  cid
  ghstore
  gtsquery
  gtsvector
  int2vector
  name
  oidvector
  pg_node_tree
  refcursor
  regclass
  regconfig
  regdictionary
  regnamespace
  regoper
  regoperator
  regproc
  regprocedure
  regrole
  regtype
  reltime
  smgr
  tid
  tinterval
  txid_snapshot
  unknown
  xid
)]

# add types to this list when their lack of support is documented, preferably
# in the form of a Github issue
KNOWN_BUGS = {
  'money' => ['multiplied by 100', 'https://github.com/confluentinc/bottledwater-pg/issues/60'],
  'numeric' => ['replaced by zero', 'https://github.com/confluentinc/bottledwater-pg/issues/4'],
}

# only use this list during development, otherwise file an issue!
UNKNOWN_BUGS = {
}<commit_msg>Remove "Numeric replaced by zero" from KNOWN_BUGS<commit_after>require 'optparse'
require 'pg'
require 'set'

require File.expand_path(File.join(File.dirname(__FILE__), '..', 'test_cluster'))

indent_level = 0

OptionParser.new do |opts|
  opts.banner = "Usage: #$PROGRAM_NAME [options]"

  opts.on('-iLEVEL', '--indent=LEVEL', 'Indent level') do |indent|
    indent_level = Integer(indent)
  end
end.parse!

INDENT = (' ' * indent_level).freeze

def iputs(level, *args)
  print(INDENT)
  print(' ' * level) unless level == 0
  puts(*args)
end

TEST_CLUSTER.bottledwater_format = :json # skip schema registry
TEST_CLUSTER.start
at_exit { TEST_CLUSTER.stop }
pg = TEST_CLUSTER.postgres

types = pg.exec(<<-SQL)
  SELECT
    -- get e.g. 'timestamp with time zone' instead of 'timestamptz'
    format_type(t.oid, NULL) AS name,
    t.typname AS typname,
    t.typcategory AS typcategory
  FROM pg_type t
  WHERE
    -- exclude composite and pseudotypes
    t.typtype NOT IN ('c', 'p')
    -- exclude invisible types
    AND pg_type_is_visible(t.oid)
    -- exclude 'element' types
    AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_type el WHERE el.oid = t.typelem AND el.typarray = t.oid)
  ORDER BY name
SQL

# Add types to this list if they require a more specific way to create a
# test value than can be inferred from just their 'typcategory'
#
# If the value is a string, it will be escaped (via String#inspect) and
# included in the generated code as a string literal.  If the value is wrapped
# in a single-element Array, it will be included unescaped in the generated
# code.  See function #genvalue below for reference.
CUSTOM_VALUE_TYPES = {
  'date'    => ['TEST_DATETIME.to_date'],

  # inet types
  'cidr'    => '192.168.1.0/24',
  'inet'    => '192.168.1.1/24',

  # range types
  'int4range' => '[1,5)',
  'int8range' => '[1,5)',
  'numrange' => '[1,5)',
  'daterange' => '[1837-06-20,1901-01-22)', # the reign of Queen Victoria
  'tsrange' =>   '["1837-06-20 00:00:00","1901-01-22 00:00:00")',
  'tstzrange' => '["1837-06-20 00:00:00+00","1901-01-22 00:00:00+00")',

  # geometric types
  'point' => '(3,4)',
  'line' => '{3,4,5}',
  'lseg' => '[(0,0),(3,4)]',
  'box' => '(3,4),(0,0)',
  'path' => '((1,2),(0,0),(3,4))',
  'polygon' => '((1,2),(0,0),(3,4))',
  'circle' => '<(1,2),5>',
}
def genvalue(value)
  case value
  when Array
    raise "Bad value expression: #{value.inspect}" if value.size != 1
    value.first
  else
    value.inspect
  end
end

# add types to this list if the test table needs to explicitly specify the
# length of the type
BOUNDED_LENGTH_TYPES = Set[*%w(
  bit
  character
)]

# add types to this list if they're too obscure to bother filing an issue for
# not supporting them (e.g. Postgres internals)
INTERNAL_TYPES = Set[*%w(
  abstime
  aclitem
  "char"
  cid
  ghstore
  gtsquery
  gtsvector
  int2vector
  name
  oidvector
  pg_node_tree
  refcursor
  regclass
  regconfig
  regdictionary
  regnamespace
  regoper
  regoperator
  regproc
  regprocedure
  regrole
  regtype
  reltime
  smgr
  tid
  tinterval
  txid_snapshot
  unknown
  xid
)]

# add types to this list when their lack of support is documented, preferably
# in the form of a Github issue
KNOWN_BUGS = {
  'money' => ['multiplied by 100', 'https://github.com/confluentinc/bottledwater-pg/issues/60'],
}

# only use this list during development, otherwise file an issue!
UNKNOWN_BUGS = {
}
{'fraction_non_alphanumeric': 0.14010788744714972, 'fraction_numerical': 0.04155124653739612, 'mean_word_length': 3.663494221617947, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 40, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '948624', 'n_tokens_mistral': 2690, 'n_tokens_neox': 2526, 'n_words': 731}