<commit_before>#!/usr/bin/env python3
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This tool checks that every create (table|view) is prefiexed by
# drop (table|view).

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import re
import sys

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def check(path):
  with open(path) as f:
    lines = [l.strip() for l in f.readlines()]

  errors = 0
  d_type, d_name = None, None
  for line in lines:
    m = re.match(r'^DROP (TABLE|VIEW) IF EXISTS (.*);$', line)
    if m is not None:
      d_type, d_name = m.group(1), m.group(2)
      continue
    m = re.match(r'^CREATE (?:VIRTUAL )?(TABLE|VIEW) (.*) (?:AS|USING).*', line)
    if m is None:
      continue
    type, name = m.group(1), m.group(2)
    if type != d_type or name != d_name:
      sys.stderr.write(('%s:\n  "%s" vs %s %s') % (path, line, d_type, d_name))
      errors += 1
    d_type, d_name = None, None
  return errors


def main():
  errors = 0
  metrics_sources = os.path.join(ROOT_DIR, 'src', 'trace_processor', 'metrics')
  for root, _, files in os.walk(metrics_sources, topdown=True):
    for f in files:
      path = os.path.join(root, f)
      if path.endswith('.sql'):
        errors += check(path)<commit_msg>tp: Add missing new line to check_sql_metrics.py output<commit_after>#!/usr/bin/env python3
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This tool checks that every create (table|view) is prefixed by
# drop (table|view).

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import re
import sys

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))


def check(path):
  with open(path) as f:
    lines = [l.strip() for l in f.readlines()]

  errors = 0
  d_type, d_name = None, None
  for line in lines:
    m = re.match(r'^DROP (TABLE|VIEW) IF EXISTS (.*);$', line)
    if m is not None:
      d_type, d_name = m.group(1), m.group(2)
      continue
    m = re.match(r'^CREATE (?:VIRTUAL )?(TABLE|VIEW) (.*) (?:AS|USING).*', line)
    if m is None:
      continue
    type, name = m.group(1), m.group(2)
    if type != d_type or name != d_name:
      sys.stderr.write(('%s:\n  "%s" vs %s %s\n') % (path, line, d_type, d_name))
      errors += 1
    d_type, d_name = None, None
  return errors


def main():
  errors = 0
  metrics_sources = os.path.join(ROOT_DIR, 'src', 'trace_processor', 'metrics')
  for root, _, files in os.walk(metrics_sources, topdown=True):
    for f in files:
      path = os.path.join(root, f)
      if path.endswith('.sql'):
        errors += check(path)
{'fraction_non_alphanumeric': 0.12115637319316688, 'fraction_numerical': 0.008409986859395532, 'mean_word_length': 3.5201900237529693, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4302914', 'n_tokens_mistral': 1263, 'n_tokens_neox': 1232, 'n_words': 450}