<commit_before>#!/usr/bin/env ruby

#
# Script to recognise text using Tesseract v3
#
# Usage:
# recognise.rb /path/to/input/dir \
#              /path/to/output/dir \
#              /path/to/tesseract/bin
#

require 'json'
require 'open3'
require 'fileutils'
require 'matrix'
require 'rmagick'

def process_line(file, line)
  re = /^([^\s]+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s\d+$/
  matches = re.match(line)
  return nil if matches.nil?
  char = matches[1]
  # NOTE: Coordinate system of tesseract has (0,0) in BOTTOM-LEFT not TOP-LEFT!
  # So we convert (x,y) -> (x,h-y)
  x1 = matches[2].to_i
  y1 = matches[3].to_i
  x2 = matches[4].to_i
  y2 = matches[5].to_i
  h = Magick::Image.ping(file).first.rows
  y2 = (h - y1)
  y1 = (h - y2)
  return {
    char: char,
    x1: x1,
    y1: y1,
    x2: x2,
    y2: y2,
    width: x2 - x1,
    height: y2 - y1
  }
end

def proc_files(in_dir, out_dir, tesseract_dir)
  # Pass this into stdin for darknet (i.e., all files we want to test)
  char_regions = {}
  Dir["#{in_dir}/*.jpg"].each do |file|
    unique_image_id = File.basename(file, '.jpg')
    char_regions[unique_image_id] = {
      char: { regions: [], string: '' }
    }
    cmd = %W(
      ./tesseract
      "#{file}"
      stdout
      quiet
      makebox
      --psm 8
    ).join(' ')
    puts "Running tesseract on #{file}..."
    start = Time.now
    Open3.popen2e(cmd, chdir: tesseract_dir) do |stdin, stdoe|
      stdin.close
      while line = stdoe.gets
        puts line
        data = process_line(file, line)
        next if data.nil?
        char_regions[unique_image_id][:char][:regions] << data
        char_regions[unique_image_id][:char][:string] << data[:char]
      end<commit_msg>FIX: Fix psm switch to single dash for tesseract 3<commit_after>#!/usr/bin/env ruby

#
# Script to recognise text using Tesseract v3
#
# Usage:
# recognise.rb /path/to/input/dir \
#              /path/to/output/dir \
#              /path/to/tesseract/bin
#

require 'json'
require 'open3'
require 'fileutils'
require 'matrix'
require 'rmagick'

def process_line(file, line)
  re = /^([^\s]+)\s(\d+)\s(\d+)\s(\d+)\s(\d+)\s\d+$/
  matches = re.match(line)
  return nil if matches.nil?
  char = matches[1]
  # NOTE: Coordinate system of tesseract has (0,0) in BOTTOM-LEFT not TOP-LEFT!
  # So we convert (x,y) -> (x,h-y)
  x1 = matches[2].to_i
  y1 = matches[3].to_i
  x2 = matches[4].to_i
  y2 = matches[5].to_i
  h = Magick::Image.ping(file).first.rows
  y2 = (h - y1)
  y1 = (h - y2)
  return {
    char: char,
    x1: x1,
    y1: y1,
    x2: x2,
    y2: y2,
    width: x2 - x1,
    height: y2 - y1
  }
end

def proc_files(in_dir, out_dir, tesseract_dir)
  # Pass this into stdin for darknet (i.e., all files we want to test)
  char_regions = {}
  Dir["#{in_dir}/*.jpg"].each do |file|
    unique_image_id = File.basename(file, '.jpg')
    char_regions[unique_image_id] = {
      char: { regions: [], string: '' }
    }
    cmd = %W(
      ./tesseract
      -psm 8
      "#{file}"
      stdout
      quiet
      makebox
    ).join(' ')
    puts "Running tesseract on #{file}..."
    start = Time.now
    Open3.popen2e(cmd, chdir: tesseract_dir) do |stdin, stdoe|
      stdin.close
      while line = stdoe.gets
        puts line
        data = process_line(file, line)
        next if data.nil?
        char_regions[unique_image_id][:char][:regions] << data
        char_regions[unique_image_id][:char][:string] << data[:char]
      end
{'fraction_non_alphanumeric': 0.1589057043073341, 'fraction_numerical': 0.01891734575087311, 'mean_word_length': 2.4404404404404403, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1915021', 'n_tokens_mistral': 1393, 'n_tokens_neox': 1384, 'n_words': 318}