<commit_before>require './lib/tabula'

input_filename = "vertical_rulings_bug.pdf"
out = File.new("output.xls", 'w')

extractor = Tabula::Extraction::CharacterExtractor.new(input_filename, :all) #:all ) # 1..2643
extractor.extract.each_with_index do |pdf_page, page_index|

  lines = Tabula::Ruling::clean_rulings(Tabula::LSD::detect_lines_in_pdf_page(input_filename, page_index))
  page_areas = [[0, 0, 1000, 1700]]

  scale_factor = pdf_page.width / 1700
  puts scale_factor

  vertical_rulings = [0, 360, 506, 617, 906, 700, 1034, 1160, 1290, 1418, 1548].map{|n| Geometry::Segment.new_by_arrays([n * scale_factor, 0], [n * scale_factor, 1000])}

  page_areas.each do |page_area|
    text = pdf_page.get_text( page_area ) #all the characters within the given area.

    Tabula::Writers.send(:TSV,
                         Tabula.make_table_with_vertical_rulings(text, {:vertical_rulings => vertical_rulings, :merge_words => true, :dontmerge => true}), 
                         out)
  end
end
out.close


#with dontmerge false (i.e. if we merge) we get crap. STCITY and no spaces in any cities.
#with dontmerge true (or commented out), MORGANTOWNWV, and some spaces (e.g. BRYN MAWR, but not FRESHMEADOWS)<commit_msg>fix part of the test script<commit_after>require './lib/tabula'

input_filename = "vertical_rulings_bug.pdf"
out = File.new("output.xls", 'w')

extractor = Tabula::Extraction::CharacterExtractor.new(input_filename, :all) #:all ) # 1..2643
extractor.extract.each_with_index do |pdf_page, page_index|

  lines = Tabula::Ruling::clean_rulings(Tabula::LSD::detect_lines_in_pdf_page(input_filename, page_index))
  page_areas = [[0, 0, 1000, 1700]]

  scale_factor = pdf_page.width / 1700
  puts scale_factor

  vertical_rulings = [0, 360, 506, 617, 906, 1034, 1160, 1290, 1418, 1548].map{|n| Geometry::Segment.new_by_arrays([n * scale_factor, 0], [n * scale_factor, 1000])}

  page_areas.each do |page_area|
    text = pdf_page.get_text( page_area ) #all the characters within the given area.

    Tabula::Writers.send(:TSV,
                         Tabula.make_table_with_vertical_rulings(text, {:vertical_rulings => vertical_rulings, :merge_words => true, :dontmerge => true}), 
                         out)
  end
end
out.close


#with dontmerge false (i.e. if we merge) we get crap. STCITY and no spaces in any cities.
#with dontmerge true (or commented out), MORGANTOWNWV, and some spaces (e.g. BRYN MAWR, but not FRESHMEADOWS)
{'fraction_non_alphanumeric': 0.12392638036809817, 'fraction_numerical': 0.04785276073619632, 'mean_word_length': 4.2602150537634405, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4857850', 'n_tokens_mistral': 989, 'n_tokens_neox': 858, 'n_words': 213}