<commit_before>#!/usr/bin/env ruby
# process arxiv downloaded pdf files

require_relative '../lib/pdf_interface'
require_relative '../lib/mongo_interface'
require 'logger'
require 'optparse'

options = {}
option_parse = OptionParser.new do |opts|
  executable_name = File.basename($PROGRAM_NAME)
  opts.banner = <<-EOF
Extract papers from downloaded arxiv pdfs and send them to mongo

Usage: #{executable_name} [options]

EOF

  #directory
  opts.on("-d", "--directory DIRECTORY", 'indicate main pdf directory') do |dir|
    options[:directory] = dir
  end

  #collection name
  options[:collection] = "arxiv" #default name
  opts.on("-c", "--collection COLLECTION", 'indicate mongo collection name') do |coll|
    options[:collection] = coll
  end
end

option_parse.parse!

MAIN_DIR = options[:directory]
COLLECTION_NAME = options[:collection]

LOG = Logger.new($stderr)
LOG.level = Logger.const_get(ENV.fetch("LOG_LEVEL", "INFO"))

def delete_dots list
  list.delete(".")
  list.delete("..")
  list
end

def get_directories directory
  dir_list = delete_dots Dir.entries(directory)
  #dir_list.select{ |dir| Dir.exist?(dir)}
end

def get_files folder
  delete_dots Dir.entries(folder)
end

def process
  mongo = MongoInterface.new("arxiv", COLLECTION_NAME)
  pdf_directories = get_directories MAIN_DIR
  pdf_directories.each do |dir|
    path = MAIN_DIR + dir + "/"
    pdf_files = get_files path
    pdf_files.each do |file|
      pdf = PDFInterface.new(path + file)
      pdf.process
      posts = [{filename: file, #pdf.filename includes path
                title: pdf.title,
                authors: pdf.authors,
                abstract: pdf.abstract,
                body: pdf.body}]<commit_msg>fix bug, limit posts max size<commit_after>#!/usr/bin/env ruby
# process arxiv downloaded pdf files

require_relative '../lib/pdf_interface'
require_relative '../lib/mongo_interface'
require 'logger'
require 'optparse'

options = {}
option_parse = OptionParser.new do |opts|
  executable_name = File.basename($PROGRAM_NAME)
  opts.banner = <<-EOF
Extract papers from downloaded arxiv pdfs and send them to mongo

Usage: #{executable_name} [options]

EOF

  #directory
  opts.on("-d", "--directory DIRECTORY", 'indicate main pdf directory') do |dir|
    options[:directory] = dir
  end

  #collection name
  options[:collection] = "arxiv" #default name
  opts.on("-c", "--collection COLLECTION", 'indicate mongo collection name') do |coll|
    options[:collection] = coll
  end
end

option_parse.parse!

MAIN_DIR = options[:directory]
COLLECTION_NAME = options[:collection]

LOG = Logger.new($stderr)
LOG.level = Logger.const_get(ENV.fetch("LOG_LEVEL", "INFO"))

def delete_dots list
  list.delete(".")
  list.delete("..")
  list
end

def get_directories directory
  dir_list = delete_dots Dir.entries(directory)
  #dir_list.select{ |dir| Dir.exist?(dir)}
end

def get_files folder
  delete_dots Dir.entries(folder)
end

def process
  mongo = MongoInterface.new("arxiv", COLLECTION_NAME)
  pdf_directories = get_directories MAIN_DIR
  pdf_directories.each do |dir|
    path = MAIN_DIR + dir + "/"
    pdf_files = get_files path
    pdf_files.each do |file|
      pdf = PDFInterface.new(path + file)
      pdf.process
      posts = [{filename: file, #pdf.filename includes path
                title: pdf.title,
                authors: pdf.authors,
                abstract: pdf.abstract,
                body: pdf.body}]
      posts_size= posts.map{ |_, value| value.size }.reduce(:+)
      if posts_size >= MongoInterface::MaxBSONSize
        max_size = MongoInterface::MaxBSONSize - post_size - pdf.body.size - 1
        posts[:body] = pdf.body[0..max_size]
{'fraction_non_alphanumeric': 0.11984711984711985, 'fraction_numerical': 0.000546000546000546, 'mean_word_length': 3.6088050314465407, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3250736', 'n_tokens_mistral': 1189, 'n_tokens_neox': 1150, 'n_words': 303}