<commit_before>package edu.mit.cci.wikilanguage.wiki

import edu.mit.cci.wiki.ArticleCache
import edu.mit.cci.wikilanguage.model.active.WikiCategory
import edu.mit.cci.wikilanguage.db.DAO
import edu.mit.cci.wikilanguage.model.Conversions._
import edu.mit.cci.util.U
import java.util.concurrent.{Executors, Callable, ExecutorService}
import scala.collection.JavaConversions.asScalaSet
import scala.collection.mutable
import java.util.Collections
import edu.mit.cci.wikilanguage.model.{Person, Category}

/**
 * User: pdeboer
 * Date: 10/13/13
 * Time: 10:17 AM
 */
class CategoryProcessor(val lang: String = "en") {
	private val processed: mutable.Set[String] = Collections.synchronizedSet(new java.util.HashSet[String]())
	private val exec = Executors.newFixedThreadPool(50)

	def shutdown() {
		exec.shutdown()
	}

	def setProcessed(category: Category) {
		processed += category.name
	}

	def isProcessed(category: Category): Boolean = synchronized {
		return processed.contains(category.name)
	}

	def addToQueue(elements: List[Category]) {
		elements.foreach(addToQueue(_))
	}

	def addToQueue(element: Category) {
		if (!isProcessed(element)) {
			processed += element.name
			exec.execute(new Worker(element))
		}
	}

	def process(name: String) {
		addToQueue(Category(name, lang)())
	}

	private class Worker(category: Category) extends Runnable {
		def run() {
			val categoriesAndPeople = new CategoryContentProcessor(category).call()
			//put found categories in queue
			addToQueue(categoriesAndPeople.categories.toList)

			//insert people
			val lifetimeAnnotator: PersonLifetimeAnnotator = new PersonLifetimeAnnotator()
			categoriesAndPeople.people.foreach(p => {
				if (DAO.personByName(p.name) == null) {
					val a = ArticleCache.get(p.name, p.lang)
					a.textFetched //fetch content of said article to ease further processing

					val personConverted: Person = a
					DAO.synchronized {
						val person = DAO.insertPerson(a) //implicit conversion allows for fetching of categories
						if (person > -1) {
							personConverted.id = person
							DAO.insertPersonMeta(personConverted, resolveCategories = true, resolveRedirects = true)
							lifetimeAnnotator.processPerson(person)
						}
					}
				}
			})
		}
	}

}

class CategoryContentProcessor(cat: Category, insertDB: Boolean = true) extends Callable[CategoriesAndPeople] {
	//automatically retry content-fetching
	def categoryContents(maxTries: Int = 5): Array[String] = {
		if (maxTries == 0) return Array.empty[String]

		val wiki = new WikiCategory(cat.name, lang = cat.lang)
		if (wiki.contents.size == 0) categoryContents(maxTries - 1) else wiki.contents
	}

	def call(): CategoriesAndPeople = {
		if (insertDB) DAO.insertCategory(cat)
<commit_msg>catch possible exception<commit_after>package edu.mit.cci.wikilanguage.wiki

import edu.mit.cci.wiki.ArticleCache
import edu.mit.cci.wikilanguage.model.active.WikiCategory
import edu.mit.cci.wikilanguage.db.DAO
import edu.mit.cci.wikilanguage.model.Conversions._
import edu.mit.cci.util.U
import java.util.concurrent.{Executors, Callable, ExecutorService}
import scala.collection.JavaConversions.asScalaSet
import scala.collection.mutable
import java.util.Collections
import edu.mit.cci.wikilanguage.model.{Person, Category}

/**
 * User: pdeboer
 * Date: 10/13/13
 * Time: 10:17 AM
 */
class CategoryProcessor(val lang: String = "en") {
	private val processed: mutable.Set[String] = Collections.synchronizedSet(new java.util.HashSet[String]())
	private val exec = Executors.newFixedThreadPool(50)

	def shutdown() {
		exec.shutdown()
	}

	def setProcessed(category: Category) {
		processed += category.name
	}

	def isProcessed(category: Category): Boolean = synchronized {
		return processed.contains(category.name)
	}

	def addToQueue(elements: List[Category]) {
		elements.foreach(addToQueue(_))
	}

	def addToQueue(element: Category) {
		if (!isProcessed(element)) {
			processed += element.name
			exec.execute(new Worker(element))
		}
	}

	def process(name: String) {
		addToQueue(Category(name, lang)())
	}

	private class Worker(category: Category) extends Runnable {
		def run() {
			val categoriesAndPeople = new CategoryContentProcessor(category).call()
			//put found categories in queue
			addToQueue(categoriesAndPeople.categories.toList)

			//insert people
			val lifetimeAnnotator: PersonLifetimeAnnotator = new PersonLifetimeAnnotator()
			categoriesAndPeople.people.foreach(p => {
				try {
					if (DAO.personByName(p.name) == null) {
						val a = ArticleCache.get(p.name, p.lang)
						a.textFetched //fetch content of said article to ease further processing

						val personConverted: Person = a
						DAO.synchronized {
							val person = DAO.insertPerson(a) //implicit conversion allows for fetching of categories
							if (person > -1) {
								personConverted.id = person
								DAO.insertPersonMeta(personConverted, resolveCategories = true, resolveRedirects = true)
								lifetimeAnnotator.processPerson(person)
							}
						}
					}
				}
				catch {
					case t:Throwable => t.printStackTrace(System.err)
				}
			})
		}
	}

}

class CategoryContentProcessor(cat: Category, insertDB: Boolean = true) extends Callable[CategoriesAndPeople] {
	//automatically retry content-fetching
	def categoryContents(maxTries: Int = 5): Array[String] = {
		if (maxTries == 0) return Array.empty[String]

		val wiki = new WikiCategory(cat.name, lang = cat.lang)
		if (wiki.contents.size == 0) categoryContents(maxTries - 1) else wiki.contents
	}

	def call(): CategoriesAndPeople = {
		if (insertDB) DAO.insertCategory(cat)

{'fraction_non_alphanumeric': 0.1108325872873769, 'fraction_numerical': 0.006087735004476276, 'mean_word_length': 4.20596458527493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2015855', 'n_tokens_mistral': 2030, 'n_tokens_neox': 1745, 'n_words': 385}