<commit_before><?php
/**
 * A service that uses the AlchemyAPI to retrieve information about content
 *
 * @author <NAME> <<EMAIL>>
 */
class AlchemyService {

	/**
	 * How many characters should content have before it is indexed?
	 *
	 * @var int
	 */
	public static $char_limit = 80;

	/**
	 * How many keywords should we use?
	 *
	 * @var int
	 */
	public static $max_keywords = 15;

	public static $config = array(
		'api_url'  => 'http://access.alchemyapi.com',
		'api_path' => '/calls/text/',
		'api_key'  => '',
	);

	public static function set_api_key($key) {
		self::$config['api_key'] = $key;
	}

	/**
	 * @var RestfulService
	 */
	protected $api;

	public function __construct() {
		$this->api = new RestfulService(Controller::join_links(
			self::$config['api_url'], self::$config['api_path']
		));
	}

	/**
	 * Updates a content item with data from the Alchemy API.
	 *
	 * @param DataObject $object
	 */
	public function alchemise(DataObject $object) {
		$content = $object->getAlchemyContent();

		if (strlen($content) < self::$char_limit) {
			return;
		}

		$this->api->setQueryString(array(
			'apikey' => self::$config['api_key'],
			'text'   => $content
		));

		$entities = $this->api->request('TextGetRankedNamedEntities');

		if (!$entities->isError()) {
			// First clear all existing entity data.
			foreach (array_keys(Alchemisable::entity_fields()) as $field) {
				$object->$field = array();
			}

			foreach ($entities->simpleXML()->entities->entity as $entity) {
				$name      = "Alc{$entity->type}";
				$relevance = $entity->relevance;

				if ($relevance > .3) {
					$values = $object->$name->getValues();
					$values[] = (string) $entity->text;
					$object->$name = $values;
				}
			}
		}

		$keywords = $this->api->request('TextGetRankedKeywords');
		$total    = 0;

		if (!$keywords->isError()) {
			$extracted = array();

			foreach ($keywords->simpleXML()->keywords->keyword as $keyword) {
				if ($total++ < self::$max_keywords) {
					$extracted[] = (string) $keyword->text;
				}
			}

			$object->AlcKeywords = $extracted;
		}

		$category = $this->api->request('TextGetCategory');
		if (!$category->isError()) {
			$object->AlcCategory = (string) $category->simpleXML()->category;
		}
	}

}
<commit_msg>Broke AlchemyService up into several methods for extracting a category, keywords or entities.<commit_after><?php
/**
 * A service that uses the AlchemyAPI to retrieve information about content
 *
 * @author <NAME> <<EMAIL>>
 */
class AlchemyService {

	/**
	 * How many keywords should we use?
	 *
	 * @var int
	 */
	public static $max_keywords = 15;

	public static $config = array(
		'api_url'  => 'http://access.alchemyapi.com',
		'api_path' => '/calls/text/',
		'api_key'  => '',
	);

	public static function set_api_key($key) {
		self::$config['api_key'] = $key;
	}

	/**
	 * @var RestfulService
	 */
	protected $api;

	public function __construct() {
		$this->api = new RestfulService(Controller::join_links(
			self::$config['api_url'], self::$config['api_path']
		));
	}

	/**
	 * @return string
	 */
	public function getCategoryFor($text) {
		$this->api->setQueryString(array(
			'apikey' => self::$config['api_key'],
			'text'   => $text
		));

		$category = $this->api->request('TextGetCategory');

		if (!$category->isError()) {
			return (string) $category->simpleXML()->category;
		}
	}

	/**
	 * @return array[]
	 */
	public function getEntitiesFor($text) {
		$this->api->setQueryString(array(
			'apikey' => self::$config['api_key'],
			'text'   => $text
		));

		$request = $this->api->request('TextGetRankedNamedEntities');

		if (!$request->isError()) {
			$entities = array();

			foreach ($request->simpleXML()->entities->entity as $entity) {
				if ($entity->relevance > .3) {
					$type = (string) $entity->type;
					$text = (string) $entity->text;

					if (!array_key_exists($type, $entities)) {
						$entities[$type] = array();
					}

					$entities[$type][] = $text;
				}
			}

			return $entities;
		} else {
			return array();
		}
	}

	/**
	 * @return array
	 */
	public function getKeywordsFor($text) {
		$this->api->setQueryString(array(
			'apikey' => self::$config['api_key'],
			'text'   => $text
		));

		$keywords = $this->api->request('TextGetRankedKeywords');
		$total    = 0;

		if (!$keywords->isError()) {
			$extracted = array();

			foreach ($keywords->simpleXML()->keywords->keyword as $keyword) {
				if ($total++ < self::$max_keywords) {
					$extracted[] = (string) $keyword->text;
				}
			}

			return $extracted;
		} else {
			return array();
		}
	}

}

{'fraction_non_alphanumeric': 0.18933742869679684, 'fraction_numerical': 0.002193944712593243, 'mean_word_length': 3.409090909090909, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 84, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '905617', 'n_tokens_mistral': 1790, 'n_tokens_neox': 1540, 'n_words': 320}