<commit_before><?php
class DrupalBlogPostBulkLoader extends CsvBulkLoader {
	
	/**
	 * @var integer Create BlogHolder records on a specific tree element
	 * (defaults to root node).
	 */
	protected $parentId = 0;

	protected $urlMap = array();

	public $columnMap = array(
		'nid' => 'DrupalNid', // requires the DrupalBlogEntryExtension
		'title' => 'Title',
		'body' => '->importContent',
		'changed' => 'LastEdited',
		'created' => '->importCreated',
		'tags' => '->importTags',
		'author_title' => '->importAuthor',
	);

	public $duplicateChecks = array(
		'DrupalNid' => array(
			'callback' => 'findDuplicateByNid'
		)
	);

	public function __construct($objectClass = 'BlogEntry') {
		parent::__construct($objectClass);
	}

	protected function processRecord($record, $columnMap, &$result, $preview = false) {
		// Find or create a holder for this blog
		$holder = $this->getHolder($record);
		$record['ParentID'] = $holder->ID;

		$objID = parent::processRecord($record, $columnMap, $result, $preview);
		$obj = BlogEntry::get()->byID($objID);

		$this->urlMap[$record['dst']] = $obj->RelativeLink();
	}

	/**
	 * @return BlogHolder
	 */
	protected function getHolder($record) {
		$filter = new URLSegmentFilter();
		$urlSegment = $filter->filter($record['blog_path']);
		$holder = BlogHolder::get()->filter(array(
			'URLSegment' => $urlSegment,
			'ParentID' => $this->parentId
		))->First();
		if(!$holder) {
			$holder = new BlogHolder(array(
				'Title' => $record['blog_title'],
				'URLSegment' => $urlSegment,
				'ParentID' => $this->parentId,
			));
			$holder->write();
		}
		return $holder;
	}

	/**
	 * @return String Apache rewrite rules from a previous import.
	 */
	public function getRewriteRules() {
		$rules = array();
		foreach($this->urlMap as $before => $after) {
			$rules[] = sprintf(
				"RewriteRule ^%s %s [R=301,L]",
				$before,
				$after
			);
		}
		return implode("\n", $rules);
	}

	/**
	 * @return BlogEntry
	 */
	protected function findDuplicateByNid($nid, $record) {
		return BlogEntry::get()->filter('DrupalNid', $nid)->First();
	}

	protected function importContent($obj, $val, $record) {
		$obj->Content = $this->cleanupHtml($val);
	}

	protected function importCreated($obj, $val, $record) {
		$obj->Date = $val;
		$obj->Created = $val;
	}

	protected function importAuthor($obj, $val, $record) {
		$obj->Author = $val;
	}

	protected function importTags($obj, $val, $record) {
		$tags = explode(',', $val);
		$val = array_map('trim', $tags);
		$obj->Tags = implode(', ', $tags);
	}

	/**
	 * Remove certain HTML clutter, mostly from Word copypaste.
	 */
	protected function cleanupHtml($val) {
		$val = preg_replace('/\s?style="[^"]*"/', '', $val);
		$val = preg_replace('/<font[^>]*>/', '', $val);
		$val = preg_replace('/<\/font[\s]*>/', '', $val);
		return $val;
	}

	public function setParentId($id) {
		$this->parentId = $id;
	}<commit_msg>Fixed bug in tag imports<commit_after><?php
class DrupalBlogPostBulkLoader extends CsvBulkLoader {
	
	/**
	 * @var integer Create BlogHolder records on a specific tree element
	 * (defaults to root node).
	 */
	protected $parentId = 0;

	protected $urlMap = array();

	public $columnMap = array(
		'nid' => 'DrupalNid', // requires the DrupalBlogEntryExtension
		'title' => 'Title',
		'body' => '->importContent',
		'changed' => 'LastEdited',
		'created' => '->importCreated',
		'tags' => '->importTags',
		'author_title' => '->importAuthor',
	);

	public $duplicateChecks = array(
		'DrupalNid' => array(
			'callback' => 'findDuplicateByNid'
		)
	);

	public function __construct($objectClass = 'BlogEntry') {
		parent::__construct($objectClass);
	}

	protected function processRecord($record, $columnMap, &$result, $preview = false) {
		// Find or create a holder for this blog
		$holder = $this->getHolder($record);
		$record['ParentID'] = $holder->ID;

		$objID = parent::processRecord($record, $columnMap, $result, $preview);
		$obj = BlogEntry::get()->byID($objID);

		$this->urlMap[$record['dst']] = $obj->RelativeLink();
	}

	/**
	 * @return BlogHolder
	 */
	protected function getHolder($record) {
		$filter = new URLSegmentFilter();
		$urlSegment = $filter->filter($record['blog_path']);
		$holder = BlogHolder::get()->filter(array(
			'URLSegment' => $urlSegment,
			'ParentID' => $this->parentId
		))->First();
		if(!$holder) {
			$holder = new BlogHolder(array(
				'Title' => $record['blog_title'],
				'URLSegment' => $urlSegment,
				'ParentID' => $this->parentId,
			));
			$holder->write();
		}
		return $holder;
	}

	/**
	 * @return String Apache rewrite rules from a previous import.
	 */
	public function getRewriteRules() {
		$rules = array();
		foreach($this->urlMap as $before => $after) {
			$rules[] = sprintf(
				"RewriteRule ^%s %s [R=301,L]",
				$before,
				$after
			);
		}
		return implode("\n", $rules);
	}

	/**
	 * @return BlogEntry
	 */
	protected function findDuplicateByNid($nid, $record) {
		return BlogEntry::get()->filter('DrupalNid', $nid)->First();
	}

	protected function importContent($obj, $val, $record) {
		$obj->Content = $this->cleanupHtml($val);
	}

	protected function importCreated($obj, $val, $record) {
		$obj->Date = $val;
		$obj->Created = $val;
	}

	protected function importAuthor($obj, $val, $record) {
		$obj->Author = $val;
	}

	protected function importTags($obj, $val, $record) {
		$tags = explode(',', $val);
		$tags = array_map('trim', $tags);
		$obj->Tags = implode(', ', $tags);
	}

	/**
	 * Remove certain HTML clutter, mostly from Word copypaste.
	 */
	protected function cleanupHtml($val) {
		$val = preg_replace('/\s?style="[^"]*"/', '', $val);
		$val = preg_replace('/<font[^>]*>/', '', $val);
		$val = preg_replace('/<\/font[\s]*>/', '', $val);
		return $val;
	}

	public function setParentId($id) {
		$this->parentId = $id;
	}
{'fraction_non_alphanumeric': 0.19944789510006902, 'fraction_numerical': 0.0013802622498274672, 'mean_word_length': 4.08955223880597, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3082862', 'n_tokens_mistral': 2189, 'n_tokens_neox': 1967, 'n_words': 438}