<commit_before><?php

namespace Craigslist;

use \Craigslist\CraigslistRequest;
use Sunra\PhpSimple\HtmlDomParser;

class CraigslistApi
{
    public $remove_duplicates;
    protected $ids = [];
    protected $titles = [];

    /**
     * Does Craigslist RSS search request
     * Can handle array of Request and remove duplicate responses
     * @param mixed array or \Aesmith\Craigslist\Request $request
     * @param boolean $remove_duplicates
     * @throws \Exception
     * @return \Aesmith\Craigslist\Response
     */
    public function get( $request, $remove_duplicates = true )
    {
        if ($request instanceof CraigslistRequest) {
            return $this->getRSS($request);
        } else if (is_array($request)) {
            $results = [];
            foreach ($request as $req) {
                $results = array_merge($results, $this->getRSS($req));
            }
            return $results;
        } else {
            throw new \Exception('Request failed, valid data not given.');
        }
    }

    // @todo - Break this out into much more manageable pieces
    private function getRSS( CraigslistRequest $request )
    {
        $body = file_get_contents($request->url());
        $listings = simplexml_load_string(utf8_encode($body));
            
        $results = [];
        foreach ($listings as $item) {	    
            $id = substr($item->link, -15, -5);

            if (!is_numeric($id)) continue;

            if ($this->remove_duplicates) {
                if (in_array($id, $this->ids) || 
                    in_array((string) $item->title, $this->titles)) {
                    continue;
                }
                $this->ids[] = $id;
                $this->titles[] = (string) $item->title;
            }

	        $children = $item->children("dc", true);

            $results[$id] = [
                'id' => $id,
                'link' => (string) $item->link,
                'title' => (string) $item->title,<commit_msg>Fix formatting<commit_after><?php

namespace Craigslist;

use \Craigslist\CraigslistRequest;
use Sunra\PhpSimple\HtmlDomParser;

class CraigslistApi
{
    public $remove_duplicates;
    protected $ids = [];
    protected $titles = [];

    /**
     * Does Craigslist RSS search request
     * Can handle array of Request and remove duplicate responses
     * @param mixed array or \Aesmith\Craigslist\Request $request
     * @param boolean $remove_duplicates
     * @throws \Exception
     * @return \Aesmith\Craigslist\Response
     */
    public function get( $request, $remove_duplicates = true )
    {
        if ($request instanceof CraigslistRequest) {
            return $this->getRSS($request);
        } else if (is_array($request)) {
            $results = [];
            foreach ($request as $req) {
                $results = array_merge($results, $this->getRSS($req));
            }
            return $results;
        } else {
            throw new \Exception('Request failed, valid data not given.');
        }
    }

    // @todo - Break this out into much more manageable pieces
    private function getRSS( CraigslistRequest $request )
    {
        $body = file_get_contents($request->url());
        $listings = simplexml_load_string(utf8_encode($body));
            
        $results = [];
        foreach ($listings as $item) {	    
            $id = substr($item->link, -15, -5);

            if (!is_numeric($id)) continue;

            if ($this->remove_duplicates) {
                if (in_array($id, $this->ids) || 
                    in_array((string) $item->title, $this->titles)) {
                    continue;
                }
                $this->ids[] = $id;
                $this->titles[] = (string) $item->title;
            }
			
			$children = $item->children("dc", true);

            $results[$id] = [
                'id' => $id,
                'link' => (string) $item->link,
                'title' => (string) $item->title,
{'fraction_non_alphanumeric': 0.1325910931174089, 'fraction_numerical': 0.0020242914979757085, 'mean_word_length': 1.8582791033984092, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4854062', 'n_tokens_mistral': 1165, 'n_tokens_neox': 1118, 'n_words': 264}