<commit_before><?php
/**
* Atarashii MAL API
*
* @author    <NAME> <<EMAIL>>
* @author    <NAME> <<EMAIL>>
* @copyright 2014 <NAME> and <NAME>
* @license   http://www.apache.org/licenses/LICENSE-2.0 Apache Public License 2.0
*/

namespace Atarashii\APIBundle\Parser;

use Symfony\Component\DomCrawler\Crawler;
use Atarashii\APIBundle\Model\Anime;
use \DateTime;

class AnimeParser
{
    public static function parse($contents, $mine = false)
    {
        $crawler = new Crawler();
        $crawler->addHTMLContent($contents, 'UTF-8');

        $animerecord = new Anime();

        # Anime ID.
        # Example:
        # <input type="hidden" name="aid" value="790">
        $animerecord->id = (int) $crawler->filter('input[name="aid"]')->attr('value');

        # Title and rank.
        # Example:
        # <h1><div style="float: right; font-size: 13px;">Ranked #96</div>Lucky ☆ Star</h1>
        $animerecord->title = str_replace($crawler->filter('h1')->children()->text(), '', $crawler->filter('h1')->text());
        $animerecord->rank = (int) str_replace('Ranked #', '', $crawler->filter('h1 div')->text());

        # Title Image
        # Example:
        # <a href="http://myanimelist.net/anime/16353/Love_Lab/pic&pid=50257"><img src="http://cdn.myanimelist.net/images/anime/12/50257.jpg" alt="Love Lab" align="center"></a>
        $animerecord->image_url = $crawler->filter('div#content tr td div img')->attr('src');

        # Alternative Titles section.
        # Example:
        # <h2>Alternative Titles</h2>
        # <div class="spaceit_pad"><span class="dark_text">English:</span> Lucky Star/div>
        # <div class="spaceit_pad"><span class="dark_text">Synonyms:</span> Lucky Star, Raki ☆ Suta</div>
        # <div class="spaceit_pad"><span class="dark_text">Japanese:</span> らき すた</div>
        $leftcolumn = $crawler->filterXPath('//div[@id="content"]/table/tr/td[@class="borderClass"]');

        # English:
        $extracted = $leftcolumn->filterXPath('//span[text()="English:"]');
        if (iterator_count($extracted) > 0) {
            $text = trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
            $animerecord->other_titles['english'] = explode(', ', $text);
        }

        # Synonyms:
        $extracted = $leftcolumn->filterXPath('//span[text()="Synonyms:"]');
        if (iterator_count($extracted) > 0) {
            $text = trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
            $animerecord->other_titles['synonyms'] = explode(', ', $text);
        }

        # Japanese:
        $extracted = $leftcolumn->filterXPath('//span[text()="Japanese:"]');
        if (iterator_count($extracted) > 0) {
            $text = trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
            $animerecord->other_titles['japanese'] = explode(', ', $text);
        }

        # Information section.
        # Example:
        # <h2>Information</h2>
        # <div><span class="dark_text">Type:</span> TV</div>
        # <div class="spaceit"><span class="dark_text">Episodes:</span> 24</div>
        # <div><span class="dark_text">Status:</span> Finished Airing</div>
        # <div class="spaceit"><span class="dark_text">Aired:</span> Apr  9, 2007 to Sep  17, 2007</div>
        # <div>
        #   <span class="dark_text">Producers:</span>
        #   <a href="http://myanimelist.net/anime.php?p=2">Kyoto Animation</a>,
        #   <a href="http://myanimelist.net/anime.php?p=104">Lantis</a>,
        #   <a href="http://myanimelist.net/anime.php?p=262">Kadokawa Pictures USA</a><sup><small>L</small></sup>,
        #   <a href="http://myanimelist.net/anime.php?p=286">Bang Zoom! Entertainment</a>
        # </div>
        # <div class="spaceit">
        #   <span class="dark_text">Genres:</span>
        #   <a href="http://myanimelist.net/anime.php?genre[]=4">Comedy</a>,
        #   <a href="http://myanimelist.net/anime.php?genre[]=20">Parody</a>,
        #   <a href="http://myanimelist.net/anime.php?genre[]=23">School</a>,
        #   <a href="http://myanimelist.net/anime.php?genre[]=36">Slice of Life</a>
        # </div>
        # <div><span class="dark_text">Duration:</span> 24 min. per episode</div>
        # <div class="spaceit"><span class="dark_text">Rating:</span> PG-13 - Teens 13 or older</div>

        # Type:
        $extracted = $leftcolumn->filterXPath('//span[text()="Type:"]');
        if (iterator_count($extracted) > 0) {
            $animerecord->type = trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
        }

        # Episodes:
        $extracted = $leftcolumn->filterXPath('//span[text()="Episodes:"]');
        if (iterator_count($extracted) > 0) {
            $animerecord->episodes = (int) trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
        } else {
            $animerecord->episodes = null;
        }

        # Status:
        $extracted = $leftcolumn->filterXPath('//span[text()="Status:"]');
        if (iterator_count($extracted) > 0) {
            $animerecord->status = strtolower(trim(str_replace($extracted->text(), '', $extracted->parents()->text())));
        }

        # Aired:
        $extracted = $leftcolumn->filterXPath('//span[text()="Aired:"]');
        if (iterator_count($extracted) > 0) {
            /*
             * NOTE: The Ruby API has a bug where yet-to-air shows that only have one date
             * get that listed as the "end_date", not the "start_date". The code below fixes
             * this and in doing so delibrately breaks compatibility in order to present the
             * data properly.
             */

            $daterange = explode(' to ', trim(str_replace($extracted->text(), '', $extracted->parents()->text())));

            //MAL always provides record dates in US-style format. We export to a non-standard format to keep compatibility with the Ruby API.
            //Sometimes the startdate doesn't contain any day. For compatibility with the Ruby API I must pass a date, dayname and time.
            if (strpos($daterange[0],',') == false) {
                $animerecord->start_date = DateTime::createFromFormat('M Y', $daterange[0])->format('D M d 00:00:00 O Y');
            } else {
                $animerecord->start_date = DateTime::createFromFormat('M j, Y', $daterange[0])->format('D M d H:i:s O Y');
            }

            //Series not yet to air won't list a range at all while currently airing series will use a "?"
            //For these, we should return a null
            if (count($daterange) < 2 || $daterange[1] == '?') {
                $animerecord->end_date = null;
            } else {
                //MAL always provides record dates in US-style format. We export to a non-standard format to keep compatibility with the Ruby API.
                $animerecord->end_date = DateTime::createFromFormat('M j, Y', $daterange[1])->format('D M d H:i:s O Y');
            }
        }

        # Genres:
        $extracted = $leftcolumn->filterXPath('//span[text()="Genres:"]');
        if (iterator_count($extracted) > 0) {
            $animerecord->genres = explode(', ', trim(str_replace($extracted->text(), '', $extracted->parents()->text())));
        }

        # Classification:<commit_msg>Fix 503 Service Unavailable if anime is 'Not available'.<commit_after><?php
/**
* Atarashii MAL API
*
* @author    <NAME> <<EMAIL>>
* @author    <NAME> <<EMAIL>>
* @copyright 2014 <NAME> and <NAME>
* @license   http://www.apache.org/licenses/LICENSE-2.0 Apache Public License 2.0
*/

namespace Atarashii\APIBundle\Parser;

use Symfony\Component\DomCrawler\Crawler;
use Atarashii\APIBundle\Model\Anime;
use \DateTime;

class AnimeParser
{
    public static function parse($contents, $mine = false)
    {
        $crawler = new Crawler();
        $crawler->addHTMLContent($contents, 'UTF-8');

        $animerecord = new Anime();

        # Anime ID.
        # Example:
        # <input type="hidden" name="aid" value="790">
        $animerecord->id = (int) $crawler->filter('input[name="aid"]')->attr('value');

        # Title and rank.
        # Example:
        # <h1><div style="float: right; font-size: 13px;">Ranked #96</div>Lucky ☆ Star</h1>
        $animerecord->title = str_replace($crawler->filter('h1')->children()->text(), '', $crawler->filter('h1')->text());
        $animerecord->rank = (int) str_replace('Ranked #', '', $crawler->filter('h1 div')->text());

        # Title Image
        # Example:
        # <a href="http://myanimelist.net/anime/16353/Love_Lab/pic&pid=50257"><img src="http://cdn.myanimelist.net/images/anime/12/50257.jpg" alt="Love Lab" align="center"></a>
        $animerecord->image_url = $crawler->filter('div#content tr td div img')->attr('src');

        # Alternative Titles section.
        # Example:
        # <h2>Alternative Titles</h2>
        # <div class="spaceit_pad"><span class="dark_text">English:</span> Lucky Star/div>
        # <div class="spaceit_pad"><span class="dark_text">Synonyms:</span> Lucky Star, Raki ☆ Suta</div>
        # <div class="spaceit_pad"><span class="dark_text">Japanese:</span> らき すた</div>
        $leftcolumn = $crawler->filterXPath('//div[@id="content"]/table/tr/td[@class="borderClass"]');

        # English:
        $extracted = $leftcolumn->filterXPath('//span[text()="English:"]');
        if (iterator_count($extracted) > 0) {
            $text = trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
            $animerecord->other_titles['english'] = explode(', ', $text);
        }

        # Synonyms:
        $extracted = $leftcolumn->filterXPath('//span[text()="Synonyms:"]');
        if (iterator_count($extracted) > 0) {
            $text = trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
            $animerecord->other_titles['synonyms'] = explode(', ', $text);
        }

        # Japanese:
        $extracted = $leftcolumn->filterXPath('//span[text()="Japanese:"]');
        if (iterator_count($extracted) > 0) {
            $text = trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
            $animerecord->other_titles['japanese'] = explode(', ', $text);
        }

        # Information section.
        # Example:
        # <h2>Information</h2>
        # <div><span class="dark_text">Type:</span> TV</div>
        # <div class="spaceit"><span class="dark_text">Episodes:</span> 24</div>
        # <div><span class="dark_text">Status:</span> Finished Airing</div>
        # <div class="spaceit"><span class="dark_text">Aired:</span> Apr  9, 2007 to Sep  17, 2007</div>
        # <div>
        #   <span class="dark_text">Producers:</span>
        #   <a href="http://myanimelist.net/anime.php?p=2">Kyoto Animation</a>,
        #   <a href="http://myanimelist.net/anime.php?p=104">Lantis</a>,
        #   <a href="http://myanimelist.net/anime.php?p=262">Kadokawa Pictures USA</a><sup><small>L</small></sup>,
        #   <a href="http://myanimelist.net/anime.php?p=286">Bang Zoom! Entertainment</a>
        # </div>
        # <div class="spaceit">
        #   <span class="dark_text">Genres:</span>
        #   <a href="http://myanimelist.net/anime.php?genre[]=4">Comedy</a>,
        #   <a href="http://myanimelist.net/anime.php?genre[]=20">Parody</a>,
        #   <a href="http://myanimelist.net/anime.php?genre[]=23">School</a>,
        #   <a href="http://myanimelist.net/anime.php?genre[]=36">Slice of Life</a>
        # </div>
        # <div><span class="dark_text">Duration:</span> 24 min. per episode</div>
        # <div class="spaceit"><span class="dark_text">Rating:</span> PG-13 - Teens 13 or older</div>

        # Type:
        $extracted = $leftcolumn->filterXPath('//span[text()="Type:"]');
        if (iterator_count($extracted) > 0) {
            $animerecord->type = trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
        }

        # Episodes:
        $extracted = $leftcolumn->filterXPath('//span[text()="Episodes:"]');
        if (iterator_count($extracted) > 0) {
            $animerecord->episodes = (int) trim(str_replace($extracted->text(), '', $extracted->parents()->text()));
        } else {
            $animerecord->episodes = null;
        }

        # Status:
        $extracted = $leftcolumn->filterXPath('//span[text()="Status:"]');
        if (iterator_count($extracted) > 0) {
            $animerecord->status = strtolower(trim(str_replace($extracted->text(), '', $extracted->parents()->text())));
        }

        # Aired:
        $extracted = $leftcolumn->filterXPath('//span[text()="Aired:"]');
        if (iterator_count($extracted) > 0) {
            /*
             * NOTE: The Ruby API has a bug where yet-to-air shows that only have one date
             * get that listed as the "end_date", not the "start_date". The code below fixes
             * this and in doing so delibrately breaks compatibility in order to present the
             * data properly.
             */

            $daterange = explode(' to ', trim(str_replace($extracted->text(), '', $extracted->parents()->text())));

            //MAL always provides record dates in US-style format. We export to a non-standard format to keep compatibility with the Ruby API.
            //Sometimes the startdate doesn't contain any day. For compatibility with the Ruby API I must pass a date, dayname and time.
            if (strpos($daterange[0],',') == false) {
                if ($daterange[0] !== 'Not available') {
                    $animerecord->start_date = DateTime::createFromFormat('M Y', $daterange[0])->format('D M d 00:00:00 O Y');
                }
            } else {
                $animerecord->start_date = DateTime::createFromFormat('M j, Y', $daterange[0])->format('D M d H:i:s O Y');
            }

            //Series not yet to air won't list a range at all while currently airing series will use a "?"
            //For these, we should return a null
            if (count($daterange) < 2 || $daterange[1] == '?') {
                $animerecord->end_date = null;
            } else {
                //MAL always provides record dates in US-style format. We export to a non-standard format to keep compatibility with the Ruby API.
                $animerecord->end_date = DateTime::createFromFormat('M j, Y', $daterange[1])->format('D M d H:i:s O Y');
            }
        }

        # Genres:
        $extracted = $leftcolumn->filterXPath('//span[text()="Genres:"]');
        if (iterator_count($extracted) > 0) {
            $animerecord->genres = explode(', ', trim(str_replace($extracted->text(), '', $extracted->parents()->text())));
        }

        # Classification:
{'fraction_non_alphanumeric': 0.18635188461015104, 'fraction_numerical': 0.013607293509320996, 'mean_word_length': 2.716561314791403, 'pattern_counts': {'":': 0, '<': 173, '<?xml version=': 0, '>': 315, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4432829', 'n_tokens_mistral': 4843, 'n_tokens_neox': 4626, 'n_words': 905}