<commit_before><?php

include(__DIR__ . '/webdata/init.inc.php');

class Crawler
{
    protected function searchDom($doms, $attr, $class)
    {
        foreach ($doms as $dom) {
            if (in_array($class, explode(' ', $dom->getAttribute($attr)))) {
                return $dom;
            }
        }
        return null;
    }

    public function getFromCNAFacebookPage($timestamp)
    {
        $access_token = getenv('FB_ACCESSTOKEN');

        $until = $timestamp + 86400;
        $url = 'https://graph.facebook.com/148395741852581/feed?limit=50&until=' . $until . '&access_token=' . urlencode($access_token) . '&fields=message,link,picture';
        $curl = curl_init($url);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
        $json = json_decode(curl_exec($curl));

        $ret = new StdClass;
        $paper_map = array(
            '中國時報' => '中國時報',
            '中時' => '中國時報',
            '聯合報' => '聯合報',
            '聯合' => '聯合報',
            '自由時報' => '自由時報',
            '自由' => '自由時報',
            '蘋果日報' => '蘋果日報',
            '蘋果' => '蘋果日報',
        );
        foreach ($json->data as $post) {
            if (false === strpos($post->message, '頭條')) {
                continue;
            }
            if (false === strpos($post->message, '蘋果')) {
                continue;
            }
            if (strtotime($post->created_time) < $timestamp) {
                break;
            }
            $message = $post->message;
            $message = str_replace('【小編報頭條】中時', '【小編報頭條】' . "\n" . '中時', $message);
            $message = preg_replace('#http[^\s]*#', '', $message);
            $message = str_replace('★', '', $message);
            $lines = explode("\n", $message);
            $ret->title = $lines[0];
            $ret->link = $post->link;
            $ret->image_link = $post->picture;
            $ret->time = $timestamp;
            $ret->headlines = array();
            foreach ($lines as $line) {
                $line = str_replace('：', ':', $line);
                if (FALSE === strpos($line, ':')) {
                    continue;
                }
                list($paper, $title) = explode(':', trim($line), 2);
                $paper = str_replace(' ', '', $paper);
                if (!array_key_exists($paper, $paper_map)) {
                    continue;
                }
                $ret->headlines[] = array($paper_map[$paper], $title);

            }

            if (count(array_unique(array_map(function($a){ return $a[0]; }, $ret->headlines))) == 4) {
                return $ret;<commit_msg>加上可能會有 ＊<commit_after><?php

include(__DIR__ . '/webdata/init.inc.php');

class Crawler
{
    protected function searchDom($doms, $attr, $class)
    {
        foreach ($doms as $dom) {
            if (in_array($class, explode(' ', $dom->getAttribute($attr)))) {
                return $dom;
            }
        }
        return null;
    }

    public function getFromCNAFacebookPage($timestamp)
    {
        $access_token = getenv('FB_ACCESSTOKEN');

        $until = $timestamp + 86400;
        $url = 'https://graph.facebook.com/148395741852581/feed?limit=50&until=' . $until . '&access_token=' . urlencode($access_token) . '&fields=message,link,picture';
        $curl = curl_init($url);
        curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
        $json = json_decode(curl_exec($curl));

        $ret = new StdClass;
        $paper_map = array(
            '中國時報' => '中國時報',
            '中時' => '中國時報',
            '聯合報' => '聯合報',
            '聯合' => '聯合報',
            '自由時報' => '自由時報',
            '自由' => '自由時報',
            '蘋果日報' => '蘋果日報',
            '蘋果' => '蘋果日報',
        );
        foreach ($json->data as $post) {
            if (false === strpos($post->message, '頭條')) {
                continue;
            }
            if (false === strpos($post->message, '蘋果')) {
                continue;
            }
            if (strtotime($post->created_time) < $timestamp) {
                break;
            }
            $message = $post->message;
            $message = str_replace('【小編報頭條】中時', '【小編報頭條】' . "\n" . '中時', $message);
            $message = preg_replace('#http[^\s]*#', '', $message);
            $message = str_replace('★', '', $message);
            $message = str_replace('＊', '', $message);
            $lines = explode("\n", $message);
            $ret->title = $lines[0];
            $ret->link = $post->link;
            $ret->image_link = $post->picture;
            $ret->time = $timestamp;
            $ret->headlines = array();
            foreach ($lines as $line) {
                $line = str_replace('：', ':', $line);
                if (FALSE === strpos($line, ':')) {
                    continue;
                }
                list($paper, $title) = explode(':', trim($line), 2);
                $paper = str_replace(' ', '', $paper);
                if (!array_key_exists($paper, $paper_map)) {
                    continue;
                }
                $ret->headlines[] = array($paper_map[$paper], $title);

            }

            if (count(array_unique(array_map(function($a){ return $a[0]; }, $ret->headlines))) == 4) {
                return $ret;
{'fraction_non_alphanumeric': 0.180454020777222, 'fraction_numerical': 0.010003847633705272, 'mean_word_length': 1.6350734921439432, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 49, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4570208', 'n_tokens_mistral': 1755, 'n_tokens_neox': 1681, 'n_words': 289}