<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/gdrive')
import requests

cookies = {
    '_ga': 'GA1.2.400921435.1650713202',
    'cookielawinfo-checkbox-necessary': 'yes',
    'cookielawinfo-checkbox-functional': 'yes',
    'cookielawinfo-checkbox-performance': 'yes',
    'cookielawinfo-checkbox-analytics': 'yes',
    'cookielawinfo-checkbox-advertisement': 'yes',
    'cookielawinfo-checkbox-others': 'yes',
    'CookieLawInfoConsent': '<KEY>
    'viewed_cookie_policy': 'yes',
    '_gid': 'GA1.2.44976476.1651291872',
    'wordpress_test_cookie': 'WP%20Cookie%20check',
}

headers = {
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    'Accept-Language': 'en-US,en;q=0.9',
    'Connection': 'keep-alive',
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
    # Requests sorts cookies= alphabetically
    # 'Cookie': '_ga=GA1.2.400921435.1650713202; cookielawinfo-checkbox-necessary=yes; cookielawinfo-checkbox-functional=yes; cookielawinfo-checkbox-performance=yes; cookielawinfo-checkbox-analytics=yes; cookielawinfo-checkbox-advertisement=yes; cookielawinfo-checkbox-others=yes; CookieLawInfoConsent=<KEY>; viewed_cookie_policy=yes; _gid=GA1.2.44976476.1651291872; wordpress_test_cookie=WP%20Cookie%20check',
    'Origin': 'https://www.antifakenewscenter.com',
    'Referer': 'https://www.antifakenewscenter.com/allcontent/',
    'Sec-Fetch-Dest': 'empty',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Site': 'same-origin',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
    'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
    'sec-ch-ua-mobile': '?0',
    'sec-ch-ua-platform': '"macOS"',
}

data = {
    'action': 'get_post_by_page',
    'category[]': [
        'การเงิน-หุ้น',
        'ความสงบและความมั่นคง',
        'นโยบายรัฐบาล-ข่าวสาร',
        'ผลิตภัณฑ์สุขภาพ',
        'ภัยพิบัติ',
        'ยาเสพติด',
        'เศรษฐกิจ',
    ],
    'pageNumber': '1',
    'pageSize': '5000', # change the amount as you want! eg. 9, 100, 1000, 5000
    'sort': 'lastest',
    'type': 'category',
}

response = requests.post('https://www.antifakenewscenter.com/wp-admin/admin-ajax.php', cookies=cookies, headers=headers, data=data)
json_data = response.json()
json_data
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
source = ''
for i in json_data['html']:
  source += i

def get_df(soup):
  arr = []
  for i in soup.find_all('div', class_='blog-container'):
    d={}
    d['headline'] = i.find('div', class_='blog-title').text.strip()
    d['url'] = i.find('a').get('href')
    d['img_src'] = i.find('img',class_='attachment-post-thumbnail size-post-thumbnail wp-post-image').get('src')
    try:
      d['tag'] = i.find('div',class_='tag-status').get_text()
    except:
      d['tag'] = None
    try:
      d['topic'] = i.find('div',class_='blog-tag tag-interest').get_text()
    except:
      d['topic'] = None
    arr.append(d)
  return pd.DataFrame(arr)
soup = BeautifulSoup(source)
df = get_df(soup)
df.head()
df.dropna(axis=0)
df.to_csv('/content/gdrive/MyDrive/Fake news/News-Dataset/ภาษาไทย_ศูนย์ต่อต้านข่าวปลอม/dataset-antifakenewscenter.csv')
df.groupby(by = ['tag']).size()
group_df = [y for x, y in df.groupby('tag', as_index=False)]
true, fabricate, fake = group_df[0].reset_index(drop=True), group_df[1].reset_index(drop=True), group_df[2].reset_index(drop=True)
fake
df['headline'] = df['headline'].str.replace('ข่าวปลอม อย่าแชร์!','')
df['headline']
df['headline'].to_csv(r'headline.txt', header=None, index=None, sep=' ', mode='a')

!mkdir fake true fabricate
import urllib.request
import urllib.parse

def download_img(tag, df):
  for i in range(len(df['img_src'])):
    try:
      url = df['img_src'][i]
      url = urllib.parse.urlsplit(url)
      url = list(url)
      url[2] = urllib.parse.quote(url[2])
      url = urllib.parse.urlunsplit(url)
      urllib.request.urlretrieve(url, "{}/{:04d}.jpg".format(tag, i))
      if i%10==0: print('Progress: {}/{}'.format(i,len(df['img_src'])))
    except Exception as e:
      print('Fetching image no.{} fail'.format(i))
      print(e)
download_img('fake',fake)
download_img('fabricate',fabricate)
download_img('true',true)
with open('/content/gdrive/My Drive/file.txt', 'w') as f:
  f.write('content')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19192144610577996, 'fraction_numerical': 0.03615264449899576, 'mean_word_length': 4.445929526123937, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '188037', 'license': 'MIT', 'n_tokens_mistral': 1868, 'n_tokens_neox': 1788, 'n_words': 271, 'path': 'Notebooks/12 Dataset _ Antifakenewscenter scraper.ipynb', 'repo_name': 'nitsirs/debunker', 'size': '4761'}