<jupyter_start><jupyter_text>Integrating Dbias to reduce bias in Google News API data<jupyter_code>!pip install GoogleNews
!pip install Dbias
!pip install https://huggingface.co/d4data/en_pipeline/resolve/main/en_pipeline-any-py3-none-any.whl<jupyter_output><empty_output><jupyter_text>loading packgaes<jupyter_code>from GoogleNews import GoogleNews
from Dbias.text_debiasing import *<jupyter_output><empty_output><jupyter_text>functions<jupyter_code>def news(topic):
    googlenews = GoogleNews()
    googlenews = GoogleNews(lang='en', region='US')
    googlenews = GoogleNews(period='7d')
    googlenews.get_news(topic)
    result=googlenews.result()
    return result

def debiasing(x):
    suggestions = run(x)
    if suggestions == None:
        return ""
    else:
      all_suggestions = []
      for sent in suggestions[0:3]:
        all_suggestions.append(sent['Sentence'])
      return "\n\n".join(all_suggestions)<jupyter_output><empty_output><jupyter_text>implementation<jupyter_code># reading the latest news, enter the topic required
latest_news = news(topic = "Covid Omicron")
news_df = pd.DataFrame(latest_news)
news_df.head()
# running Dbias to recommend unbiased/reduced bias news fragment
news_df['suggestion'] = news_df['title'].apply(lambda x: debiasing(x))
news_df[['title', 'suggestion']]
# saving the dataframe to csv
news_df.to_csv("news-unbiased.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11656005685856433, 'fraction_numerical': 0.0035536602700781805, 'mean_word_length': 5.97029702970297, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '122742', 'license': 'MIT', 'n_tokens_mistral': 499, 'n_tokens_neox': 485, 'n_words': 90, 'path': 'example notebooks/Dbias_newsapi.ipynb', 'repo_name': 'dreji18/Fairness-in-AI', 'size': '1407'}