<commit_before># need to rewrite from scratch

# # libraries
# import json

# # my classes
# from parser import Parser
# from twitter import Twitter
# from database import Database
# from sentiment_analysis import Sentiment_analysis

# # Global variables
# CREDENTIALS_FILE = "/Users/dmytrokurbakov/Desktop/credentials.json"
# CONFIG_FILE = "twitter_accounts.json"
# POSITIVE = "sentiment_analysis/negative_tweets.json"
# NEGATIVE = "sentiment_analysis/positive_tweets.json"
# CLASSIFIER_PATH = "sentiment_analysis/classifier.pickle"

# if __name__ == '__main__':
#     p = Parser()
#     credentials_string = p.read_file(CREDENTIALS_FILE)
#     credentials_dict = p.parse_string(credentials_string)

#     TWITTER_CKEY = credentials_dict.get("twitter_ckey")
#     TWITTER_ATOCKEN = credentials_dict.get("twitter_tocken")
#     TWITTER_CSECRET = credentials_dict.get("twitter_csecret")
#     TWITTER_ATOCKEN_SECRET = credentials_dict.get("twitter_tocken_secret")
#     twitter_accounts = p.read_file("twitter_accounts.json")
#     twitter_accounts = p.parse_string(twitter_accounts)

#     twitter_api = Twitter(TWITTER_CKEY, TWITTER_CSECRET, TWITTER_ATOCKEN, TWITTER_ATOCKEN_SECRET)

#     sa = Sentiment_analysis(POSITIVE, NEGATIVE, CLASSIFIER_PATH)
#     classifier = sa.load_classifier()

#     final_list = list()
#     for account in twitter_accounts:

#         twitter_out_data = twitter_api.get_twitts(twitter_id=account.get("id"), last_twitter_id=account.get("max_twitter_id"))
#         parsed_twitter_data = p.parse_twitter_data(twitter_out_data)

#         for news in parsed_twitter_data:

#             res = dict()

#             res["id"] = news.get("news_id")
#             res["news_created_at"] = news.get("news_created_at")
#             res["news_text"] = news.get("news_text")
#             res["twitter_name"] = account.get("twitter_name")
#             res["twitter_tag"] = account.get("twitter_tag")
#             res["twitter_page"] = account.get("twitter_page")<commit_msg>started to rewrite the run script<commit_after># need to rewrite from scratch

# libraries
import json
import os

# my classes
from classes.class_parser import Parser
from classes.class_twitter import Twitter
from classes.class_database import Database
from classes.class_sentiment_analysis import SentimentAnalysis

# Global variables

CREDENTIALS_FILE = "../../credentials.json"
CONFIG_FILE = "data/twitter_accounts.json"
POSITIVE = "data/negative_tweets.json"
NEGATIVE = "data/positive_tweets.json"
CLASSIFIER_PATH = "sentiment_analysis/classifier.pickle"

if __name__ == '__main__':
    p = Parser()
    credentials_string = p.read_file(CREDENTIALS_FILE)
    credentials_dict = p.parse_string(credentials_string)

    TWITTER_CKEY = credentials_dict.get("twitter_ckey")
    TWITTER_ATOCKEN = credentials_dict.get("twitter_tocken")
    TWITTER_CSECRET = credentials_dict.get("twitter_csecret")
    TWITTER_ATOCKEN_SECRET = credentials_dict.get("twitter_tocken_secret")

    twitter_api = Twitter(TWITTER_CKEY, TWITTER_CSECRET, TWITTER_ATOCKEN, TWITTER_ATOCKEN_SECRET)

    sa = SentimentAnalysis(POSITIVE, NEGATIVE, CLASSIFIER_PATH)
    classifier = sa.load_classifier()

    # get data from Elasticsearch!!!!!!
    # twitter_accounts = p.read_file(CONFIG_FILE)
    # twitter_accounts = p.parse_string(twitter_accounts)

    final_list = list()
    for account in twitter_accounts:

        twitter_out_data = twitter_api.get_twitts(twitter_id=account.get("id"), last_twitter_id=account.get("max_twitter_id"))
        parsed_twitter_data = p.parse_twitter_data(twitter_out_data)

#         for news in parsed_twitter_data:

#             res = dict()

#             res["id"] = news.get("news_id")
#             res["news_created_at"] = news.get("news_created_at")
#             res["news_text"] = news.get("news_text")
#             res["twitter_name"] = account.get("twitter_name")
#             res["twitter_tag"] = account.get("twitter_tag")
#             res["twitter_page"] = account.get("twitter_page")
{'fraction_non_alphanumeric': 0.1088960877149265, 'fraction_numerical': 0.0, 'mean_word_length': 4.359145527369827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3587492', 'n_tokens_mistral': 1380, 'n_tokens_neox': 1325, 'n_words': 210}