<commit_before>import urllib.request
import json
import datetime
import csv
import time
import os
from time import gmtime, strftime
from fb_posts_realtime import init_kafka, send_message
from fb_comments_page import scrapeFacebookPageFeedComments
class FB_SCRAPE(object):
    def __init__(self, useKafka, useES, useSQL, outputJSON):
        self.producer = None
        if useKafka:
            self.producer = init_kafka(os.environ['KAFKA_PORT'])
        self.ES = useES
        self.useSQL = useSQL
        self.JSON = outputJSON
        self.file_id = ""
        self.access_token = ""
        self.tstamp = ""
        self.sesh = strftime("%Y-%m-%d%H", gmtime())
        self.dir = "data/files/" + self.sesh + "/"

        if not os.path.exists(self.dir):
            os.makedirs(self.dir)

    def request_until_succeed(self, url):
        req = urllib.request.Request(url)
        success = False
        while success is False:
            try:
                response = urllib.request.urlopen(req)
                if response.getcode() == 200:
                    success = True
            except Exception as e:
                print(e)
                time.sleep(5)

                print("Error for URL %s: %s" % (url, datetime.datetime.now()))
                print("Retrying.")

        return response.read().decode(response.headers.get_content_charset())


    # Needed to write tricky unicode correctly to csv
    def unicode_normalize(self, text):
        return text.translate({ 0x2018:0x27, 0x2019:0x27, 0x201C:0x22, 0x201D:0x22,
                                0xa0:0x20 })

    def getFacebookPageFeedData(self, page_id, access_token, num_statuses,tStamp):

        # Construct the URL string; see http://stackoverflow.com/a/37239851 for
        # Reactions parameters
        base = "https://graph.facebook.com/v2.6"<commit_msg>removed printing of confidential info<commit_after>import urllib.request
import json
import datetime
import csv
import time
import os
from time import gmtime, strftime
from fb_posts_realtime import init_kafka, send_message
from fb_comments_page import scrapeFacebookPageFeedComments
class FB_SCRAPE(object):
    def __init__(self, useKafka, useES, useSQL, outputJSON):
        self.producer = None
        if useKafka:
            self.producer = init_kafka(os.environ['KAFKA_PORT'])
        self.ES = useES
        self.useSQL = useSQL
        self.JSON = outputJSON
        self.file_id = ""
        self.access_token = ""
        self.tstamp = ""
        self.sesh = strftime("%Y-%m-%d%H", gmtime())
        self.dir = "data/files/" + self.sesh + "/"

        if not os.path.exists(self.dir):
            os.makedirs(self.dir)

    def request_until_succeed(self, url):
        req = urllib.request.Request(url)
        success = False
        while success is False:
            try:
                response = urllib.request.urlopen(req)
                if response.getcode() == 200:
                    success = True
            except Exception as e:
                print(e)
                time.sleep(5)

                #print("Error for URL %s: %s" % (url, datetime.datetime.now()))
                print("Retrying.")

        return response.read().decode(response.headers.get_content_charset())


    # Needed to write tricky unicode correctly to csv
    def unicode_normalize(self, text):
        return text.translate({ 0x2018:0x27, 0x2019:0x27, 0x201C:0x22, 0x201D:0x22,
                                0xa0:0x20 })

    def getFacebookPageFeedData(self, page_id, access_token, num_statuses,tStamp):

        # Construct the URL string; see http://stackoverflow.com/a/37239851 for
        # Reactions parameters
        base = "https://graph.facebook.com/v2.6"
{'fraction_non_alphanumeric': 0.10109949048002145, 'fraction_numerical': 0.026280504156610353, 'mean_word_length': 2.2127476313522827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4061658', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1112, 'n_words': 259}