<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
__author__="<NAME>, <NAME>" 

import sys
import requests
import json
import codecs
import argparse
import datetime
import time
import re

from acscsv.twacscsv import TwacsCSV
from simple_n_grams.simple_n_grams import SimpleNGrams

reload(sys)
sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
sys.stdin = codecs.getreader('utf-8')(sys.stdin)

# formatter of data from API 
TIME_FMT = "%Y%m%d%H%M"
LOCAL_DATA_DIRECTORY = "./data/"
PAUSE = 3 # seconds between page requests

class GnipSearchAPI:

    USE_CASES = ["json", "wordcount","users", "rate", "links", "timeline", "geo"]
    
    def __init__(self, token_list_size=20):
        # Some constants to configure column retrieval
        DATE_INDEX = 1
        TEXT_INDEX = 2
        LINKS_INDEX = 3
        USER_NAME_INDEX = 7 
        # default tokenizer and character limit
        space_tokenizer = False
        char_upper_cutoff = 20  # longer than for normal words because of user names
        self.token_list_size = int(token_list_size)
        # re for the acceptable datetime formats
        timeRE = re.compile("([0-9]{4}).([0-9]{2}).([0-9]{2}).([0-9]{2}):([0-9]{2})")
        # parse the command line options
        self.options = self.args().parse_args()
        # get a parser for the twitter columns
        # TODO: use the updated retriveal methods in gnacs instead of this
        self.twitter_parser = TwacsCSV(",", None, False, True, False, True, False, False, False)
        # set up the job
        if self.options.use_case.startswith("links"):
            char_upper_cutoff=100
            space_tokenizer = True
        self.freq = SimpleNGrams(charUpperCutoff=char_upper_cutoff, space_tokenizer=space_tokenizer)
        if self.options.use_case.startswith("user"):
            self.index = USER_NAME_INDEX
        elif self.options.use_case.startswith("wordc"):
            self.index = TEXT_INDEX
        elif self.options.use_case.startswith("rate"):
            self.index = DATE_INDEX
        elif self.options.use_case.startswith("link"):
            self.index = LINKS_INDEX
        elif self.options.use_case.startswith("time"):
            if not self.options.stream_url.endswith("counts.json"): 
                self.options.stream_url = self.options.stream_url[:-5] + "/counts.json"
            if self.options.count_bucket not in ['day', 'minute', 'hour']:
                print >> sys.stderr, "Error. Invalid count bucket: %s \n"%str(self.options.count_bucket)
                sys.exit()
        if self.options.start:
            dt = re.search(timeRE, self.options.start)
            if not dt:
                print >> sys.stderr, "Error. Invalid start-date format: %s \n"%str(self.options.start)
                sys.exit()    
            else:
                f =''
                for i in range(re.compile(timeRE).groups):
                    f += dt.group(i+1) 
                self.fromDate = f
        if self.options.end:
            dt = re.search(timeRE, self.options.end)
            if not dt:
                print >> sys.stderr, "Error. Invalid end-date format: %s \n"%str(self.options.end)
                sys.exit()
            else:
                e =''
                for i in range(re.compile(timeRE).groups):
                    e += dt.group(i+1) 
                self.toDate = e
        self.name_munger(self.options.filter)

    def args(self):
        twitter_parser = argparse.ArgumentParser(
                description="GnipSearch supports the following use cases: %s"%str(self.USE_CASES))
        twitter_parser.add_argument("use_case", metavar= "USE_CASE", choices=self.USE_CASES, 
                help="Use case for this search.")
        twitter_parser.add_argument("-a", "--paged", dest="paged", action="store_true", 
                default=False, help="Paged access to ALL available results (Warning: this makes many requests)")
        twitter_parser.add_argument("-c", "--csv", dest="csv_flag", action="store_true", 
                default=False,
                help="Return comma-separated 'date,counts' or geo data.")
        twitter_parser.add_argument("-b", "--bucket", dest="count_bucket", 
                default="day", 
                help="Bucket size for counts query. Options are day, hour, minute (default is 'day').")
        twitter_parser.add_argument("-e", "--end-date", dest="end", 
                default=None,
                help="End of datetime window, format 'YYYY-mm-DDTHH:MM' (default: most recent activities)")
        twitter_parser.add_argument("-f", "--filter", dest="filter", default="from:jrmontag OR from:gnip",
                help="PowerTrack filter rule (See: http://support.gnip.com/customer/portal/articles/901152-powertrack-operators)")
        twitter_parser.add_argument("-l", "--stream-url", dest="stream_url", 
                default="https://search.gnip.com/accounts/shendrickson/search/wayback.json",
                help="Url of search endpoint. (See your Gnip console.)")
        twitter_parser.add_argument("-n", "--results-max", dest="max", default=100, 
                help="Maximum results to return (default 100)")
        twitter_parser.add_argument("-p", "--password", dest="pwd", 
                help="Password")
        twitter_parser.add_argument("-q", "--query", dest="query", action="store_true", 
                default=False, help="View API query (no data)")
        twitter_parser.add_argument("-s", "--start-date", dest="start", 
                default=None,
                help="Start of datetime window, format 'YYYY-mm-DDTHH:MM' (default: 30 days ago)")
        twitter_parser.add_argument("-u", "--user-name", dest="user", default="<EMAIL>",
                help="User name")
        twitter_parser.add_argument("-w", "--file-name", dest="file_name", default=False,  action="store_true", 
                help="Create files in ./data if flag is set (default: no output files)")
        return twitter_parser
    
    def name_munger(self, f):
        """Creates a file name per input rule when reading multiple input rules."""
        f = re.sub(' +','_',f)
        f = f.replace(':','_')
        f = f.replace('"','_Q_')
        f = f.replace('(','_p_') 
        f = f.replace(')','_p_') 
        self.file_name_prefix = f

    def req(self):
        try:
            s = requests.Session()
            s.headers = {'Accept-encoding': 'gzip'}
            s.auth = (self.options.user, self.options.pwd)
            res = s.post(self.options.stream_url, data=json.dumps(self.rule_payload))
        except requests.exceptions.ConnectionError, e:
            print >> sys.stderr, "Error (%s). Exiting without results."%str(e)
            sys.exit()
        except requests.exceptions.HTTPError, e:
            print >> sys.stderr, "Error (%s). Exiting without results."%str(e)
            sys.exit()
        return res.text

    def parse_JSON(self):
        acs = []
        repeat = True
        page_count = 0
        while repeat:
            doc = self.req()
            try:
                tacs =  json.loads(doc)
                if "results" in tacs:
                    acs.extend(tacs["results"])
                if "error" in tacs:
                    print >> sys.stderr, "Error, invalid request"
                    print >> sys.stderr, "Query: %s"%self.rule_payload
                    print >> sys.stderr, "Response: %s"%doc
            except ValueError:
                print >> sys.stderr, "Error, results not parsable"
                print >> sys.stderr, doc
                sys.exit()
            # 
            repeat = False
            if self.options.paged:
                if len(acs) > 0:
                    if self.options.file_name:
                        file_name = LOCAL_DATA_DIRECTORY + "{0}_{1}.json".format(
                                str(datetime.datetime.utcnow().strftime(
                                    "%Y%m%d%H%M%S"))
                              , str(self.file_name_prefix))
                        with codecs.open(file_name, "wb","utf-8") as out:
                            print >> sys.stderr, "(writing to file ...)"
                            for item in acs:
                                out.write(json.dumps(item)+"\n")
                    else:
                        # if writing to file, don't keep track of all the data in memory
                        acs = []
                else:
                    print >> sys.stderr, "no results returned for rule:{0}".format(str(self.rule_payload))
                if "next" in tacs:
                    self.rule_payload["next"]=tacs["next"]
                    repeat = True
                    page_count += 1
                    print >> sys.stderr, "Fetching page {}...".format(page_count)
                else:
                    if "next" in self.rule_payload:
                        del self.rule_payload["next"]
                    repeat = False
                time.sleep(PAUSE)
        return acs

    def __call__(self):
        self.rule_payload = {
                                'query': self.options.filter<commit_msg>fix but in file writing output and rename vars for clarity<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
__author__="<NAME>, <NAME>" 

import sys
import requests
import json
import codecs
import argparse
import datetime
import time
import re

from acscsv.twacscsv import TwacsCSV
from simple_n_grams.simple_n_grams import SimpleNGrams

reload(sys)
sys.stdout = codecs.getwriter('utf-8')(sys.stdout)
sys.stdin = codecs.getreader('utf-8')(sys.stdin)

# formatter of data from API 
TIME_FMT = "%Y%m%d%H%M"
LOCAL_DATA_DIRECTORY = "./data/"
PAUSE = 3 # seconds between page requests

class GnipSearchAPI:

    USE_CASES = ["json", "wordcount","users", "rate", "links", "timeline", "geo"]
    
    def __init__(self, token_list_size=20):
        # Some constants to configure column retrieval
        DATE_INDEX = 1
        TEXT_INDEX = 2
        LINKS_INDEX = 3
        USER_NAME_INDEX = 7 
        # default tokenizer and character limit
        space_tokenizer = False
        char_upper_cutoff = 20  # longer than for normal words because of user names
        self.token_list_size = int(token_list_size)
        # re for the acceptable datetime formats
        timeRE = re.compile("([0-9]{4}).([0-9]{2}).([0-9]{2}).([0-9]{2}):([0-9]{2})")
        # parse the command line options
        self.options = self.args().parse_args()
        # get a parser for the twitter columns
        # TODO: use the updated retriveal methods in gnacs instead of this
        self.twitter_parser = TwacsCSV(",", None, False, True, False, True, False, False, False)
        # set up the job
        if self.options.use_case.startswith("links"):
            char_upper_cutoff=100
            space_tokenizer = True
        self.freq = SimpleNGrams(charUpperCutoff=char_upper_cutoff, space_tokenizer=space_tokenizer)
        if self.options.use_case.startswith("user"):
            self.index = USER_NAME_INDEX
        elif self.options.use_case.startswith("wordc"):
            self.index = TEXT_INDEX
        elif self.options.use_case.startswith("rate"):
            self.index = DATE_INDEX
        elif self.options.use_case.startswith("link"):
            self.index = LINKS_INDEX
        elif self.options.use_case.startswith("time"):
            if not self.options.stream_url.endswith("counts.json"): 
                self.options.stream_url = self.options.stream_url[:-5] + "/counts.json"
            if self.options.count_bucket not in ['day', 'minute', 'hour']:
                print >> sys.stderr, "Error. Invalid count bucket: %s \n"%str(self.options.count_bucket)
                sys.exit()
        if self.options.start:
            dt = re.search(timeRE, self.options.start)
            if not dt:
                print >> sys.stderr, "Error. Invalid start-date format: %s \n"%str(self.options.start)
                sys.exit()    
            else:
                f =''
                for i in range(re.compile(timeRE).groups):
                    f += dt.group(i+1) 
                self.fromDate = f
        if self.options.end:
            dt = re.search(timeRE, self.options.end)
            if not dt:
                print >> sys.stderr, "Error. Invalid end-date format: %s \n"%str(self.options.end)
                sys.exit()
            else:
                e =''
                for i in range(re.compile(timeRE).groups):
                    e += dt.group(i+1) 
                self.toDate = e
        self.name_munger(self.options.filter)

    def args(self):
        twitter_parser = argparse.ArgumentParser(
                description="GnipSearch supports the following use cases: %s"%str(self.USE_CASES))
        twitter_parser.add_argument("use_case", metavar= "USE_CASE", choices=self.USE_CASES, 
                help="Use case for this search.")
        twitter_parser.add_argument("-a", "--paged", dest="paged", action="store_true", 
                default=False, help="Paged access to ALL available results (Warning: this makes many requests)")
        twitter_parser.add_argument("-c", "--csv", dest="csv_flag", action="store_true", 
                default=False,
                help="Return comma-separated 'date,counts' or geo data.")
        twitter_parser.add_argument("-b", "--bucket", dest="count_bucket", 
                default="day", 
                help="Bucket size for counts query. Options are day, hour, minute (default is 'day').")
        twitter_parser.add_argument("-e", "--end-date", dest="end", 
                default=None,
                help="End of datetime window, format 'YYYY-mm-DDTHH:MM' (default: most recent activities)")
        twitter_parser.add_argument("-f", "--filter", dest="filter", default="from:jrmontag OR from:gnip",
                help="PowerTrack filter rule (See: http://support.gnip.com/customer/portal/articles/901152-powertrack-operators)")
        twitter_parser.add_argument("-l", "--stream-url", dest="stream_url", 
                default="https://search.gnip.com/accounts/shendrickson/search/wayback.json",
                help="Url of search endpoint. (See your Gnip console.)")
        twitter_parser.add_argument("-n", "--results-max", dest="max", default=100, 
                help="Maximum results to return (default 100)")
        twitter_parser.add_argument("-p", "--password", dest="pwd", 
                help="Password")
        twitter_parser.add_argument("-q", "--query", dest="query", action="store_true", 
                default=False, help="View API query (no data)")
        twitter_parser.add_argument("-s", "--start-date", dest="start", 
                default=None,
                help="Start of datetime window, format 'YYYY-mm-DDTHH:MM' (default: 30 days ago)")
        twitter_parser.add_argument("-u", "--user-name", dest="user", default="<EMAIL>",
                help="User name")
        twitter_parser.add_argument("-w", "--file-name", dest="file_name", default=False,  action="store_true", 
                help="Create files in ./data if flag is set (default: no output files)")
        return twitter_parser
    
    def name_munger(self, f):
        """Creates a file name per input rule when reading multiple input rules."""
        f = re.sub(' +','_',f)
        f = f.replace(':','_')
        f = f.replace('"','_Q_')
        f = f.replace('(','_p_') 
        f = f.replace(')','_p_') 
        self.file_name_prefix = f

    def req(self):
        try:
            s = requests.Session()
            s.headers = {'Accept-encoding': 'gzip'}
            s.auth = (self.options.user, self.options.pwd)
            res = s.post(self.options.stream_url, data=json.dumps(self.rule_payload))
        except requests.exceptions.ConnectionError, e:
            print >> sys.stderr, "Error (%s). Exiting without results."%str(e)
            sys.exit()
        except requests.exceptions.HTTPError, e:
            print >> sys.stderr, "Error (%s). Exiting without results."%str(e)
            sys.exit()
        return res.text

    def parse_JSON(self):
        acs = []
        repeat = True
        page_count = 0
        while repeat:
            doc = self.req()
            try:
                tmp_response =  json.loads(doc)
                if "results" in tmp_response:
                    acs.extend(tmp_response["results"])
                if "error" in tmp_response:
                    print >> sys.stderr, "Error, invalid request"
                    print >> sys.stderr, "Query: %s"%self.rule_payload
                    print >> sys.stderr, "Response: %s"%doc
            except ValueError:
                print >> sys.stderr, "Error, results not parsable"
                print >> sys.stderr, doc
                sys.exit()
            # 
            repeat = False
            if self.options.paged:
                if len(acs) > 0:
                    if self.options.file_name:
                        file_name = LOCAL_DATA_DIRECTORY + "{0}_{1}.json".format(
                                str(datetime.datetime.utcnow().strftime(
                                    "%Y%m%d%H%M%S"))
                              , str(self.file_name_prefix))
                        with codecs.open(file_name, "wb","utf-8") as out:
                            print >> sys.stderr, "(writing to file ...)"
                            for item in tmp_response["results"]:
                                out.write(json.dumps(item)+"\n")
                    else:
                        # if writing to file, don't keep track of all the data in memory
                        acs = []
                else:
                    print >> sys.stderr, "no results returned for rule:{0}".format(str(self.rule_payload))
                if "next" in tmp_response:
                    self.rule_payload["next"]=tmp_response["next"]
                    repeat = True
                    page_count += 1
                    print >> sys.stderr, "Fetching page {}...".format(page_count)
                else:
                    if "next" in self.rule_payload:
                        del self.rule_payload["next"]
                    repeat = False
                time.sleep(PAUSE)
        return acs

    def __call__(self):
        if self.options.paged:
            # avoid making many small requests
            self.options.max = 500
        self.rule_payload = {
                                'query': self.options.filter
{'fraction_non_alphanumeric': 0.12711221837088388, 'fraction_numerical': 0.006011698440207972, 'mean_word_length': 1.8716951788491447, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 63, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1592829', 'n_tokens_mistral': 5440, 'n_tokens_neox': 5185, 'n_words': 1234}