<commit_before>'''
Created on May 8, 2015

@author: Sherry

Beutiful Soup External Library: http://www.crummy.com/software/BeautifulSoup/
'''

from bs4 import *
import requests
import re
from json import dumps

class TwitterScreenScrape:
    
    consumer_key = ''
    consumer_secret = ''
    
    access_token = ''
    access_secret = ''
        
    def __init__(self):
        pass
        
    def get_twitter_data(self, app_list):
        twitter_dict = {}
        app_dict = {}
        
        for app in app_list:
            tweet_dict = self.search_tweet(app)
            app_dict[app] = tweet_dict
            
        twitter_dict['twitter'] = app_dict
        
        return twitter_dict 
    
    def search_tweet(self, app_name):
        tweet_dict = {}
        info_dict = {}
        tweet_ID = 0   
        #url_file = requests.get('https://twitter.com/search?f=realtime&q=' + str.lower(app) + '%20app&src=typd')
        url_file = requests.get('https://twitter.com/search?q=' + str.lower(app_name) + '%20app&src=typd')
        b_soup = BeautifulSoup(url_file.content)
            
        for tweet in b_soup.find_all('div', {'data-follows-you': 'false'}):
        #for tweet in b_soup.find_all('div', {'class': 'tweet'}):
            if tweet:
                date = tweet.find('a', {'class': 'tweet-timestamp js-permalink js-nav js-tooltip'})
                info_dict['created_at'] = re.findall('\d\s\w\w\w\s\d\d\d\d', str(date.get('title')))[0] if date else ''
                    
                text = tweet.find('p', {'class': 'TweetTextSize  js-tweet-text tweet-text'})
                info_dict['text'] = text.text if text else ''
                    
                source = tweet.get('data-permalink-path')
                info_dict['source'] = ('http://twitter.com' + source) if source else ''
                
                user_URL = requests.get(info_dict['source'])
                b_soup_2 = BeautifulSoup(user_URL.content)
                
                retweet = b_soup_2.find('a', {'class': 'request-retweeted-popup'})
                info_dict['retweet_count'] = retweet.get('data-tweet-stat-count') if retweet else 0
                                                            
                for username_list in tweet.find_all('span', {'class': 'username js-action-profile-name'}):
                    user_dict = {}
                    
                    username = username_list.find('b').text
                    user_dict['screen_name'] = username if username else ''
                        
                    user_URL = requests.get('https://twitter.com/' + username)
                    b_soup_3 = BeautifulSoup(user_URL.content)
                        
                    name = b_soup_3.find('a', {'class': 'ProfileHeaderCard-nameLink'})
                    user_dict['name'] = name.text if name else ''
                        
                    location = b_soup_3.find('span', {'class': 'ProfileHeaderCard-locationText'})
                    user_dict['location'] = location.text.strip() if location else ''
                        
                    followers = b_soup_3.find('a', {'data-nav': 'followers'})
                    user_dict['followers_count'] = ((followers.get('title')).strip(' Followers')).replace(',', '') if str(followers.text) != "" else 0
                        
                    user_date = b_soup_3.find('span', {'class': 'ProfileHeaderCard-joinDateText'})
                    user_dict['created_at'] = re.findall('\d\s\w\w\w\s\d\d\d\d', str(user_date.get('title')))[0] if user_date else ''
                
                    info_dict['user'] = user_dict
                
                tweet_dict['tweet_' + str(tweet_ID)] = info_dict
                
                '''
                print ('-' * 5), 'Tweet ', tweet_ID, ('-' * 5)
                print info_dict['created_at']
                print info_dict['text']
                print info_dict['retweet_count']
                print info_dict['source']
                print user_dict['screen_name']
                print user_dict['name']
                print user_dict['location']
                print user_dict['followers_count']
                print user_dict['created_at']
                '''                
                tweet_ID += 1
        
        return tweet_dict<commit_msg>Fixed duplicate tweet error<commit_after>'''
Created on May 8, 2015

@author: Sherry

Beutiful Soup External Library: http://www.crummy.com/software/BeautifulSoup/
'''

from bs4 import *
import requests
import re
from json import dumps

class TwitterScreenScrape:
        
    def __init__(self):
        pass
        
    def get_twitter_data(self, app_list):
        twitter_dict = {}
        app_dict = {}
        
        for app in app_list:
            app_dict = {}
            
            tweet_dict = self.search_tweet(app)
            app_dict[app] = tweet_dict
            
        twitter_dict['twitter'] = app_dict
        
        return twitter_dict 
    
    def search_tweet(self, app_name):
        tweet_dict = {}
        info_dict = {}
        tweet_ID = 0   
        #url_file = requests.get('https://twitter.com/search?f=realtime&q=' + str.lower(app) + '%20app&src=typd')
        url_file = requests.get('https://twitter.com/search?q=' + str.lower(app_name) + '%20app&src=typd')
        b_soup = BeautifulSoup(url_file.content)
            
        for tweet in b_soup.find_all('div', {'data-follows-you': 'false'}):
        #for tweet in b_soup.find_all('div', {'class': 'tweet'}):
            if tweet:
                info_dict = {}
                
                date = tweet.find('a', {'class': 'tweet-timestamp js-permalink js-nav js-tooltip'})
                info_dict['created_at'] = re.findall('\d\s\w\w\w\s\d\d\d\d', str(date.get('title')))[0] if date else ''
                    
                text = tweet.find('p', {'class': 'TweetTextSize  js-tweet-text tweet-text'})
                info_dict['text'] = text.text if text else ''
                    
                source = tweet.get('data-permalink-path')
                info_dict['source'] = ('http://twitter.com' + source) if source else ''
                
                user_URL = requests.get(info_dict['source'])
                b_soup_2 = BeautifulSoup(user_URL.content)
                
                retweet = b_soup_2.find('a', {'class': 'request-retweeted-popup'})
                info_dict['retweet_count'] = retweet.get('data-tweet-stat-count') if retweet else 0
                                                            
                for username_list in tweet.find_all('span', {'class': 'username js-action-profile-name'}):
                    user_dict = {}
                    
                    username = username_list.find('b').text
                    user_dict['screen_name'] = username if username else ''
                        
                    user_URL = requests.get('https://twitter.com/' + username)
                    b_soup_3 = BeautifulSoup(user_URL.content)
                        
                    name = b_soup_3.find('a', {'class': 'ProfileHeaderCard-nameLink'})
                    user_dict['name'] = name.text if name else ''
                        
                    location = b_soup_3.find('span', {'class': 'ProfileHeaderCard-locationText'})
                    user_dict['location'] = location.text.strip() if location else ''
                        
                    followers = b_soup_3.find('a', {'data-nav': 'followers'})
                    user_dict['followers_count'] = ((followers.get('title')).strip(' Followers')).replace(',', '') if str(followers.text) != "" else 0
                        
                    user_date = b_soup_3.find('span', {'class': 'ProfileHeaderCard-joinDateText'})
                    user_dict['created_at'] = re.findall('\d\s\w\w\w\s\d\d\d\d', str(user_date.get('title')))[0] if user_date else ''
                
                    info_dict['user'] = user_dict
                
                tweet_dict['tweet_' + str(tweet_ID)] = info_dict
                
                '''
                print ('-' * 5), 'Tweet ', tweet_ID, ('-' * 5)
                print info_dict['created_at']
                print info_dict['text']
                print info_dict['retweet_count']
                print info_dict['source']
                print user_dict['screen_name']
                print user_dict['name']
                print user_dict['location']
                print user_dict['followers_count']
                print user_dict['created_at']
                '''                
                tweet_ID += 1
        
        return tweet_dict
{'fraction_non_alphanumeric': 0.12946172386089752, 'fraction_numerical': 0.00573855158957879, 'mean_word_length': 1.533139534883721, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6054563', 'n_tokens_mistral': 2600, 'n_tokens_neox': 2435, 'n_words': 436}