<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, csv, json
from time import time 
import datetime
from collections import Counter
import networkx as nx
import community
import locale

import lib.tweetminer as minetweet
from lib.users import UserAPI
from lib.nlp import NLPMiner
from lib.mongo import MongoDB

t0=time()
minetweet.init_tweet_regex()

locale.setlocale(locale.LC_ALL, "")

nlp=NLPMiner()

stoplist=[i.strip() for i in open("lib/stopwords/zh-stopwords","r")]
stoplist+=[i.strip() for i in open("lib/stopwords/stopwords.txt","r")]
stoplist+=["转发","微博","说 ","一个","【 ","年 ","转 ","请","＂ ","问题","知道","中 ","已经","现在","说","【",'＂',"年","中","今天","应该","真的","月","希望","想","日","这是","太","转","支持"]
# stoplist+=["事儿","中国"]

api=UserAPI()
words_users_time=[]

def get_province(_userid):
    province_code= api.get_province(_userid)
    # print province_code
    try :
        return api.provinces[province_code]
    except KeyError :
        return 0

def test_com(meme_id):
    print meme_id

# Connect to Mongo
db=MongoDB("weibodata").db

def analyze_meme(meme_id,collection):

    # Init
    tstart=time()<commit_msg>fix path<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, csv, json
from time import time 
import datetime
from collections import Counter
import networkx as nx
import community
import locale

import lib.tweetminer as minetweet
from lib.users import UserAPI
from lib.nlp import NLPMiner
from lib.mongo import MongoDB

t0=time()
minetweet.init_tweet_regex()

locale.setlocale(locale.LC_ALL, "")

nlp=NLPMiner()

stoplist=[i.strip() for i in open(os.path.abspath(os.path.join(os.path.dirname(__file__), "lib/stopwords/zh-stopwords"))),"r")]
stoplist+=[i.strip() for i in open(os.path.abspath(os.path.join(os.path.dirname(__file__), "lib/stopwords/stopwords.txt"))),"r")]
stoplist+=["转发","微博","说 ","一个","【 ","年 ","转 ","请","＂ ","问题","知道","中 ","已经","现在","说","【",'＂',"年","中","今天","应该","真的","月","希望","想","日","这是","太","转","支持"]
# stoplist+=["事儿","中国"]

api=UserAPI()
words_users_time=[]

def get_province(_userid):
    province_code= api.get_province(_userid)
    # print province_code
    try :
        return api.provinces[province_code]
    except KeyError :
        return 0

def test_com(meme_id):
    print meme_id

# Connect to Mongo
db=MongoDB("weibodata").db

def analyze_meme(meme_id,collection):

    # Init
    tstart=time()
{'fraction_non_alphanumeric': 0.19271051529116046, 'fraction_numerical': 0.0025136154168412233, 'mean_word_length': 5.2513089005235605, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '864334', 'n_tokens_mistral': 928, 'n_tokens_neox': 937, 'n_words': 184}