<commit_before>#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
    Template for new modules
"""

import time
import datetime
import redis
import os
from packages import lib_words
from packages.Date import Date
from pubsublogger import publisher
from Helper import Process
from pyfaup.faup import Faup

# Config Var
threshold_total_sum = 200 # Above this value, a keyword is eligible for a progression
threshold_increase = 1.0  # The percentage representing the keyword occurence since num_day_to_look
max_set_cardinality = 10  # The cardinality of the progression set
num_day_to_look = 5       # the detection of the progression start num_day_to_look in the past

    field = url_parsed[field_name]
    if field is not None:
        prev_score = server.hget(field, date)
        if prev_score is not None:
            print field + ' prev_score:' + prev_score
            server.hset(field, date, int(prev_score) + 1)
            if int(prev_score) + 1 > threshold_need_to_look:  # threshold for false possitive
                if(check_for_progression(server, field, date)):
                    to_plot.add(field)
        else:
            server.hset(field, date, 1)
            if field_name == "domain": #save domain in a set for the monthly plot
                domain_set_name = "domain_set_" + date[0:6]
                server.sadd(domain_set_name, field)
                print "added in " + domain_set_name +": "+ field

def get_date_range(num_day):
    curr_date = datetime.date.today()
    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
    date_list = []

    for i in range(0, num_day+1):
        date_list.append(date.substract_day(i))
    return date_list

def compute_progression(server, field_name, num_day, url_parsed):
    redis_progression_name = 'top_progression_'+field_name
    redis_progression_name_set = 'top_progression_'+field_name+'_set'

    keyword = url_parsed[field_name]
    if keyword is not None:
        date_range = get_date_range(num_day) 
        # check if this keyword is eligible for progression
        keyword_total_sum = 0 
        value_list = []
        for date in date_range:
            curr_value = server.hget(keyword, date)
            value_list.append(int(curr_value if curr_value is not None else 0))<commit_msg>Fixed bug introduced in merge-conflict<commit_after>#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
    Template for new modules
"""

import time
import datetime
import redis
import os
from packages import lib_words
from packages.Date import Date
from pubsublogger import publisher
from Helper import Process
from pyfaup.faup import Faup

# Config Var
threshold_total_sum = 200 # Above this value, a keyword is eligible for a progression
threshold_increase = 1.0  # The percentage representing the keyword occurence since num_day_to_look
max_set_cardinality = 10  # The cardinality of the progression set
num_day_to_look = 5       # the detection of the progression start num_day_to_look in the past

def analyse(server, field_name, date, url_parsed):
    field = url_parsed[field_name]
    if field is not None:
        prev_score = server.hget(field, date)
        if prev_score is not None:
            server.hset(field, date, int(prev_score) + 1)
        else:
            server.hset(field, date, 1)
            if field_name == "domain": #save domain in a set for the monthly plot
                domain_set_name = "domain_set_" + date[0:6]
                server.sadd(domain_set_name, field)
                print "added in " + domain_set_name +": "+ field

def get_date_range(num_day):
    curr_date = datetime.date.today()
    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
    date_list = []

    for i in range(0, num_day+1):
        date_list.append(date.substract_day(i))
    return date_list

def compute_progression(server, field_name, num_day, url_parsed):
    redis_progression_name = 'top_progression_'+field_name
    redis_progression_name_set = 'top_progression_'+field_name+'_set'

    keyword = url_parsed[field_name]
    if keyword is not None:
        date_range = get_date_range(num_day) 
        # check if this keyword is eligible for progression
        keyword_total_sum = 0 
        value_list = []
        for date in date_range:
            curr_value = server.hget(keyword, date)
            value_list.append(int(curr_value if curr_value is not None else 0))
{'fraction_non_alphanumeric': 0.08091706001348618, 'fraction_numerical': 0.009215554057091482, 'mean_word_length': 3.1549953314659196, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1625177', 'n_tokens_mistral': 1420, 'n_tokens_neox': 1363, 'n_words': 390}