<commit_before>#!/usr/bin/env python
# Given a complete year files with data in the form (page, week, visits)
# this script will generate a convenient csv file which will store for
# each page and for each years's week the total number of visits.
#
# Written by <NAME> (2017)
# Email: giovanni.det at gmail.com

"""Generate year files with page visits
Usage:
  generate_weekly_data.py <input> <output> [-c <columns>]

Options:
  -c    specify custom columns name.
  -h, --help
"""

import fileinput
import pandas as pd
import datetime
from docopt import docopt

# Parse the command line
arguments = docopt(__doc__)

# Set up an empty dictionary
all_data={}

# If the columns are set, add them to the dataframe
columns_df = []
with open(argumens["<columns>"], "r") as f:
    for line in f:
        all_data[line]=[]

# Future index
index_year=[];

# Read from file
with open(arguments["<input>"], "r") as f:
    for line in f:

        # Split the line given
        # 0: page name
        # 1: date-hour
        # 2: visits count
        # 3: page size
        total = line.split(",")

        # Get date-hour pair
        # 0: date
        # 1: hour
        date = total[1].split("-")

        # Generate year month and day.
        year = date[0][0:4]
        month = date[0][4:6]
        day = date[0][6:8]

        # Get week number
        week_number = datetime.date(int(year), int(month), int(day)).isocalendar()[1]

        # Set up an empty list if the key
        # is null
        if all_data.get(total[0], []) == []:
            all_data[total[0]] = [0 for x in range(53)]

        # Sum the visits
        all_data[total[0]][int(week_number)-1] += int(total[2]);


for i in range(0, 53):
	if i<10:
		number="0"+str(i)
	else:
		number=str(i)
	index_year.append(year+"-"+number)

# Generate a pandas dataframe with all the data
print index_year
df = pd.DataFrame(all_data);
df = df.set_index([index_year]);

# Print the dataframe to show the result
print(df)

# Save it to file<commit_msg>[generate_weekly_data] Fix errors (new lines and dictionary initialization).<commit_after>#!/usr/bin/env python
# Given a complete year files with data in the form (page, week, visits)
# this script will generate a convenient csv file which will store for
# each page and for each years's week the total number of visits.
#
# Written by <NAME> (2017)
# Email: giovanni.det at gmail.com

"""Generate year files with page visits
Usage:
  generate_weekly_data.py <input> <output> [-c <columns>]

Options:
  -c    specify custom columns name.
  -h, --help
"""

import fileinput
import pandas as pd
import datetime
from docopt import docopt

# Parse the command line
arguments = docopt(__doc__)

# Set up an empty dictionary
all_data={}

# If the columns are set, add them to the dataframe
columns_df = []
with open(arguments["<columns>"], "r") as f:
    for line in f:
        all_data[line.replace("\n", "")]=[0 for x in range(53)]

# Future index
index_year=[];

# Read from file
with open(arguments["<input>"], "r") as f:
    for line in f:

        # Split the line given
        # 0: page name
        # 1: date-hour
        # 2: visits count
        # 3: page size
        total = line.split(",")

        # Get date-hour pair
        # 0: date
        # 1: hour
        date = total[1].split("-")

        # Generate year month and day.
        year = date[0][0:4]
        month = date[0][4:6]
        day = date[0][6:8]

        # Get week number
        week_number = datetime.date(int(year), int(month), int(day)).isocalendar()[1]

        # Set up an empty list if the key
        # is null
        if all_data.get(total[0], []) == []:
            all_data[total[0]] = [0 for x in range(53)]

        # Sum the visits
        all_data[total[0]][int(week_number)-1] += int(total[2]);


for i in range(1, 54):
	if i<10:
		number="0"+str(i)
	else:
		number=str(i)
	index_year.append(year+"-"+number)

# Generate a pandas dataframe with all the data
print index_year
df = pd.DataFrame(all_data);
df = df.set_index([index_year]);

# Print the dataframe to show the result
print(df)

# Save it to file
{'fraction_non_alphanumeric': 0.1272549975621648, 'fraction_numerical': 0.017796196977084348, 'mean_word_length': 2.799074074074074, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2476101', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1333, 'n_words': 473}