<commit_before>#!/usr/bin/env python

from collections import Counter
import os
import argparse
import subprocess
import datetime
import calendar
import random
import psycopg2
import requests
import urllib

REMOTE_PATH = 'https://dumps.wikimedia.org/other/pagecounts-raw/%(year)04d/%(year)04d-%(month)02d/pagecounts-%(year)04d%(month)02d%(day)02d-%(hour)02d0000.gz'
LOCAL_PATH = 'pagecounts-%(year)04d%(month)02d%(day)02d-%(hour)02d0000.gz'

def setup_db(connection_string):
  conn = psycopg2.connect(connection_string)
  cursor = conn.cursor()
  cursor.execute('DROP TABLE IF EXISTS wikistats')
  cursor.execute('CREATE TABLE wikistats ('
                 '    title TEXT PRIMARY KEY,'
                 '    viewcount INTEGER'
                 ')')
  return conn, cursor


def fetch_dumps(dump_dir, dumps_to_fetch):
  today = datetime.datetime.today()
  year = today.year
  if today.month <= 2:
    year -= 1
  if calendar.isleap(year):
    days = 366
  else:
    days = 365
  for i in range(dumps_to_fetch):
    local_path = None
    remote_path = None
    while not local_path or os.path.isdir(local_path):
      # pick a date from a week a go to a week and a year ago
      random_day = today - datetime.timedelta(days=random.randint(8, days + 7))
      random_hour = random.randint(1, 24)
      d = {'year': random_day.year, 'month': random_day.month, 'day': random_day.day, 'hour': random_hour}
      remote_path = REMOTE_PATH % d
      local_path = os.path.join(dump_dir, LOCAL_PATH % d)
    print 'getting', local_path
    data = requests.get(remote_path).content
    with file(local_path, 'wb') as fout:
      fout.write(data)


def main(dump_dir, cursor, dumps_to_fetch):<commit_msg>Import wikistat with offset of one month<commit_after>#!/usr/bin/env python

from collections import Counter
import os
import argparse
import subprocess
import datetime
import calendar
import random
import psycopg2
import requests
import urllib

REMOTE_PATH = 'https://dumps.wikimedia.org/other/pagecounts-raw/%(year)04d/%(year)04d-%(month)02d/pagecounts-%(year)04d%(month)02d%(day)02d-%(hour)02d0000.gz'
LOCAL_PATH = 'pagecounts-%(year)04d%(month)02d%(day)02d-%(hour)02d0000.gz'

def setup_db(connection_string):
  conn = psycopg2.connect(connection_string)
  cursor = conn.cursor()
  cursor.execute('DROP TABLE IF EXISTS wikistats')
  cursor.execute('CREATE TABLE wikistats ('
                 '    title TEXT PRIMARY KEY,'
                 '    viewcount INTEGER'
                 ')')
  return conn, cursor


def fetch_dumps(dump_dir, dumps_to_fetch):
  # don't try anything in the last month, it might not be online yet
  last_date = datetime.datetime.today() - datetime.timedelta(30)
  year = last_date.year
  if last_date.month <= 2:
    year -= 1
  if calendar.isleap(year):
    days = 366
  else:
    days = 365
  for i in range(dumps_to_fetch):
    local_path = None
    remote_path = None
    while not local_path or os.path.isdir(local_path):
      random_day = last_date - datetime.timedelta(days=random.randint(1, days))
      random_hour = random.randint(1, 24)
      d = {'year': random_day.year, 'month': random_day.month, 'day': random_day.day, 'hour': random_hour}
      remote_path = REMOTE_PATH % d
      local_path = os.path.join(dump_dir, LOCAL_PATH % d)
    print 'getting', local_path
    data = requests.get(remote_path).content
    with file(local_path, 'wb') as fout:
      fout.write(data)


def main(dump_dir, cursor, dumps_to_fetch):
{'fraction_non_alphanumeric': 0.1165750650853341, 'fraction_numerical': 0.02632340179346254, 'mean_word_length': 3.6416107382550336, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4177040', 'n_tokens_mistral': 1268, 'n_tokens_neox': 1174, 'n_words': 277}