<commit_before>'''Netlify comments processing script.'''
import os
import json
import pathlib
import hashlib
from collections import OrderedDict

import yaml
import requests
from cryptography.fernet import Fernet


SECRET = os.environ['SECRET']
COMMENT_DIR = os.environ.get('COMMENT_DIR', './_data/comments')

ACCESS_TOKEN = os.environ.get('NETLIFY_ACCESS_TOKEN')
FORM_ID = os.environ.get('NETLIFY_FORM_ID')
SITE_ID = os.environ.get('NETLIFY_SITE_ID')

BASE = 'https://api.netlify.com/api/v1'
SITE = f'{BASE}/sites/{SITE_ID}'
SUBMISSIONS_ENDPOINT = f'{SITE}/forms/{FORM_ID}/submissions?access_token={ACCESS_TOKEN}'


def get_comments():
    '''Get a map of post_uuid => list of comment dicts.'''
    raw_comments = requests.get(f'{SITE}/forms/{FORM_ID}/submissions',
                                params={'access_token': ACCESS_TOKEN})
    comments = json.loads(raw_comments.content.decode('utf-8'))
    comments.sort(key=lambda x: x['number'])
    result = OrderedDict()
    for comment in comments:
        key = comment['data']['page_uuid']
        if key not in result:
            result[key] = []
        result[key].append(comment)
    return result


def transform_comment(netlify_comment):
    '''Convert Netlify form data to a normal comment.'''
    data = netlify_comment['data']
    return {'page_id': data['page_id'],
            'page_uuid': data['page_uuid'],
            'page_date': data['page_date'],
            'page_title': data['page_title'],
            'date': netlify_comment['created_at'],
            'name': data['name'],
            'email': hashlib.md5(data['email'].encode('ascii')).hexdigest(),
            'bucket': encrypt(data['email']),
            'website': data['website'],
            'message': data['message']}


def encrypt(data):
	'''Encrypt the data using SECRET key.'''
	f = Fernet(SECRET)
	s = json.dumps(data)
	return f.encrypt(s.encode('utf8'))


def decrypt(text):
	'''Decrypt the text using SECRET.'''
	f = Fernet(SECRET)
	s = f.decrypt(f)
	return json.loads(s.decode('utf8'))


def update_comments(file, comments):
    '''Update comments in the YAML data files of each post.'''
    file.seek(0)
    old_comments = yaml.load(file) or []
    # Use comment date as ID
    old_comment_ids = [cmnt['date'] for cmnt in old_comments]
    new_comments = list(filter(lambda x: x['date'] not in old_comment_ids,
        map(transform_comment, comments)))
    if new_comments:
        yaml.dump(new_comments,
            file,
            default_flow_style=False,
            allow_unicode=True)


def main():
    '''Update comments.'''
    netlify_comments = get_comments()
    pathlib.Path(COMMENT_DIR).mkdir(parents=True, exist_ok=True)
    for page_uuid, page_comments in netlify_comments.items():
        if page_comments:
            uid = os.path.basename(page_comments[0]['data']['page_id'])
            with open(os.path.join(COMMENT_DIR, f'{uid}.yml'), 'a+', encoding='utf8') as file:
                update_comments(file, page_comments)


if __name__ == '__main__':
    main()
<commit_msg>Fix decrypt.<commit_after>'''Netlify comments processing script.'''
import os
import json
import pathlib
import hashlib
from collections import OrderedDict

import yaml
import requests
from cryptography.fernet import Fernet


SECRET = os.environ['SECRET']
COMMENT_DIR = os.environ.get('COMMENT_DIR', './_data/comments')

ACCESS_TOKEN = os.environ.get('NETLIFY_ACCESS_TOKEN')
FORM_ID = os.environ.get('NETLIFY_FORM_ID')
SITE_ID = os.environ.get('NETLIFY_SITE_ID')

BASE = 'https://api.netlify.com/api/v1'
SITE = f'{BASE}/sites/{SITE_ID}'
SUBMISSIONS_ENDPOINT = f'{SITE}/forms/{FORM_ID}/submissions?access_token={ACCESS_TOKEN}'


def get_comments():
    '''Get a map of post_uuid => list of comment dicts.'''
    raw_comments = requests.get(f'{SITE}/forms/{FORM_ID}/submissions',
                                params={'access_token': ACCESS_TOKEN})
    comments = json.loads(raw_comments.content.decode('utf-8'))
    comments.sort(key=lambda x: x['number'])
    result = OrderedDict()
    for comment in comments:
        key = comment['data']['page_uuid']
        if key not in result:
            result[key] = []
        result[key].append(comment)
    return result


def transform_comment(netlify_comment):
    '''Convert Netlify form data to a normal comment.'''
    data = netlify_comment['data']
    return {'page_id': data['page_id'],
            'page_uuid': data['page_uuid'],
            'page_date': data['page_date'],
            'page_title': data['page_title'],
            'date': netlify_comment['created_at'],
            'name': data['name'],
            'email': hashlib.md5(data['email'].encode('ascii')).hexdigest(),
            'bucket': encrypt(data['email']),
            'website': data['website'],
            'message': data['message']}


def encrypt(data):
	'''Encrypt the data using SECRET key.'''
	f = Fernet(SECRET)
	s = json.dumps(data)
	return f.encrypt(s.encode('utf8'))


def decrypt(text):
	'''Decrypt the text using SECRET.'''
	f = Fernet(SECRET)
	s = f.decrypt(text.decode('utf8'))
	return json.loads(s)


def update_comments(file, comments):
    '''Update comments in the YAML data files of each post.'''
    file.seek(0)
    old_comments = yaml.load(file) or []
    # Use comment date as ID
    old_comment_ids = [cmnt['date'] for cmnt in old_comments]
    new_comments = list(filter(lambda x: x['date'] not in old_comment_ids,
        map(transform_comment, comments)))
    if new_comments:
        yaml.dump(new_comments,
            file,
            default_flow_style=False,
            allow_unicode=True)


def main():
    '''Update comments.'''
    netlify_comments = get_comments()
    pathlib.Path(COMMENT_DIR).mkdir(parents=True, exist_ok=True)
    for page_uuid, page_comments in netlify_comments.items():
        if page_comments:
            uid = os.path.basename(page_comments[0]['data']['page_id'])
            with open(os.path.join(COMMENT_DIR, f'{uid}.yml'), 'a+', encoding='utf8') as file:
                update_comments(file, page_comments)


if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.14597701149425288, 'fraction_numerical': 0.002627257799671593, 'mean_word_length': 3.3978339350180504, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '309391', 'n_tokens_mistral': 2005, 'n_tokens_neox': 1978, 'n_words': 423}