<commit_before>#!/usr/bin/env python
import pymongo
import psycopg2
import json

client = pymongo.MongoClient()
db = client['penguin']
classification_collection = db["penguin_classifications"]
subject_collection = db["penguin_subjects"]

# conn = psycopg2.connect("dbname='postgres' user='postgres' host='localhost' password='<PASSWORD>'")
# conn.autocommit = True
# cur = conn.cursor()
# cur.execute("create database penguins")

conn = psycopg2.connect("dbname='penguins' user='postgres' host='localhost' password='<PASSWORD>'")
conn.autocommit = True
cur = conn.cursor()
# cur.execute("drop table classifications")
# cur.execute("create table classifications (zooniverse_id text, user_id text, annotations json, PRIMARY KEY(zooniverse_id, user_id))")
cur.execute("create index ids_ on classifications (zooniverse_id ASC)")

# for ii,classification in enumerate(classification_collection.find()):
#
#     zooniverse_id = classification["subjects"][0]["zooniverse_id"]
#     if ii % 100 == 0:
#         print(ii)
#
#     if "user_name" in classification:
#         id_ = classification["user_name"]
#         id_ = id_.encode('ascii','ignore')
#         id_ = id_.replace("'","")
#     else:
#         id_ = classification["user_ip"]
#
#     if "finished_at" in classification["annotations"][1]:
#         continue
#
#     annotations = json.dumps(classification["annotations"])
#     # annotations = annotations.replace("\"","")
#     annotations = annotations.replace("'","")
#     try:
#         cur.execute("insert into classifications values ('"+str(zooniverse_id)+"','"+str(id_)+"','"+annotations + "')")
#     except psycopg2.IntegrityError as e:
#         pass
#
# conn.commit()<commit_msg>uncommented a bunch of lines<commit_after>#!/usr/bin/env python
import pymongo
import psycopg2
import json

client = pymongo.MongoClient()
db = client['penguin']
classification_collection = db["penguin_classifications"]
subject_collection = db["penguin_subjects"]

conn = psycopg2.connect("dbname='postgres' user='postgres' host='localhost' password='<PASSWORD>'")
conn.autocommit = True
cur = conn.cursor()
cur.execute("create database penguins")

conn = psycopg2.connect("dbname='penguins' user='postgres' host='localhost' password='<PASSWORD>'")
conn.autocommit = True
cur = conn.cursor()

cur.execute("create table classifications (zooniverse_id text, user_id text, annotations json, PRIMARY KEY(zooniverse_id, user_id))")
cur.execute("create index ids_ on classifications (zooniverse_id ASC)")

for ii,classification in enumerate(classification_collection.find()):

    zooniverse_id = classification["subjects"][0]["zooniverse_id"]
    if ii % 100 == 0:
        print(ii)

    if "user_name" in classification:
        id_ = classification["user_name"]
        id_ = id_.encode('ascii','ignore')
        id_ = id_.replace("'","")
    else:
        id_ = classification["user_ip"]

    if "finished_at" in classification["annotations"][1]:
        continue

    annotations = json.dumps(classification["annotations"])
    # annotations = annotations.replace("\"","")
    annotations = annotations.replace("'","")
    try:
        cur.execute("insert into classifications values ('"+str(zooniverse_id)+"','"+str(id_)+"','"+annotations + "')")
    except psycopg2.IntegrityError as e:
        pass

conn.commit()
{'fraction_non_alphanumeric': 0.15393902807123452, 'fraction_numerical': 0.006036824630244492, 'mean_word_length': 4.939068100358423, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2203718', 'n_tokens_mistral': 1061, 'n_tokens_neox': 1009, 'n_words': 212}