<commit_before># -*- coding: utf-8 -*-

"""
## RETHINKDB
# Convert old rethinkdb schema
# into new cool and fancy json automatic documents
"""

from __future__ import absolute_import
import os
import glob
from restapi.resources.services.rethink import RethinkConnection, RDBquery
from restapi import get_logger
from rethinkdb import r
from rethinkdb.net import DefaultCursorEmpty
from datetime import datetime
from elasticsearch import Elasticsearch
from confs.config import args, UPLOAD_FOLDER

import logging
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)

ES_HOST = {"host": "el", "port": 9200}
EL_INDEX = "autocomplete"
STEPS = {}
TESTING = False
#TESTING = True

# Tables
t1 = "stepstemplate"
t2 = "steps"
t3 = "stepscontent"
t4 = "docs"
tin = "datakeys"
t2in = "datavalues"
t3in = "datadocs"
t4in = "datapending"

# Connection
RethinkConnection()
# Query main object
query = RDBquery()

######################
# Parameters
if args.rm:
    logger.info("Remove previous data")
    tables = query.list_tables()
    if tin in tables:
        query.get_query().table_drop(tin).run()
    if t2in in tables:
        query.get_query().table_drop(t2in).run()
    if t3in in tables:
        query.get_query().table_drop(t3in).run()
    # if t4in in tables:
    #     query.get_query().table_drop(t4in).run()


#################################
# MAIN
#################################
def convert_schema():
    """ Do all ops """

    ######################
    # Make tests
    if TESTING:
        test_query()
        # test_el()

    ######################
    # Conversion from old structure to the new one
    tables = query.list_tables()

    if tin not in tables:
        convert_submission()
    if t2in not in tables:
        convert_search()
    if t3in not in tables:
        convert_docs()
    # remove pending files...
    ##if t4in not in tables:
    convert_pending_images()

    # check_indexes(t2in)

#################################
#################################


def convert_pending_images():
    """ Find images not linked to documents """

    q = query.get_table_query(t3in)
    cursor = q['images'] \
        .map(
            lambda images:
                {'file': images['filename'], 'record': images['recordid']}
         ).distinct().run()

    logger.info("Obtained pending data")
    images = glob.glob(os.path.join(UPLOAD_FOLDER, "*.jpg"))

    # Missing
    for obj in list(cursor):

        # Check if exists
        absfile = os.path.join(UPLOAD_FOLDER, obj['file'].pop())
        if absfile in images:
            images.pop(images.index(absfile))
        elif len(obj['record']) > 0:
            # Remove images which are not physical uploaded
            q.get(obj['record'].pop()).delete().run()
            logger.debug("Removed pending file '%s' from table", absfile)

# // TO FIX:
# MAKE A TABLE OF DRAFTS INSTEAD

    # REMOVE
    # # Remove physical images which do not belong to any record?
    # for image in images:
    #     os.unlink(image)
    #     logger.debug("Removed unused image '%s'" % image)

    # q = query.get_table_query(t4in)
    # q = query.get_table_query(t5in)
    # REMOVE


def split_and_html_strip(string):
    """ Compute words from transcriptions """
    words = []
    START = '<'
    END = '>'
    skip = False
    word = ""
    for char in string:
        if char == START:
            skip = True
            continue
        elif char == END:
            skip = False
            continue
        if skip:
            continue
        if char.isalpha():
            word += char
        elif word != "" and len(word) > 3:
            words.append(word)  # word.lower())
            word = ""

    return set(words)


def convert_docs():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t4)
    qtin = query.get_table_query(t3in)

    pkey = 'record'
    q = query.get_query()

    if t3in in list(q.table_list().run()):
        q.table_drop(t3in).run()
    q.table_create(t3in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t3in)

    # Query
    res = qt1.group('recordid').order_by('code').run()
    key = 'transcriptions'
    for record, rows in res.items():
        images = []
    # Check images
        for row in rows:
            if key in row:
                words = set()
    # Fix transcriptions
                for trans in row[key]:
                    words = words | split_and_html_strip(trans)
                row[key+'_split'] = list(words)
            images.append(row)

        # Insert
        qtin.insert({pkey: record, 'images': images}).run()
        logger.info("Insert of record '%s'" % record)


def convert_search():
    """ Convert Data needed for search """<commit_msg>Adding extra info on backend documents uploaded<commit_after># -*- coding: utf-8 -*-

"""
## RETHINKDB
# Convert old rethinkdb schema
# into new cool and fancy json automatic documents
"""

from __future__ import absolute_import
import os
import glob
from restapi.resources.services.rethink import RethinkConnection, RDBquery
from restapi.resources.custom.docs import image_destination
from restapi import get_logger
from rethinkdb import r
from rethinkdb.net import DefaultCursorEmpty
from datetime import datetime
from elasticsearch import Elasticsearch
from confs.config import args, UPLOAD_FOLDER

import logging
logger = get_logger(__name__)
logger.setLevel(logging.DEBUG)

ES_HOST = {"host": "el", "port": 9200}
EL_INDEX = "autocomplete"
STEPS = {}
TESTING = False
#TESTING = True

# Tables
t1 = "stepstemplate"
t2 = "steps"
t3 = "stepscontent"
t4 = "docs"
tin = "datakeys"
t2in = "datavalues"
t3in = "datadocs"
t4in = "datapending"

# Connection
RethinkConnection()
# Query main object
query = RDBquery()

######################
# Parameters
if args.rm:
    logger.info("Remove previous data")
    tables = query.list_tables()
    if tin in tables:
        query.get_query().table_drop(tin).run()
    if t2in in tables:
        query.get_query().table_drop(t2in).run()
    if t3in in tables:
        query.get_query().table_drop(t3in).run()
    # if t4in in tables:
    #     query.get_query().table_drop(t4in).run()


#################################
# MAIN
#################################
def convert_schema():
    """ Do all ops """

    ######################
    # Make tests
    if TESTING:
        test_query()
        # test_el()

    ######################
    # Conversion from old structure to the new one
    tables = query.list_tables()

    if tin not in tables:
        convert_submission()
    if t2in not in tables:
        convert_search()
    if t3in not in tables:
        convert_docs()
    # remove pending files...
    ##if t4in not in tables:
    convert_pending_images()

    # check_indexes(t2in)

#################################
#################################


def convert_pending_images():
    """ Find images not linked to documents """

    q = query.get_table_query(t3in)
    cursor = q['images'] \
        .map(
            lambda images:
                {'file': images['filename'], 'record': images['recordid']}
         ).distinct().run()

    logger.info("Obtained pending data")
    images = glob.glob(os.path.join(UPLOAD_FOLDER, "*.jpg"))

    # Missing
    for obj in list(cursor):

        # Check if exists
        absfile = os.path.join(UPLOAD_FOLDER, obj['file'].pop())
        if absfile in images:
            images.pop(images.index(absfile))
        elif len(obj['record']) > 0:
            # Remove images which are not physical uploaded
            q.get(obj['record'].pop()).delete().run()
            logger.debug("Removed pending file '%s' from table", absfile)

# // TO FIX:
# MAKE A TABLE OF DRAFTS INSTEAD

    # REMOVE
    # # Remove physical images which do not belong to any record?
    # for image in images:
    #     os.unlink(image)
    #     logger.debug("Removed unused image '%s'" % image)

    # q = query.get_table_query(t4in)
    # q = query.get_table_query(t5in)
    # REMOVE


def split_and_html_strip(string):
    """ Compute words from transcriptions """
    words = []
    START = '<'
    END = '>'
    skip = False
    word = ""
    for char in string:
        if char == START:
            skip = True
            continue
        elif char == END:
            skip = False
            continue
        if skip:
            continue
        if char.isalpha():
            word += char
        elif word != "" and len(word) > 3:
            words.append(word)  # word.lower())
            word = ""

    return set(words)


def convert_docs():
    """ Convert Data needed for search """
    qt1 = query.get_table_query(t4)
    qtin = query.get_table_query(t3in)

    pkey = 'record'
    q = query.get_query()

    if t3in in list(q.table_list().run()):
        q.table_drop(t3in).run()
    q.table_create(t3in, primary_key=pkey).run()
    logger.info("Startup table '%s'" % t3in)

    # Query
    res = qt1.group('recordid').order_by('code').run()
    key = 'transcriptions'
    for record, rows in res.items():
        images = []
    # Check images
        for row in rows:
            if key in row:
                words = set()
    # Fix transcriptions
                for trans in row[key]:
                    words = words | split_and_html_strip(trans)
                row[key+'_split'] = list(words)
            images.append(row)

        # Insert
        qtin.insert({
            pkey: record,
            'images': images,
            'type': image_destination({})
        }).run()
        logger.info("Insert of record '%s'" % record)


def convert_search():
    """ Convert Data needed for search """
{'fraction_non_alphanumeric': 0.15530656707127297, 'fraction_numerical': 0.007262164124909223, 'mean_word_length': 2.538913362701909, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5247822', 'n_tokens_mistral': 3002, 'n_tokens_neox': 2909, 'n_words': 794}