<commit_before>import os
import json
import time, datetime
from backend.app import app, db

from backend import models
from backend.models import *
import parsers
from sqlalchemy import types

import requests

import logging
logger = logging.getLogger(__name__)
logging.getLogger('sqlalchemy.engine').level = logging.WARN

STATIC_HOST = app.config['STATIC_HOST']
db.echo = False

def strip_filpath(filepath):

    return "/".join(filepath.split("/")[1::])


def dump_db(name):
    try:
        os.remove(name)
    except Exception as e:
        logger.debug(e)
        pass
    return


def read_data(filename):
    start = time.time()

    logger.debug("reading " + filename)
    with open('data/' + filename, 'r') as f:
        records = []
        lines = f.readlines()
        for line in lines:
            records.append(json.loads(line))
    return records

def db_date_from_utime(utime):
    try:
        return datetime.datetime.fromtimestamp(float(utime))
    except:
        try:
            datetime.strptime(utime, '%Y-%m-%dT%H:%m:%s')
        except:
            return None

def construct_obj(obj, mappings):
    """
    Returns a result with the properties mapped by mapping, including all revisions
    """
    result_obj = {}
    # print obj
    for key in mappings.keys():
        if (obj.has_key(key)):
            result_obj[key] = obj[key]
    if (obj.has_key("revisions") and (type(obj["revisions"]) is list) and (len(obj["revisions"]) > 0)):
        for revision in obj["revisions"]:
            tmp = construct_obj(revision, mappings)
            result_obj.update(tmp)
    return result_obj

def find_files(obj):
    files = []
    if (obj.has_key("files") and (type(obj["files"]) is list) and (len(obj["files"]) > 0)):
        # print obj["files"]
        for f in obj["files"]:
            fobj = File(
                    filemime=f["filemime"],
                    origname = f["origname"],
                    url = "http://eu-west-1-pmg.s3-website-eu-west-1.amazonaws.com/" + f["filepath"],
                )
            db.session.add(fobj)
            files.append(fobj)
        db.session.commit()
    if (obj.has_key("audio") and (type(obj["audio"]) is list) and (len(obj["audio"]) > 0)):
        # print obj["audio"]
        for f in obj["audio"]:
            fobj = File(
                    filemime=f["filemime"],
                    origname = f["origname"],
                    url = "http://eu-west-1-pmg.s3-website-eu-west-1.amazonaws.com/audio/" + f["filename"],
                    playtime = f["playtime"],
                    description = f["title_format"]
                )
            db.session.add(fobj)
            files.append(fobj)
        db.session.commit()
    return files

def find_committee(obj):
    organisations = []<commit_msg>Remove files from urls and add audio for audio, and don't include cdn<commit_after>import os
import json
import time, datetime
from backend.app import app, db

from backend import models
from backend.models import *
import parsers
from sqlalchemy import types

import requests

import logging
logger = logging.getLogger(__name__)
logging.getLogger('sqlalchemy.engine').level = logging.WARN

STATIC_HOST = app.config['STATIC_HOST']
db.echo = False

def strip_filpath(filepath):

    return "/".join(filepath.split("/")[1::])


def dump_db(name):
    try:
        os.remove(name)
    except Exception as e:
        logger.debug(e)
        pass
    return


def read_data(filename):
    start = time.time()

    logger.debug("reading " + filename)
    with open('data/' + filename, 'r') as f:
        records = []
        lines = f.readlines()
        for line in lines:
            records.append(json.loads(line))
    return records

def db_date_from_utime(utime):
    try:
        return datetime.datetime.fromtimestamp(float(utime))
    except:
        try:
            datetime.strptime(utime, '%Y-%m-%dT%H:%m:%s')
        except:
            return None

def construct_obj(obj, mappings):
    """
    Returns a result with the properties mapped by mapping, including all revisions
    """
    result_obj = {}
    # print obj
    for key in mappings.keys():
        if (obj.has_key(key)):
            result_obj[key] = obj[key]
    if (obj.has_key("revisions") and (type(obj["revisions"]) is list) and (len(obj["revisions"]) > 0)):
        for revision in obj["revisions"]:
            tmp = construct_obj(revision, mappings)
            result_obj.update(tmp)
    return result_obj

def find_files(obj):
    files = []
    if (obj.has_key("files") and (type(obj["files"]) is list) and (len(obj["files"]) > 0)):
        # print obj["files"]
        for f in obj["files"]:
            fobj = File(
                    filemime=f["filemime"],
                    origname = f["origname"],
                    url = f["filepath"].replace("files/", ""),
                )
            db.session.add(fobj)
            files.append(fobj)
        db.session.commit()
    if (obj.has_key("audio") and (type(obj["audio"]) is list) and (len(obj["audio"]) > 0)):
        # print obj["audio"]
        for f in obj["audio"]:
            fobj = File(
                    filemime=f["filemime"],
                    origname = f["origname"],
                    url = "audio/" + f["filename"].replace("files/", ""),
                    playtime = f["playtime"],
                    description = f["title_format"]
                )
            db.session.add(fobj)
            files.append(fobj)
        db.session.commit()
    return files

def find_committee(obj):
    organisations = []
{'fraction_non_alphanumeric': 0.13593104686658286, 'fraction_numerical': 0.002513916322499551, 'mean_word_length': 2.173789173789174, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2522161', 'n_tokens_mistral': 1730, 'n_tokens_neox': 1658, 'n_words': 401}