<commit_before># -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import gzip
from django.db import DataError

from papers.models import Paper
from papers.models import OaiSource
from papers.baremodels import BareOaiRecord
from papers.doi import doi_to_crossref_identifier
from papers.doi import doi_to_url

class OadoiAPI(object):
    """
    An interface to import an OAdoi dump into dissemin
    """
    def __init__(self):
        self.oadoi_source, _ = OaiSource.objects.get_or_create(
            identifier='oadoi_repo',
            defaults=
            {'name':'OAdoi',
            'oa':True,
            'priority':-10,
            'default_pubtype':'preprint'})

        self.crossref_source = OaiSource.objects.get(identifier='crossref')

    def load_dump(self, filename):
        """
        Reads a dump from the disk and loads it to the db
        """
        with gzip.open(filename, 'r') as f:
            headers = []
            for line in f:
                fields = line.decode('utf-8').strip().split(',')
                if not headers:
                    headers = fields
                    continue
                if len(fields) != len(headers):
                    continue
                self.create_oairecord(dict(zip(headers,fields)))

    def create_oairecord(self, record):
        """
        Given one line of the dump (represented as a dict),
        add it to the corresponding paper (if it exists)
        """
        doi = record['doi']
        paper = Paper.get_by_doi(doi)
        if not paper:
            print('no such paper for doi {doi}'.format(doi))
            return

        url = record['url']
        identifier='oadoi:'+url
        source = self.oadoi_source

        if record['host_type'] == 'publisher':
            url = doi_to_url(doi)
            identifier = doi_to_crossref_identifier(doi)
            source = self.crossref_source

        record = BareOaiRecord(
            paper=paper,
            doi=doi,
            pubtype=paper.doctype,
            source=source,
            identifier=identifier,
            splash_url=url,
            pdf_url=record['url'])
        try:
            paper.add_oairecord(record)
            paper.update_availability()<commit_msg>Fix bug in debug print<commit_after># -*- encoding: utf-8 -*-
from __future__ import unicode_literals

import gzip
from django.db import DataError

from papers.models import Paper
from papers.models import OaiSource
from papers.baremodels import BareOaiRecord
from papers.doi import doi_to_crossref_identifier
from papers.doi import doi_to_url

class OadoiAPI(object):
    """
    An interface to import an OAdoi dump into dissemin
    """
    def __init__(self):
        self.oadoi_source, _ = OaiSource.objects.get_or_create(
            identifier='oadoi_repo',
            defaults=
            {'name':'OAdoi',
            'oa':True,
            'priority':-10,
            'default_pubtype':'preprint'})

        self.crossref_source = OaiSource.objects.get(identifier='crossref')

    def load_dump(self, filename):
        """
        Reads a dump from the disk and loads it to the db
        """
        with gzip.open(filename, 'r') as f:
            headers = []
            for line in f:
                fields = line.decode('utf-8').strip().split(',')
                if not headers:
                    headers = fields
                    continue
                if len(fields) != len(headers):
                    continue
                self.create_oairecord(dict(zip(headers,fields)))

    def create_oairecord(self, record):
        """
        Given one line of the dump (represented as a dict),
        add it to the corresponding paper (if it exists)
        """
        doi = record['doi']
        paper = Paper.get_by_doi(doi)
        if not paper:
            print('no such paper for doi {doi}'.format(doi=doi))
            return

        url = record['url']
        identifier='oadoi:'+url
        source = self.oadoi_source

        if record['host_type'] == 'publisher':
            url = doi_to_url(doi)
            identifier = doi_to_crossref_identifier(doi)
            source = self.crossref_source

        record = BareOaiRecord(
            paper=paper,
            doi=doi,
            pubtype=paper.doctype,
            source=source,
            identifier=identifier,
            splash_url=url,
            pdf_url=record['url'])
        try:
            paper.add_oairecord(record)
            paper.update_availability()
{'fraction_non_alphanumeric': 0.09661728940968384, 'fraction_numerical': 0.0017687375635640063, 'mean_word_length': 1.8760330578512396, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7009811', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1220, 'n_words': 315}