<commit_before>import sqlite3

import requests

from .HashStore import LocationService, MimetypeService, IPLDHashService
from .Multihash import Multihash
from .NameResolver import NameResolverDir, NameResolverFile, NameResolverSearchItem, NameResolverSearch
from .miscutils import httpget
from .Errors import SearchException

class DOI(NameResolverDir):
    """
    DOI is a class for Document Object Identifiers, an academic namespace
    typically of form   10.1234/abc123
    URL: `/xxx/doi/10.pub_id/pub_specific_id` (forwarded here by ServerGateway methods)

    Implements name resolution of the DOI namespace, via a sqlite database (provided by <NAME>)

    Case insensitive, have some significant punctuation, but sometimes presented with insignificant punctuation

    Future Work
    * Build way to preload the hashstore with the hashes and URLs from the sqlite

    TODO - ssome of this will end up in NameResolverDir as we build other classe and see commonalities

    """

    """
    $ cd data; sqlite3 idents_files_urls.sqlite .schema
    CREATE TABLE files_id_doi (doi text not null, sha1 char(40) not null, type text);
    CREATE UNIQUE INDEX file_id_doi_sha1 on files_id_doi (sha1);
    CREATE INDEX files_doi on files_id_doi (doi);
    CREATE TABLE files_metadata (sha1 char(40) not null, mimetype text, size_bytes integer, md5 char(32));
    CREATE INDEX files_metadata_sha1 on files_metadata (sha1);
    CREATE TABLE urls (sha1 char(40) not null, url text not null, datetime integer);
    CREATE INDEX url_sha1 on urls (sha1);
    """

    # SQLITE="../data/idents_files_urls_sqlite"   # Old version in Python2 when working dir was "python"
    SQLITE = "data/idents_files_urls.sqlite"
    #_sqliteconnection=None  #TODO-SQL should be per thread

    def __init__(self, namespace, publisher, *identifier, **kwargs):
        """
        Creates the object

        :param namespace:
        :param publisher:   Publisher ID allocated by DOI.org, always of form 10.nnnn
        :param identifier:  Id for an academic article or resource, allocated by the publisher. Case insensitive, Alphanumeric plus a few (undefined) punctuation.
                            Its an array, because may be multiple fields seperated by /, can safely re-concatenate
        :param kwargs:      Any other args to the URL, ignored for now.
        """
        #Note if you dont have your own way of using sqlite I suggest SqliteWrap from https://github.com/mitra42/sqlite_models
        """
        Pseudo-code
        * Canonicalize pub_specific_id (lowercase, strip chars)
        * Look up in sqlite to find the [hash, location]* of the file (no need to retrieve it)
        * Convert hashes to multihashes
        * For each multihash: location_store(multihash, location)
        *   Create a DOIfile(..)
        *   self.push(DOIfile)
        """
        verbose = kwargs.get("verbose", False)
        if verbose: print("DOI.__init__", namespace, publisher, identifier)
        super(DOI, self).__init__(namespace, publisher, *identifier)
        db = self.sqliteconnection(verbose)                     # Lazy connection to database
        self.doi = self.canonical(publisher, *identifier)    # "10.nnnn/xxxx/yyyy"
        self._metadata = {}
        if verbose: print("DOI.__init__ getting metadata for", self.doi)
        self.doi_org_metadata = {}  # Will hold metadata retrieved from doi.org
        #self.get_doi_metadata(verbose)
        if verbose: print("DOI.__init__ looking up", self.doi)
        sha1_list = list(db.execute('SELECT * FROM files_id_doi WHERE doi = ?;', [self.doi]))

        if verbose: print("DOI.__init__ iterating over", len(sha1_list), "rows")
        for row in sha1_list:
            _, sha1_hex, _ = row
            doifile = DOIfile(doi=self.doi, multihash=Multihash(sha1_hex=sha1_hex), verbose=verbose)
            self.push(doifile)
        if verbose: print("DOI.__init__ completing")

    @classmethod
    def sqliteconnection(cls, verbose=False):
        """
        # This below wont work in a multithreaded server, as sqlite connections cant be used across threads and doesnt appear
        # to be a way to access thread variables.  #TODO-THREADING
        if not cls._sqliteconnection:
            if verbose: print("DOI.sqliteconnection connecting to DB")
            cls._sqliteconnection = sqlite3.connect(cls.SQLITE)
            if verbose: print("DOI.sqliteconnection connected to DB")
        return cls._sqliteconnection
        """
        # Using a new sqlite connection for each request
        return sqlite3.connect(cls.SQLITE)<commit_msg>Till fix cors need to fetch on server<commit_after>import sqlite3

import requests

from .HashStore import LocationService, MimetypeService, IPLDHashService
from .Multihash import Multihash
from .NameResolver import NameResolverDir, NameResolverFile, NameResolverSearchItem, NameResolverSearch
from .miscutils import httpget
from .Errors import SearchException

class DOI(NameResolverDir):
    """
    DOI is a class for Document Object Identifiers, an academic namespace
    typically of form   10.1234/abc123
    URL: `/xxx/doi/10.pub_id/pub_specific_id` (forwarded here by ServerGateway methods)

    Implements name resolution of the DOI namespace, via a sqlite database (provided by <NAME>)

    Case insensitive, have some significant punctuation, but sometimes presented with insignificant punctuation

    Future Work
    * Build way to preload the hashstore with the hashes and URLs from the sqlite

    TODO - ssome of this will end up in NameResolverDir as we build other classe and see commonalities

    """

    """
    $ cd data; sqlite3 idents_files_urls.sqlite .schema
    CREATE TABLE files_id_doi (doi text not null, sha1 char(40) not null, type text);
    CREATE UNIQUE INDEX file_id_doi_sha1 on files_id_doi (sha1);
    CREATE INDEX files_doi on files_id_doi (doi);
    CREATE TABLE files_metadata (sha1 char(40) not null, mimetype text, size_bytes integer, md5 char(32));
    CREATE INDEX files_metadata_sha1 on files_metadata (sha1);
    CREATE TABLE urls (sha1 char(40) not null, url text not null, datetime integer);
    CREATE INDEX url_sha1 on urls (sha1);
    """

    # SQLITE="../data/idents_files_urls_sqlite"   # Old version in Python2 when working dir was "python"
    SQLITE = "data/idents_files_urls.sqlite"
    #_sqliteconnection=None  #TODO-SQL should be per thread

    def __init__(self, namespace, publisher, *identifier, **kwargs):
        """
        Creates the object

        :param namespace:
        :param publisher:   Publisher ID allocated by DOI.org, always of form 10.nnnn
        :param identifier:  Id for an academic article or resource, allocated by the publisher. Case insensitive, Alphanumeric plus a few (undefined) punctuation.
                            Its an array, because may be multiple fields seperated by /, can safely re-concatenate
        :param kwargs:      Any other args to the URL, ignored for now.
        """
        #Note if you dont have your own way of using sqlite I suggest SqliteWrap from https://github.com/mitra42/sqlite_models
        """
        Pseudo-code
        * Canonicalize pub_specific_id (lowercase, strip chars)
        * Look up in sqlite to find the [hash, location]* of the file (no need to retrieve it)
        * Convert hashes to multihashes
        * For each multihash: location_store(multihash, location)
        *   Create a DOIfile(..)
        *   self.push(DOIfile)
        """
        verbose = kwargs.get("verbose", False)
        if verbose: print("DOI.__init__", namespace, publisher, identifier)
        super(DOI, self).__init__(namespace, publisher, *identifier)
        db = self.sqliteconnection(verbose)                     # Lazy connection to database
        self.doi = self.canonical(publisher, *identifier)    # "10.nnnn/xxxx/yyyy"
        self._metadata = {}
        if verbose: print("DOI.__init__ getting metadata for", self.doi)
        self.doi_org_metadata = {}  # Will hold metadata retrieved from doi.org
        self.get_doi_metadata(verbose)
        if verbose: print("DOI.__init__ looking up", self.doi)
        sha1_list = list(db.execute('SELECT * FROM files_id_doi WHERE doi = ?;', [self.doi]))

        if verbose: print("DOI.__init__ iterating over", len(sha1_list), "rows")
        for row in sha1_list:
            _, sha1_hex, _ = row
            doifile = DOIfile(doi=self.doi, multihash=Multihash(sha1_hex=sha1_hex), verbose=verbose)
            self.push(doifile)
        if verbose: print("DOI.__init__ completing")

    @classmethod
    def sqliteconnection(cls, verbose=False):
        """
        # This below wont work in a multithreaded server, as sqlite connections cant be used across threads and doesnt appear
        # to be a way to access thread variables.  #TODO-THREADING
        if not cls._sqliteconnection:
            if verbose: print("DOI.sqliteconnection connecting to DB")
            cls._sqliteconnection = sqlite3.connect(cls.SQLITE)
            if verbose: print("DOI.sqliteconnection connected to DB")
        return cls._sqliteconnection
        """
        # Using a new sqlite connection for each request
        return sqlite3.connect(cls.SQLITE)
{'fraction_non_alphanumeric': 0.07774043152987098, 'fraction_numerical': 0.00997506234413965, 'mean_word_length': 3.027947598253275, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2429582', 'n_tokens_mistral': 2721, 'n_tokens_neox': 2533, 'n_words': 912}