<commit_before># -*- coding: utf-8 -*-

import hashlib
import binascii

from thrift.transport.THttpClient import THttpClient
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from evernote.edam.userstore import UserStore
from evernote.edam.notestore import NoteStore
import evernote.edam.type.ttypes as Types
import evernote.edam.error.ttypes as Errors

from evernote.api.client import EvernoteClient

from .settings import EVERNOTE_NOTEBOOK

import logging

class Sink(object):
    pass

class EvernoteSink(Sink):
    def __init__(self, token, sandbox=False):
        """Initialize evernote connection.
        Client connection handle is assigned to the client property.
        Two properties user_store and note_store are provided for the convenience.
        """
        self.token = token
        self.client = EvernoteClient(token=self.token, sandbox=sandbox)
        self.user_store = self.client.get_user_store()
        self.note_store = self.client.get_note_store()

    def image_resource(self, item):
        #FIXME create pdf resource
        md5 = hashlib.md5()
        md5.update(item.content)
        hashvalue = md5.digest()

        data = Types.Data()
        data.size = len(item.content)  #FIXME better ways of doing this calculation?
        data.bodyHash = hashvalue
        data.body = item.content

        resource = Types.Resource()
        resource.mime = item.content_type
        resource.data = data
        return resource

    def pdf_resource(self, item):
        #FIXME create pdf resource
        md5 = hashlib.md5()
        md5.update(item.content)
        hashvalue = md5.digest()

        data = Types.Data()
        data.size = len(item.content)  #FIXME better ways of doing this calculation?
        data.bodyHash = hashvalue
        data.body = item.content

        resource = Types.Resource()
        resource.mime = 'application/pdf'
        resource.data = data
        return resource

    def note_attribute(self, source_url=''):
        attribute = Types.NoteAttributes()
        attribute.sourceURL = source_url
        return attribute

    def create_note(self, title, content, notebook_name='', tags='', attribute=None, resources=None):
        note = Types.Note()
        note.title = title
        if attribute:
            note.attribute = attribute
        if tags:
            note.tagNames = tags.split()  # Assuming no spaces in tags

        if notebook_name:
            notebooks = self.note_store.listNotebooks(self.token)
            for notebook in notebooks:
                if notebook.name == notebook_name:
                    note.notebookGuid = notebook.guid
                    break
            else:
                pass  # create a note in default notebook

        note.content = """<?xml version="1.0" encoding="UTF-8"?>
        <!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
        <en-note>{}""".format(content.encode('ascii', 'xmlcharrefreplace'))

        if resources:
            note.resources = resources
            for r in resources:
                note.content += """<en-media type="{}" hash="{}"/>""".format(r.mime, binascii.hexlify(r.data.bodyHash))

        note.content += "</en-note>"

        logging.debug(note.content)

        created_note = self.note_store.createNote(self.token, note)
        return created_note

    def push(self, item):
        kwargs = {
            'title': item.title.encode('utf-8', 'xmlcharrefreplace'),
            'content': item.body,
            'tags': item.tags,
            'notebook_name': EVERNOTE_NOTEBOOK,
        }
<commit_msg>Source url saved to Evernote<commit_after># -*- coding: utf-8 -*-

import hashlib
import binascii

from thrift.transport.THttpClient import THttpClient
from thrift.protocol.TBinaryProtocol import TBinaryProtocol
from evernote.edam.userstore import UserStore
from evernote.edam.notestore import NoteStore
import evernote.edam.type.ttypes as Types
import evernote.edam.error.ttypes as Errors

from evernote.api.client import EvernoteClient

from .settings import EVERNOTE_NOTEBOOK

import logging

class Sink(object):
    pass

class EvernoteSink(Sink):
    def __init__(self, token, sandbox=False):
        """Initialize evernote connection.
        Client connection handle is assigned to the client property.
        Two properties user_store and note_store are provided for the convenience.
        """
        self.token = token
        self.client = EvernoteClient(token=self.token, sandbox=sandbox)
        self.user_store = self.client.get_user_store()
        self.note_store = self.client.get_note_store()

    def image_resource(self, item):
        #FIXME create pdf resource
        md5 = hashlib.md5()
        md5.update(item.content)
        hashvalue = md5.digest()

        data = Types.Data()
        data.size = len(item.content)  #FIXME better ways of doing this calculation?
        data.bodyHash = hashvalue
        data.body = item.content

        resource = Types.Resource()
        resource.mime = item.content_type
        resource.data = data
        return resource

    def pdf_resource(self, item):
        #FIXME create pdf resource
        md5 = hashlib.md5()
        md5.update(item.content)
        hashvalue = md5.digest()

        data = Types.Data()
        data.size = len(item.content)  #FIXME better ways of doing this calculation?
        data.bodyHash = hashvalue
        data.body = item.content

        resource = Types.Resource()
        resource.mime = 'application/pdf'
        resource.data = data
        return resource

    def note_attribute(self, source_url=''):
        attributes = Types.NoteAttributes()
        attributes.sourceURL = source_url
        return attributes 

    def create_note(self, title, content, notebook_name='', tags='', attributes=None, resources=None):
        note = Types.Note()
        note.title = title
        if attributes:
            note.attributes =attributes 
        if tags:
            note.tagNames = tags.split()  # Assuming no spaces in tags

        if notebook_name:
            notebooks = self.note_store.listNotebooks(self.token)
            for notebook in notebooks:
                if notebook.name == notebook_name:
                    note.notebookGuid = notebook.guid
                    break
            else:
                pass  # create a note in default notebook

        note.content = """<?xml version="1.0" encoding="UTF-8"?>
        <!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">
        <en-note>{}""".format(content.encode('ascii', 'xmlcharrefreplace'))

        if resources:
            note.resources = resources
            for r in resources:
                note.content += """<en-media type="{}" hash="{}"/>""".format(r.mime, binascii.hexlify(r.data.bodyHash))

        note.content += "</en-note>"

        logging.debug(note.content)

        created_note = self.note_store.createNote(self.token, note)
        return created_note

    def push(self, item):
        kwargs = {
            'title': item.title.encode('utf-8', 'xmlcharrefreplace'),
            'content': item.body,
            'tags': item.tags,
            'notebook_name': EVERNOTE_NOTEBOOK,
            'attributes': self.note_attribute(item.url),
        }

{'fraction_non_alphanumeric': 0.10090734121528733, 'fraction_numerical': 0.00384932636788562, 'mean_word_length': 2.334097158570119, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 2, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7438663', 'n_tokens_mistral': 2069, 'n_tokens_neox': 1990, 'n_words': 508}