<jupyter_start><jupyter_text>Abstract AttributesNot working yet.<jupyter_code>def abstractattrs(*attrs):
    def wrapper(cls):
        
        def method(cls, *args, **kwargs):
            super(cls, cls).__init_subclass__(*args, **kwargs)
            for attr in attrs:
                if not hasattr(cls, attr):
                    raise RuntimeError(f'Class must have attribute {attr}.') 
                    
        setattr(cls, '__init_subclass__', classmethod(method))
        return cls
    return wrapper              
@abstractattrs('name', 'age')
class Base:
    def __init__(self, age=7):
        self.age = age 
class Child(Base):
    a = 'a'
class AbstractAttrs:
    
    def __init__(self, *args):
        print(args)
        self.args = list(args)
    
    @classmethod
    def __init_subclass__(cls, **kwargs):
        super().__init_subclass__(**kwargs)
#         if not hasattr(cls, f'_{cls.__name__}__attrs'):
#             print('no attrs')
        print('init subclass', AbstractAttrs._args())
        for arg in AbstractAttrs.args:
            assert hasattr(cls, arg), \
                f'{cls.__name__} must have class variable {arg}.'
class Callback(AbstractAttrs):
    pass
c = Callback('order', 'is_val')
class NewCallback(Callback):
    
    def __init__(self, a, order=3):
        self.a = a
        self.order = order
nc = NewCallback('a', 'my_order')
nc
nc._attrs
from abc import ABC, abstractmethod, abstractproperty
class Abstract(ABC):
    def __init__(self, a):
        self.a = a
    @abstractmethod
    def walk(self):
        pass
    
    @abstractproperty
    def b(self):
        pass
class Child(Abstract):
    b = 3
        
    def walk(self):
        return self.a
    
#     def b(self):
#         print('in b, method not prop')
#         return self.a*2
c = Child(3)
c.b<jupyter_output><empty_output><jupyter_text>Metaclass approachThis seems more promising.<jupyter_code>from inspect import ismethod, getattr_static
def hasstatic(cls, meth_name):
    """Check if a class possesses a staticmethod of a given name. Similar to
    hasattr
    
    Parameters
    ----------
    cls: Type or any
        A class or an instance (seems to work on both, though more extensive
        testing may be needed for more complex scenarios).
    meth_name: str
        Name of method to check. If the class/instance does not contain any
        attribute with this name, function returns False.
        
    Returns
    -------
    bool: True if `cls` has a staticmethod with name `meth_name`.
    """
    return isinstance(getattr_static(cls, meth_name, None), staticmethod)
class Foo:
    def __init__(self, a):
        self.a = a
    @staticmethod
    def bar(self):
        return 'bar'
hasstatic(Foo, 'bar')
hasstatic(Foo, '__init__')
hasstatic(Foo(3), 'bar')
hasstatic(Foo(3), '__init__')
hasstatic(Foo(3), 'fake')
class AbstractAttrs(type):
    """Basically the attribute equivalent of abc.abstractmethod: this allows 
    us to define an abstract parent class that requires its children to 
    possess certain class and/or instance attributes. This differs from
    abc.abstractproperty in a few ways:
    
    1. abstractproperty ignores instance attributes. AbstractAttrs lets us
    specify required instance attributes and/or class attributes and
    distinguish between the two.
    2. abstractproperty considers the requirement fulfilled by methods,
    properties, and class attributes. AbstractAttrs does not allow methods 
    (including classmethods and staticmethods) to fulfill either requirement,
    though properties can fulfill either.
    
    Examples
    --------
    This class defines required instance attributes and class attributes, 
    but you can also specify one or the other. If you don't care whether an
    attribute is at the class or instance level, you can simply use
    @abc.abstractproperty.
    
    class Parent(metaclass=AbstractAttrs, inst_attrs=['name'], 
                 class_attrs=['order', 'is_val']):
        pass
    """

    def __new__(cls, name, bases, methods, **meta_kwargs):
        """This provides user-defined parent classes with an 
        `__init_subclass__` method that checks for class attributes. Errors
        will occur when the parent class is defined, not when instances of it
        are constructed.
        """
        class_ = type.__new__(cls, name, bases, methods)
        class_attrs = meta_kwargs.get('class_attrs', [])
        inst_attrs = meta_kwargs.get('inst_attrs', [])
        
        def __init_subclass__(cls, **kwargs):
            super().__init_subclass__(**kwargs)
            for attr in class_attrs:
                # TypeError maintains consistency with abstractmethod.
                # Remaining checks occur at instantiation.
                if not hasattr(cls, attr):
                    raise TypeError(f'{cls} must have class attribute '
                                    f'`{attr}`.')
        
        # Make sure we distinguish between the abstract parent class that 
        # defines an interface and the child classes that implement it. The
        # abstract parent should not define the required attributes: it merely
        # enforces the requirement that its children do. We want the children
        # to inherit class_attrs and inst_attrs without overwriting them when
        # their own __new__ is called, so that AbstractAttrs.__call__ can
        # use them for validation. Only change this if you're very confident
        # you understand the repercussions.
        if class_attrs or inst_attrs:
            class_.__init_subclass__ = classmethod(__init_subclass__)
            class_._is_parent = True
            class_.class_attrs = class_attrs
            class_.inst_attrs = inst_attrs
        else:
            class_._is_parent = False
        return class_

    def __call__(cls, *args, **kwargs):
        """This is called when we create instances of our classes. Parents are
        initialized normally, while children undergo a series of checks for
        each of our required attributes.
        """
        inst = cls.__new__(cls, *args, **kwargs)
        if not isinstance(inst, cls): return inst
        
        inst.__init__(*args, **kwargs)
        if cls._is_parent: return inst
        
        # Validate children.
        for attr in inst.inst_attrs:
            # TypeError maintains consistency with abstractmethod.
            if not hasattr(inst, attr):
                raise TypeError(f'Instances of {type(inst)} must '
                                f'have instance attribute `{attr}`.')
            elif ismethod(getattr(inst, attr)):
                raise TypeError(f'`{attr}` must be an instance attribute, '
                                'not a method.')
                
        # In AbstractAttrs.__new__, methods are still unbound so we couldn't
        # easily check this until now.
        for attr in inst.class_attrs:
            # `ismethod` must check inst, not cls (cls.method is a function 
            # while inst.method is a method). staticmethod can be retrieved 
            # from either.
            if ismethod(getattr(inst, attr)) or hasstatic(inst, attr):
                raise TypeError(f'`{attr}` must be a class attribute, not a '
                                'method.')
            elif isinstance(getattr(cls, attr), property):
                raise TypeError(
                    f'`{attr}` must be a class attribute, not a property. '
                    'Properties fulfill instance attribute requirements but '
                    'not class attribute requirements.'
                )
        return inst
class Parent(metaclass=AbstractAttrs, inst_attrs=['name'],
             class_attrs=['order', 'is_val']):
    pass
p = Parent()
p.class_attrs, p.inst_attrs, p._is_parent
# Implements required class vars but not required inst vars. Error will be
# thrown on object instantiation, not class definition.
class Child(Parent):
    
    order = 33
    is_val = True
    
    def __init__(self, a):
        self.a = a
        print('child init')
c = Child(4)
c
# Possesses one of the required class vars but the other is defined as an
# instance var. Throws error at class definition. If we want to accept either,
# we could use abc.abstractmethod or simply define our instance variables.
class BadChild(Parent):
    
    order = 1
    
    def __init__(self, b, is_val):
        self.b = b
        self.is_val = is_val
# Trying to sneakily use a method to fulfill an instance var requirement.
class TrickyChild(Parent):
    
    order = 33
    is_val = True
    
    def __init__(self, a):
        self.a = a
        print('child init')
        
    def name(self):
        print('in name')
tc = TrickyChild(44)
# Trying to use a property to fulfill an instance var requirement 
# (acceptable).
class TrickyChild2(Parent):
    
    order = 33
    is_val = True
    
    def __init__(self, a):
        self.a = a
        print('child init')
        
    @property
    def name(self):
        return 'name'
tc = TrickyChild2(44)
tc.name
TrickyChild2(1).name
TrickyChild2.name
# Trying to use a property to fulfill a class var requirement. UNacceptable.
class TrickyChild3(Parent):
    
    order = 33
    
    def __init__(self, a, name):
        self.a = a
        self.name = name
        print('child init')
        
    @property
    def is_val(self):
        return 'is val'
tc = TrickyChild3(44, 'sam')
# Trying to use a method to fulfill a class var req. UNacceptable.
class TrickyChild4(Parent):
    
    order = 33
    
    def __init__(self, a, name):
        self.a = a
        self.name = name
        print('child init')
        
    def is_val(self):
        print(self, type(self))
        return self.a, 'is_val'
        
    def testing(self):
        return self.a*2, 'is_val'
tc = TrickyChild4(44, 'sam')
# Trying to use a classmethod to fulfill a class var req. UNacceptable.
class TrickyChild5(Parent):
    
    order = 33
    
    def __init__(self, a, name):
        self.a = a
        self.name = name
        print('child init')
        
    @classmethod
    def is_val(cls):
        return 'testing classmethod'
tc = TrickyChild5(44, 'sam')
# Trying to use a staticmethod to fulfill a class var req. UNacceptable.
class TrickyChild6(Parent):
    
    order = 33
    
    def __init__(self, a, name):
        self.a = a
        self.name = name
        print('child init')
        
    @staticmethod
    def is_val(a):
        return a
tc = TrickyChild6(44, 'sam')
# Implements desired interface.
class GoodChild(Parent):
    
    order = 100
    is_val = False
    
    def __init__(self, name):
        self.name = name
        
    def walk(self):
        return self.order, self.is_val, self.name
ian = GoodChild('Ian')
ian.walk()<jupyter_output><empty_output><jupyter_text>DataGet top 1000 sites for lsh dict testing.<jupyter_code>r = requests.get('https://dataforseo.com/top-1000-websites')
soup = bs(r.content)
s3_path = ('s3://commoncrawl/projects/hyperlinkgraph/cc-main-2018-aug-sep-oct'
           '/domain/cc-main-2018-aug-sep-oct-domain-ranks.txt.gz')
df = pd.read_csv(s3_path, nrows=10_000, sep='\t')
def _process_hostname(host):
    parts = host.split('.')
    return '.'.join(parts[1:]) + '.' + parts[0]
df['host'] = df['#host_rev'].apply(_process_hostname)<jupyter_output><empty_output><jupyter_text>LSH<jupyter_code>from bs4 import BeautifulSoup as bs
from datasketch import lsh, MinHashLSH, MinHash, MinHashLSHForest
import numpy as np
import pandas as pd
import requests
from tldextract import extract

# from htools import *
set1 = set(['minhash', 'is', 'a', 'probabilistic', 'data', 'structure', 'for',
            'estimating', 'the', 'similarity', 'between', 'datasets'])
set2 = set(['minhash', 'is', 'a', 'probability', 'data', 'structure', 'for',
            'estimating', 'the', 'similarity', 'between', 'documents'])
set3 = set(['minhash', 'is', 'probability', 'data', 'structure', 'for',
            'estimating', 'the', 'similarity', 'between', 'documents'])

m1 = MinHash(num_perm=128)
m2 = MinHash(num_perm=128)
m3 = MinHash(num_perm=128)
for d in set1:
    m1.update(d.encode('utf8'))
for d in set2:
    m2.update(d.encode('utf8'))
for d in set3:
    m3.update(d.encode('utf8'))

# Create LSH index
lsh = MinHashLSH(threshold=0.5, num_perm=128)
lsh.insert("m2", m2)
lsh.insert("m3", m3)
result = lsh.query(m1)
result
def ngrams(word, n=3, step=1, drop_last=False):
    """To get non-overlapping sequences, pass in same value for `step` as `n`."""
    stop = max(1, step+len(word)-n)
    ngrams_ = []
    for i in range(0, stop, step):
        ngrams_.append(word[i:i+n])
    if drop_last and len(ngrams_[-1]) < n: ngrams_ = ngrams_[:-1]
    return ngrams_
w = 'dinosaur'
tw = 'I'
ws = ['dino', 'rhino', 'dog', 'cat', 'lion', 'tyrannosaurus rex',
      'tiger', 'owl', 'rhinocerous', 'kino']
s = 'The mailman was running late today, possibly due to the heavy '\
    'snowfall from the previous night.'
ngrams(w)
ngrams(w, 2)
ngrams(w, 3, 3)
ngrams(w, 3, 3, True)
ngrams(s)
def lsh_hash_word(word, num_perm=128, **ngram_kwargs):
    mhash = MinHash(num_perm=num_perm)
    for ng in ngrams(word, **ngram_kwargs):
        mhash.update(ng.encode('utf8'))
    return mhash
m = lsh_hash_word(w)
forest = MinHashLSHForest(num_perm=128)

for i, word in enumerate(ws):
    forest.add(i, lsh_hash_word(word))
forest.index()
ws
idx = forest.query(lsh_hash_word(w), 3)
idx, select(ws, idx)
from htools.core import select
idx = forest.query(lsh_hash_word('ionjwl'), 3)
idx, select(ws, idx)
idx = forest.query(lsh_hash_word('ionowl'), 7)
idx, select(ws, idx)
idx = forest.query(lsh_hash_word('xxx'), 7)
idx<jupyter_output><empty_output><jupyter_text>DataGet top 1000 sites for lsh dict testing.<jupyter_code>def _process_hostname(host):
    parts = host.split('.')
    return '.'.join(parts[1:]) + '.' + parts[0]
s3_path = ('s3://commoncrawl/projects/hyperlinkgraph/cc-main-2018-aug-sep-oct'
           '/domain/cc-main-2018-aug-sep-oct-domain-ranks.txt.gz')
df = pd.read_csv(s3_path, nrows=100_000, sep='\t', 
                 usecols=['#pr_pos', '#host_rev'])
df['host'] = df['#host_rev'].apply(_process_hostname)<jupyter_output><empty_output><jupyter_text>LSHDict<jupyter_code>from abc import ABC, abstractmethod
from fuzzywuzzy import fuzz, process
import matplotlib.pyplot as plt
import inspect

from htools.meta import add_docstring
# Visualizing how n_candidates changes as a function of vocab size.
x = np.arange(0, 1_000_000, 1_000)
y = np.clip(x // 1000, 10, 500)
plt.plot(x, y)
# abstractmethod doesn't work because we're inheriting from dict which has a
# metaclass that's not ABCMeta.
class _FuzzyDictBase(dict):
    
    def __init_subclass__(cls, **kwargs):
        super().__init_subclass__(**kwargs)
        if not callable(getattr(cls, 'similar', None)):
            raise TypeError('Children of _FuzzyDictBase must define method '
                            '`similar`.')
    
    def __getitem__(self, key):
        try:
            return super().__getitem__(key)
        except KeyError:
            return self.similar(key, mode='values', n_keys=1)[0]
        
    def _filter_similarity_pairs(self, pairs, mode='keys_values'):
        """
        mode: str
            Determines what information to return. In each case, items are
            sorted in descending order of similarity.
            - "keys_values": list of (key, value) tuples.
            - "keys_similarities": list of (key, similarity) tuples.
            - "keys_values_similarities": list of (key, value, similarity) 
                tuples.
            - "keys": list of keys (strings).
            - "values": list of values corresponding to the nearest keys (type
                is dependent on what your dict values are).
        """
        if mode == 'values':
            return [self[p[0]] for p in pairs]
        elif mode == 'keys_values':
            return [(p[0], self[p[0]]) for p in pairs]
        elif mode == 'keys_similarities':
            return pairs
        elif mode == 'keys':
            return [p[0] for p in pairs]
        elif mode == 'keys_values_similarities':
            return [(p[0], self[p[0]], p[1]) for p in pairs]
        else:
            raise ValueError(
                'Unrecognized value for `mode`. Should be one of ("values", '
                '"keys", keys_values", "keys_similarities", or '
                '"keys_values_similarities").'
            )
class LSHDict(_FuzzyDictBase):
    """Dictionary that returns the value corresponding to a key's nearest
    neighbor if the key isn't present in the dict. This is intended for use
    as a word2index dict when using embeddings in deep learning: e.g. if we
    have domain embeddings for the top 100k websites, some of our options for 
    dealing with unknown domains are:
    
    1. Encode all of them as <UNK>. This loses a lot of information.
    2. Create a FuzzyKeyDict which will search for similar keys using variants
    of Levenshtein distance. Lookup is O(N) and for 100k domains, that comes
    out to 0.6 seconds per item. We might have thousands or millions of 
    lookups over the course of training so this can be a significant cost.
    3. Create an LSHDict (lookups are O(1)). Indexing into the dict as usual 
    (e.g. my_lsh_dict[key]) will provide the key's index if present and the
    (approximate) nearest neighbor's index otherwise. Either way, the result
    can be used to index into your embedding layer. 
    4. Create an LSHDict and use the `similar_values` method to return n>1
    neighbors. Then pass their indices to an Embedding layer and 
    compute the sum/average/weighted average of the results. This may be 
    preferable to #3 cases such as web domain lookup, where similar URLs are
    not guaranteed to represent similar sites. (This is basically
    equivalent to an EmbeddingBag layer, but in torch that doesn't store
    intermediate representations so we wouldn't be able to use our pretrained
    embeddings.)
    """

    def __init__(self, data, n_candidates=None, n_keys=3, ngram_size=3,
                 scorer=fuzz.ratio):
        """
        Parameters
        ----------
        data: dict or list[tuple]
            The base dictionary. Unlike FuzzyKeyDict, we require this since
            adding items one by one is computationally infeasible for large
            datasets. Just build up your dictionary first.
        n_candidates: int or None
            Number of reasonably similar keys to retrieve when trying to index
            in with a key that's missing (or when using the `similar` method).
            You can override this in `similar` but not when using 
            __getitem__'s square bracket syntax. If not specified, this will
            be auto initialized to vocab size/1,000, clipped to lie in 
            [20, 500]. See `similar` docstring for more on this.
        n_keys: int
            Default number of similar keys to retrieve in `similar`.
        scorer: function
            Default scoring function to use to narrow `n_candidates` keys down
            to `n_keys`. Should be a fuzzywuzzy function where scores lie in 
            [0, 100] and higher values indicate high similarity.
        """
        if len(data) < 10_000 and len(next(iter(data))) < 100:
            warnings.warn(
                'It looks like you\'re working with a relatively small '
                'amount of data. FuzzyKeyDict may be fast enough for your '
                'use case and would provide the set of strictly most similar '
                'keys rather than an approximation of that set.'
            )
            
        super().__init__(data)
        self.scorer = scorer
        self.hash_word = partial(lsh_hash_word, n=ngram_size)
        self.forest = MinHashLSHForest(num_perm=128)
        self._initialize_forest()
        
        # Datasketch's LSH implementation usually gives pretty decent results
        # even with numbers as low as 5-10, but increasing that by a factor of
        # 10 comes with minimal time cost: Fuzzywuzzy matching doesn't get 
        # particularly slow until we get into the thousands. The fact that 
        # we cap this at 500 makes this lookup asymptotically O(1) while
        # FuzzyKeyDict's is O(N).
        self.n_candidates = n_candidates or np.clip(len(self) // 1_000,
                                                    20, 500)
        self.n_keys = n_keys
        
    def __setitem__(self, key, val):
        """Try to add keys all at once in the constructor because adding new
        keys can be extremely slow.
        """
        super().__setitem__(key, val)
        self._update_forest(key, val)
        
    def _update_forest(self, key, val, index=True):
        """Used in __setitem__ to update our LSH Forest. Forest's index method
        seems to recompute everything so adding items to a large LSHDict will
        be incredibly slow. Luckily, our deep learning use case rarely/never
        requires us to update object2index dicts after instantiation so that's
        not as troubling as it might seem.
        
        Parameters
        ----------
        key: str
        val: any
        index: bool
            If True, reindex the forest (essentially making the key 
            queryable). This should be False when initializing the forest so
            we just index once after everything's been added.
        """
        self.forest.add(key, self.hash_word(key))
        if index: self.forest.index()
        
    def _initialize_forest(self):
        """Called once in __init__ to add all items to LSH Forest. This is
        necessary because dict specifically calls its own __setitem__, not
        its children's.
        """
        for k, v in self.items():
            self._update_forest(k, v, False)
        self.forest.index()
    
    @add_docstring(_FuzzyDictBase._filter_similarity_pairs)
    def similar(self, key, mode='keys_values', n_candidates=None, 
                n_keys=None, scorer=None):
        """Find a list of similar keys. This is used in __getitem__ but can
        also be useful as a user-facing method if you want to get more than
        1 neighbor or you want to get similarity scores as well.
        
        Parameters
        ----------
        key: str
            Word/URL/etc. to find similar keys to.
        mode: str
            See section below `Returns`.
        n_candidates: int or None
            Number of similar candidats to retrieve. This uses Jaccard
            Similarity which isn't always a great metric for string 
            similarity. This is also where the LSH comes in so they're not
            strictly the n best candidates, but rather a close approximation
            of that set. If None, this will fall back to self.n_candidates.
            Keep in mind this determines how many keys to 
        n_keys: int or None
            Number of similar keys to return. If None, this will fall back to
            self.n_keys. 
        scorer: function or None
            Fuzzywuzzy scoring function, e.g. fuzz.ratio or 
            fuzz.partial_ratio, which will be used to score each candidate and
            select which to return. Higher scores indicate higher levels of
            similarity. If None, this will fall back to self.scorer.
            
        Returns
        -------
        list: List if `mode` is "keys" or "values". List of tuples otherwise.
        """
        candidates = self.forest.query(self.hash_word(key), 
                                       n_candidates or self.n_candidates)
        if not candidates: raise KeyError('No similar keys found.')
            
        # List of (key, score) where higher means more similar.
        pairs = process.extract(key, candidates, 
                                limit=n_keys or self.n_keys,
                                scorer=scorer or self.scorer)
        return self._filter_similarity_pairs(pairs, mode=mode)
class FuzzyKeyDict(_FuzzyDictBase):
    """Dictionary that will try to find the most similar key if a key is
    missing and return its corresponding value. This could be useful when
    working with embeddings, where we could map missing items to the indices
    of one or more existing embeddings.

    Examples
    --------
    d = FuzzyKeyDict(limit=3)
    d['dog'] = 0
    d['cat'] = 1
    d['alley cat'] = 2
    d['pig'] = 3
    d['cow'] = 4
    d['cowbell'] = 5
    d['baby cow'] = 6

    # Keys and similarity scores for the most similar keys.
    >>> d.similar('house cat', mode='keys_similarities')
    [('alley cat', 56), ('cat', 50), ('cowbell', 25)]

    # "house cat" not in dict so we get the value for the most similar key.
    >>> d['house cat']
    2

    # "cat" is in dict so __getitem__ functions like a regular dict.
    >>> d['cat']
    1
    """

    def __init__(self, data=(), n_keys=3, scorer=fuzz.ratio):
        """
        Parameters
        ----------
        data: Iterable (optional)
            Sequence of pairs, such as a dictionary or a list of tuples. If
            provided, this will be used to populate the FuzzyKeyDict.
        n_keys: int
            Default number of similar keys to retrieve when using the 
            `similar` method.
        scorer: function
            Default scoring function to use to select the most similar keys in
            `similar`. Should be a fuzzywuzzy function where scores lie in 
            [0, 100] and higher values indicate high similarity.
        """
        super().__init__(data)
        self.n_keys = n_keys
        self.scorer = scorer

    @add_docstring(_FuzzyDictBase._filter_similarity_pairs)
    def similar(self, key, mode='keys_values', n_keys=None, scorer=None):
        """Find the keys in the dictionary that are most similar to the given
        key and return some relevant information (the keys, their similarity
        scores, their corresponding values, etc.) depending on what the user
        specifies.

        Parameters
        ----------
        key: str
            This can be present or missing from the dictionary, though in
            practice it's often more useful when it's missing. We'll search
            the existing keys and find the strings that are most similar.
        mode: str
            See section below `Returns`.
        n_keys: int or None
            Number of similar keys to return. If None, this will fall back to
            self.n_keys. 
        scorer: function or None
            Fuzzywuzzy scoring function, e.g. fuzz.ratio or 
            fuzz.partial_ratio, which will be used to score each candidate and
            select which to return. Higher scores indicate higher levels of
            similarity. If None, this will fall back to self.scorer.

        Returns
        -------
        list: List if `mode` is "keys" or "values". List of tuples otherwise.
        """
        pairs = process.extract(key, 
                                self.keys(), 
                                limit=n_keys or self.n_keys,
                                scorer=scorer or fuzz.ratio)
        return self._filter_similarity_pairs(pairs, mode=mode)
# df = pd.read_csv('../data/top500Domains.csv', usecols=['Root Domain', 'Rank'])
# df = df.rename(columns={'Root Domain': 'domain', 'Rank': 'rank_'})
# d2i = dict(zip(df.domain, df.rank_))
d2i = dict(df.head(1_000).set_index('host')['#pr_pos'])
ld = LSHDict(d2i, 3)
fd = FuzzyKeyDict(d2i)
ld['youtube.com']
ld['goguardian.com']
'goguardian.com' in ld
ld.similar('goguardian.com', n_candidates=50, n_keys=4)
ld.similar('goguardian.com', 'keys_values', n_candidates=50, n_keys=4)
ld.similar('goguardian.com', 'keys_similarities', n_candidates=50, n_keys=4)
ld.similar('goguardian.com', 'keys_values_similarities', n_candidates=50, n_keys=4)
ld.similar('goguardian.com', 'keys', n_candidates=50, n_keys=4)
ld.similar('goguardian.com', 'values', n_candidates=50, n_keys=4)
fd['goguardian.com']
fd.similar('goguardian.com')
fd.similar('goguardian.com', 'keys_similarities', n_keys=5)
ld['miniclip.com']
ld.similar('bloodyelbow.com', n_candidates=100)
fd.similar('bloodyelbow.com', 'keys_similarities')
ld['pausd.edu']
fd.similar('pausd.edu', 'keys_values')
ld['addictinggames.com']
ld.similar('addictinggames.com', n_candidates=100)
fd.similar('addictinggames.com')<jupyter_output><empty_output><jupyter_text>Toy Example<jupyter_code>w2i = dict(zip(ws, np.random.permutation(len(ws))))
w2i
ld = LSHDict(w2i)
ld
ld['dog'], ld['sea lion'], ld['munchkin'], ld['kitten']
ld.forest.query(lsh_hash_word('estion'), 3)
ld.forest.query(lsh_hash_word('brontosaur'), 3)
ld['dragon'] = 6<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09778102901731285, 'fraction_numerical': 0.011042602849479222, 'mean_word_length': 2.4319187089061565, 'pattern_counts': {'":': 5, '<': 33, '<?xml version=': 0, '>': 40, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '541105', 'license': 'Apache-2.0', 'n_tokens_mistral': 8720, 'n_tokens_neox': 8291, 'n_words': 2874, 'path': 'scratch_notebooks/lsh.ipynb', 'repo_name': 'hdmamin/mytorch', 'size': '28707'}