<commit_before>from __future__ import print_function, division, absolute_import

import sys
sys.path.append("../../..")

import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from sqlalchemy.orm import scoped_session, sessionmaker, relationship, backref
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.ext.declarative import declarative_base

import zach_settings as zs

Engine = sa.create_engine(zs.SQLALCHEMY_DATABASE_URI,
                       #encoding='utf-8',
                       #client_encoding='utf-8',
                       echo="debug",
                       )

Session = scoped_session(sessionmaker(bind=Engine)) #, extension=VersionedListener()))
#Xsession = Session()

Base = declarative_base(bind=Engine)
Metadata = sa.MetaData()
Metadata.bind = Engine


class ProbeSet(Base):
    __tablename__ = 'ProbeSet'
    __table_args__ = {'autoload': True}

def main():
    for ps in page_query(Session.query(ProbeSet)):   #all()
    #for ps in probe_sets:
        print("--->", ps)
        
def page_query(q):
    """http://stackoverflow.com/a/1217947/1175849"""
    offset = 0
    while True:
        r = False
        for elem in q.limit(1000).offset(offset):
           r = True
           yield elem
        offset += 1000
        if not r:
            break

if __name__ == "__main__":
    main()<commit_msg>Continued working on quick_search_table.py, which now gets all of the terms from the ProbeSet table to be included in the quick search<commit_after>from __future__ import print_function, division, absolute_import

import sys
sys.path.append("../../..")

import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from sqlalchemy.orm import scoped_session, sessionmaker, relationship, backref
from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.ext.declarative import declarative_base

import zach_settings as zs

Engine = sa.create_engine(zs.SQLALCHEMY_DATABASE_URI,
                       #encoding='utf-8',
                       #client_encoding='utf-8',
                       #echo="debug",
                       )

Session = scoped_session(sessionmaker(bind=Engine)) #, extension=VersionedListener()))
#Xsession = Session()

Base = declarative_base(bind=Engine)
Metadata = sa.MetaData()
Metadata.bind = Engine


class ProbeSet(Base):
    __tablename__ = 'ProbeSet'
    __table_args__ = {'autoload': True}
    
#QuickSearch = sa.Table("QuickSearch", Metadata,
#                    sa.Column('table_name', sa.String),
#                    sa.Column('the_key', sa.String),
#                    sa.Column('terms', sa.String))

class QuickSearch(Base):
    table_name = Column(String)
    the_key = Column(String)
    terms = Column(String)
    
    def __init__(self, table_name, the_key, terms):
        self.table_name = table_name
        self.the_key = the_key
        self.terms = get_unique_terms(terms)

def get_unique_terms(*args):
    if not args:
        return None
    unique = set()
    for item in args:
        #print("locals:", locals())
        if not item:
            continue
        for token in item.split():
            if token.startswith(('(','[')):
                token = token[1:]
            if token.endswith((')', ']')):
                token = token[:-1]
            if len(token) > 2:
                unique.add(token)
    print("\nUnique terms are: {}\n".format(unique))
    return " ".join(unique)


def main():
    for ps in page_query(Session.query(ProbeSet)):   #all()
        terms = get_unique_terms(ps.Name,
                         ps.Symbol,
                         ps.description)
        
        
        
def page_query(q):
    """http://stackoverflow.com/a/1217947/1175849"""
    offset = 0
    while True:
        r = False
        for elem in q.limit(1000).offset(offset):
           r = True
           yield elem
        offset += 1000
        if not r:
            break

if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.11235668789808917, 'fraction_numerical': 0.013503184713375796, 'mean_word_length': 2.3271186440677964, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6070620', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1158, 'n_words': 285}