# Path: prefixes/models.py
# class Prefix(db.Expando):
#     """
#     Count all domains in the datastore that start with the same
#     prefix. The key name is the prefix, with a leading dot and
#     optional property 'resume' if incomplete.
#     """
#     length = db.IntegerProperty(required=True)
#     count = db.IntegerProperty(required=True, default=0)
#     com = db.IntegerProperty()
#     percentage = db.FloatProperty()
#     timestamp = db.DateTimeProperty()
# 
#     @classmethod
#     def name_cut(cls, name, length=None):
#         """
#         >>> Prefix.name_cut('123456789')
#         '123456789'
#         >>> Prefix.name_cut('123456789', 3)
#         '123'
#         """
#         if length is None:
#             return name
#         else:
#             return name[:length]
# 
# class Suffix(db.Expando):
#     """
#     Count all domains in the datastore that end with the same suffix.
#     The key name is the suffix backwards, with a leading dot
#     and optional property 'resume' if incomplete.
#     """
#     length = db.IntegerProperty(required=True)
#     count = db.IntegerProperty(required=True, default=0)
#     com = db.IntegerProperty()
#     percentage = db.FloatProperty()
#     timestamp = db.DateTimeProperty()
# 
#     @classmethod
#     def name_cut(cls, name, length=None):
#         """
#         >>> Suffix.name_cut('123456789')
#         '987654321'
#         >>> Suffix.name_cut('123456789', 3)
#         '987'
#         """
#         if length is None:
#             return name[::-1]
#         else:
#             return name[::-1][:length]
# 
# Path: tools/retry.py
# def retry(func, *args, **kwargs):
#     for attempt in range(MAX_ATTEMPTS):
#         if attempt:
#             seconds = min(300, 2 ** attempt)
#             logging.warning(
#                 "Attempt %d of %d will start in %d seconds." % (
#                     attempt + 1, MAX_ATTEMPTS, seconds))
#             time.sleep(seconds)
#         try:
#             return func(*args, **kwargs)
#         except (datastore_errors.Timeout, apiproxy_errors.Error,
#                 urllib2.URLError, socket.error), error:
#             logging.error(type(error))
#             logging.error(str(error))
#             if attempt + 1 >= MAX_ATTEMPTS:
#                 raise


import os
import sys
from common.appenginepatch.aecmd import setup_env
from google.appengine.ext import db
from google.appengine.ext.remote_api import remote_api_stub
from prefixes.models import Prefix, Suffix
from tools.retry import retry

# Setup project environment in the parent directory.
sys.path[0] = os.path.dirname(sys.path[0])
setup_env()




def fetch_file(Model, length):
    kind = Model.kind()
    filename = '.data/popular/%ses.%d.txt' % (kind.split('_')[-1], length)
    outfile = open(filename, 'w')
    start = db.Key.from_path(kind, '-')
    while True:
        query = Model.all().filter('length', length).order('__key__')
        query.filter('__key__ >', start)
        prefixes = retry(query.fetch, 1000)
        for prefix in prefixes:
            name = prefix.key().name()
            if name.startswith('.'):
                continue
            if kind == 'prefixes_suffix':
                name = name[::-1]
            outfile.write('%d %s\n' % (prefix.com, name))
        if len(prefixes) < 1000:
            break
        start = prefixes[-1].key()
    outfile.close()



# Path: prefixes/models.py
# class Prefix(db.Expando):
#     """
#     Count all domains in the datastore that start with the same
#     prefix. The key name is the prefix, with a leading dot and
#     optional property 'resume' if incomplete.
#     """
#     length = db.IntegerProperty(required=True)
#     count = db.IntegerProperty(required=True, default=0)
#     com = db.IntegerProperty()
#     percentage = db.FloatProperty()
#     timestamp = db.DateTimeProperty()
# 
#     @classmethod
#     def name_cut(cls, name, length=None):
#         """
#         >>> Prefix.name_cut('123456789')
#         '123456789'
#         >>> Prefix.name_cut('123456789', 3)
#         '123'
#         """
#         if length is None:
#             return name
#         else:
#             return name[:length]
# 
# class Suffix(db.Expando):
#     """
#     Count all domains in the datastore that end with the same suffix.
#     The key name is the suffix backwards, with a leading dot
#     and optional property 'resume' if incomplete.
#     """
#     length = db.IntegerProperty(required=True)
#     count = db.IntegerProperty(required=True, default=0)
#     com = db.IntegerProperty()
#     percentage = db.FloatProperty()
#     timestamp = db.DateTimeProperty()
# 
#     @classmethod
#     def name_cut(cls, name, length=None):
#         """
#         >>> Suffix.name_cut('123456789')
#         '987654321'
#         >>> Suffix.name_cut('123456789', 3)
#         '987'
#         """
#         if length is None:
#             return name[::-1]
#         else:
#             return name[::-1][:length]
# 
# Path: tools/retry.py
# def retry(func, *args, **kwargs):
#     for attempt in range(MAX_ATTEMPTS):
#         if attempt:
#             seconds = min(300, 2 ** attempt)
#             logging.warning(
#                 "Attempt %d of %d will start in %d seconds." % (
#                     attempt + 1, MAX_ATTEMPTS, seconds))
#             time.sleep(seconds)
#         try:
#             return func(*args, **kwargs)
#         except (datastore_errors.Timeout, apiproxy_errors.Error,
#                 urllib2.URLError, socket.error), error:
#             logging.error(type(error))
#             logging.error(str(error))
#             if attempt + 1 >= MAX_ATTEMPTS:
#                 raise


# Path: tools/download_prefixes.py
import os
import sys
from common.appenginepatch.aecmd import setup_env
from google.appengine.ext import db
from google.appengine.ext.remote_api import remote_api_stub
from prefixes.models import Prefix, Suffix
from tools.retry import retry


# Setup project environment in the parent directory.
sys.path[0] = os.path.dirname(sys.path[0])
setup_env()




def fetch_file(Model, length):
    kind = Model.kind()
    filename = '.data/popular/%ses.%d.txt' % (kind.split('_')[-1], length)
    outfile = open(filename, 'w')
    start = db.Key.from_path(kind, '-')
    while True:
        query = Model.all().filter('length', length).order('__key__')
        query.filter('__key__ >', start)
        prefixes = retry(query.fetch, 1000)
        for prefix in prefixes:
            name = prefix.key().name()
            if name.startswith('.'):
                continue
            if kind == 'prefixes_suffix':
                name = name[::-1]
            outfile.write('%d %s\n' % (prefix.com, name))
        if len(prefixes) < 1000:
            break
        start = prefixes[-1].key()
    outfile.close()



def auth_func():