<commit_before>#! /usr/bin/env python

"""utility methods"""

import re, exifread, os.path, hashlib, logging
from PIL import Image
# our own libs
import aws

# set up logging
logger = logging.getLogger('cigarbox')


def setup_custom_logger(name):
    formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s')

    handler = logging.StreamHandler()
    handler.setFormatter(formatter)

    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)
    logger.addHandler(handler)
    return logger

def genThumbnail(filename,abbr,regen=False):
  '''generate a single thumbnail'''
  # define the sizes of the various thumbnails
  thumbnailDefinitions={
    's': (75,75), #should be square eventually
    'q': (150,150), #should be square eventually
    't': (100,100),
    'm': (240,240),
    'n': (320,230),
    'k': (500,500),
    'c': (800,800),
    'b': (1024,1024)}
  size = thumbnailDefinitions[abbr]
  thumbFileName = filename.split('.')[0] + '_' + abbr + '.jpg'
  if os.path.isfile(thumbFileName) and regen == False:
    return(thumbFileName)
  else:
    try:
      logger.info('Generating thumbnail: %s' %(thumbFileName))
      im = Image.open(filename)
      icc_profile = im.info.get('icc_profile')
      im.thumbnail(size,Image.ANTIALIAS)
      im.save(thumbFileName, 'JPEG', icc_profile=icc_profile, quality=95)
      return(thumbFileName)
    except IOError as e:
      raise e

def genThumbnails(filename,config,regen):
  t_thumb = genThumbnail(filename,abbr='t',regen=regen)
  aws.uploadToS3(t_thumb,t_thumb,config)

  genThumbnail(filename,abbr='m',regen=regen)
  genThumbnail(filename,abbr='n',regen=regen)
  genThumbnail(filename,abbr='c',regen=regen)
  genThumbnail(filename,abbr='b',regen=regen)


# base58 functions for short URL's

alphabet = '123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ'
base = len(alphabet)

def b58encode(div, s=''):
  if div >= base:
    div, mod = divmod(div, base)
    return b58encode(div, alphabet[mod] + s)<commit_msg>properly gen all thumbnail sizes<commit_after>#! /usr/bin/env python

"""utility methods"""

import re, exifread, os.path, hashlib, logging
from PIL import Image
# our own libs
import aws

# set up logging
logger = logging.getLogger('cigarbox')


def setup_custom_logger(name):
    formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(module)s - %(message)s')

    handler = logging.StreamHandler()
    handler.setFormatter(formatter)

    logger = logging.getLogger(name)
    logger.setLevel(logging.INFO)
    logger.addHandler(handler)
    return logger

def genThumbnail(filename,abbr,regen=False):
  '''generate a single thumbnail'''
  # define the sizes of the various thumbnails
  thumbnailDefinitions={
    's': (75,75), #should be square eventually
    'q': (150,150), #should be square eventually
    't': (100,100),
    'm': (240,240),
    'n': (320,230),
    'k': (500,500),
    'c': (800,800),
    'b': (1024,1024)}
  size = thumbnailDefinitions[abbr]
  thumbFileName = filename.split('.')[0] + '_' + abbr + '.jpg'
  if os.path.isfile(thumbFileName) and regen == False:
    return(thumbFileName)
  else:
    try:
      logger.info('Generating thumbnail: %s' %(thumbFileName))
      im = Image.open(filename)
      icc_profile = im.info.get('icc_profile')
      im.thumbnail(size,Image.ANTIALIAS)
      im.save(thumbFileName, 'JPEG', icc_profile=icc_profile, quality=95)
      return(thumbFileName)
    except IOError as e:
      raise e

def genThumbnails(filename,config,regen=False):
  thumbTypes = ['t','m','n','c','b']
  for thumbType in thumbTypes:
    thumbFilename = genThumbnail(filename,abbr=thumbType,regen=regen)
    aws.uploadToS3(thumbFilename,thumbFilename,config,regen=regen)


# base58 functions for short URL's

alphabet = '123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ'
base = len(alphabet)

def b58encode(div, s=''):
  if div >= base:
    div, mod = divmod(div, base)
    return b58encode(div, alphabet[mod] + s)
{'fraction_non_alphanumeric': 0.14353707414829658, 'fraction_numerical': 0.03356713426853707, 'mean_word_length': 4.462380300957593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5956007', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1398, 'n_words': 295}