<commit_before>#! /usr/bin/python
#-*- coding:utf-8 -*

import os
import glob
import shutil

import conf
from web.models import Shelter, Category, ShelterPicture
from bootstrap import db

#! /usr/bin/python
#-*- coding:utf-8 -*

import os
import glob
import shutil

import conf
from web.models import Shelter, Category, ShelterPicture
from bootstrap import db
from PIL import Image, ImageFile

def import_shelters_pictures(folder):
    
    ImageFile.LOAD_TRUNCATED_IMAGES = True
    imgwidth = 1280
    
    shelters = Shelter.query.all()

    for shelter in shelters:
        shelter_rid = shelter.get_values_of_attribute(attribute_name='ID')[0].name
        print("Shelter_rid '{}' ...".format(shelter_rid))
			
        for picture in glob.glob(folder + shelter_rid + '/**.jpg'):
            picture_name = os.path.basename(picture)
            print("Picture name '{}' ...".format(picture_name))
            
            try:
                category_name = picture_name.split('_')[1]
                picture_subject = os.path.splitext(picture_name.split('_')[2])[0]
            except:
                print("failed")
                continue

            print("Picture subject '{}' ...".format(picture_subject))
            print("Category name '{}' ...".format(category_name))
            category = Category.query.filter(Category.name==category_name,
                                    Category.parent_id!=None).first()

            if category:
                if picture_subject.lower() == 'facade':
                    new_picture = ShelterPicture(file_name=picture_name,
                                    shelter_id=shelter.id,
                                    category_id=category.id,
                                    is_main_picture=True)
                else:
                    new_picture = ShelterPicture(file_name=picture_name,
                                    shelter_id=shelter.id,
                                    category_id=category.id)

                db.session.add(new_picture)
                db.session.commit()

                path = os.path.join(conf.SHELTERS_PICTURES_SERVER_PATH,
                                                            str(shelter.id))
                if not os.path.exists(path):
                    os.makedirs(path)

                #shutil.copy(picture, path)
                
                im = Image.open(picture)
                
                if im.size[0] > imgwidth:
                    ratio = (imgwidth/float(im.size[0]))
                    hsize = int((float(im.size[1])*float(ratio)))
                    print((imgwidth, hsize))
					
                try:
                	resized_im = im.resize((imgwidth,hsize), Image.BICUBIC)
                	resized_im.save(os.path.join(path , picture_name), "JPEG", quality=70, optimize=True, progressive=True)
                except OSError
                   pass
                else:
                	im.save(os.path.join(path, picture_name), "JPEG", quality=70, optimize=True, progressive=True)
                print("Copy from '{}' ...".format(picture))
                print("Copy to '{}' ...".format(path))
<commit_msg>bugfix<commit_after>#! /usr/bin/python
#-*- coding:utf-8 -*

import os
import glob
import shutil

import conf
from web.models import Shelter, Category, ShelterPicture
from bootstrap import db

#! /usr/bin/python
#-*- coding:utf-8 -*

import os
import glob
import shutil

import conf
from web.models import Shelter, Category, ShelterPicture
from bootstrap import db
from PIL import Image, ImageFile

def import_shelters_pictures(folder):
    
    ImageFile.LOAD_TRUNCATED_IMAGES = True
    imgwidth = 1280
    
    shelters = Shelter.query.all()

    for shelter in shelters:
        shelter_rid = shelter.get_values_of_attribute(attribute_name='ID')[0].name
        print("Shelter_rid '{}' ...".format(shelter_rid))
			
        for picture in glob.glob(folder + shelter_rid + '/**.jpg'):
            picture_name = os.path.basename(picture)
            print("Picture name '{}' ...".format(picture_name))
            
            try:
                category_name = picture_name.split('_')[1]
                picture_subject = os.path.splitext(picture_name.split('_')[2])[0]
            except:
                print("failed")
                continue

            print("Picture subject '{}' ...".format(picture_subject))
            print("Category name '{}' ...".format(category_name))
            category = Category.query.filter(Category.name==category_name,
                                    Category.parent_id!=None).first()

            if category:
                if picture_subject.lower() == 'facade':
                    new_picture = ShelterPicture(file_name=picture_name,
                                    shelter_id=shelter.id,
                                    category_id=category.id,
                                    is_main_picture=True)
                else:
                    new_picture = ShelterPicture(file_name=picture_name,
                                    shelter_id=shelter.id,
                                    category_id=category.id)

                db.session.add(new_picture)
                db.session.commit()

                path = os.path.join(conf.SHELTERS_PICTURES_SERVER_PATH,
                                                            str(shelter.id))
                if not os.path.exists(path):
                    os.makedirs(path)

                #shutil.copy(picture, path)
                
                im = Image.open(picture)
                
                if im.size[0] > imgwidth:
                    ratio = (imgwidth/float(im.size[0]))
                    hsize = int((float(im.size[1])*float(ratio)))
                    print((imgwidth, hsize))
					
                try:
                	resized_im = im.resize((imgwidth,hsize), Image.BICUBIC)
                	resized_im.save(os.path.join(path , picture_name), "JPEG", quality=70, optimize=True, progressive=True)
                except OSError:
                   pass
                else:
                	im.save(os.path.join(path, picture_name), "JPEG", quality=70, optimize=True, progressive=True)
                print("Copy from '{}' ...".format(picture))
                print("Copy to '{}' ...".format(path))

{'fraction_non_alphanumeric': 0.10979087452471482, 'fraction_numerical': 0.005386565272496832, 'mean_word_length': 1.5404426559356137, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5987642', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1696, 'n_words': 306}