<commit_before>from framework.db import models
from framework.config import config
from framework.lib.general import cprint
import os

class ResourceDB(object):
    def __init__(self, Core):
        self.Core = Core
        self.LoadResourceDBFromFile(self.Core.Config.FrameworkConfigGet("DEFAULT_RESOURCES_PROFILE"))

    def LoadResourceDBFromFile(self, file_path): # This needs to be a list instead of a dictionary to preserve order in python < 2.7
        cprint("Loading Resources from: " + file_path + " ..")
        resources = self.GetResourcesFromFile(file_path)
        # resources = [(Type, Name, Resource), (Type, Name, Resource),]
        for Type, Name, Resource in resources:
            # Need more filtering to avoid duplicates
            if not self.Core.DB.session.query(models.Resource).filter_by(resource_type = Type, resource_name = Name, resource = Resource).all():
                self.Core.DB.session.add(models.Resource(resource_type = Type, resource_name = Name, resource = Resource))
        self.Core.DB.session.commit()

    def GetResourcesFromFile(self, resource_file):
        resources = []
        ConfigFile = self.Core.open(resource_file, 'r').read().splitlines() # To remove stupid '\n' at the end
        for line in ConfigFile:
            if '#' == line[0]:
                continue # Skip comment lines
            try:
                Type, Name, Resource = line.split('_____')
                # Resource = Resource.strip()
                resources.append((Type, Name, Resource))
            except ValueError:
                cprint("ERROR: The delimiter is incorrect in this line at Resource File: "+str(line.split('_____')))
        return resources
<commit_msg>[db] Bug when resources getting duplicated when loading them at the start<commit_after>from framework.db import models
from framework.config import config
from framework.lib.general import cprint
import os

class ResourceDB(object):
    def __init__(self, Core):
        self.Core = Core
        self.LoadResourceDBFromFile(self.Core.Config.FrameworkConfigGet("DEFAULT_RESOURCES_PROFILE"))

    def LoadResourceDBFromFile(self, file_path): # This needs to be a list instead of a dictionary to preserve order in python < 2.7
        cprint("Loading Resources from: " + file_path + " ..")
        resources = self.GetResourcesFromFile(file_path)
        # Delete all old resources which are not edited by user
        # because we may have updated the resource
        self.Core.DB.session.query(models.Resource).filter_by(dirty=False).delete()
        # resources = [(Type, Name, Resource), (Type, Name, Resource),]
        for Type, Name, Resource in resources:
            # Need more filtering to avoid duplicates
            if not self.Core.DB.session.query(models.Resource).filter_by(resource_type = Type, resource_name = Name, resource = Resource).all():
                self.Core.DB.session.add(models.Resource(resource_type = Type, resource_name = Name, resource = Resource))
        self.Core.DB.session.commit()

    def GetResourcesFromFile(self, resource_file):
        resources = []
        ConfigFile = self.Core.open(resource_file, 'r').read().splitlines() # To remove stupid '\n' at the end
        for line in ConfigFile:
            if '#' == line[0]:
                continue # Skip comment lines
            try:
                Type, Name, Resource = line.split('_____')
                # Resource = Resource.strip()
                resources.append((Type, Name, Resource))
            except ValueError:
                cprint("ERROR: The delimiter is incorrect in this line at Resource File: "+str(line.split('_____')))
        return resources

{'fraction_non_alphanumeric': 0.1005708072845882, 'fraction_numerical': 0.0016308779559662953, 'mean_word_length': 2.7743589743589743, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1188479', 'n_tokens_mistral': 968, 'n_tokens_neox': 935, 'n_words': 291}