<jupyter_start><jupyter_text>Base algorithm<jupyter_code>from duplication.run import *
from duplication.similarity_metrics import jaccard
from duplication.detectors import NaiveDetector
@dataclass
class DetectionResult:
    """
    Class for storing results of detection
    """
    clones: List[Tuple[EntityData, EntityData, float]]


class EntityTypes(Enum):
    Function = 0
    Class = 1


@dataclass
class EntityData:
    entity_type: EntityTypes
    path: str
    lang: str
    bag_of_tokens: List[str]



File = Tuple[str, str]


class Detector:
    @staticmethod
    def extract_entity_from_object_data(obj: ObjectData, path, lang) -> EntityData:
        entity_type = ""
        if obj.object_type == ObjectTypes.FUNCTION:
            entity_type = "function"
        if obj.object_type == ObjectTypes.CLASS:
            entity_type = "class"

        entity_tokens = get_identifiers_sequence_from_code(obj.content, lang)
        return EntityData(entity_type, path, lang, entity_tokens)
    
    @staticmethod
    def extract_entity_from_file_data(file: FileData) -> EntityData:
        entity_path = file_data.path
        entity_lang = file_data.lang
        return [Detector.extract_entity_from_object_data(obj, entity_path, entity_lang) for obj in file_data.objects]
    
    @staticmethod
    def extract_entities_from_files_data(files: List[FileData]) -> List[EntityData]:
        def acc(value: List[EntityData], file_data: FileData) -> List[EntityData]:
            value.extend(Detector.extract_entity_from_file_data(file_data))
            return value
        return functools.reduce(acc, files, [])
    
    @staticmethod
    def extract_data_from_files(files: List[File]) -> List[FileData]:
        return [get_data_from_file(file, lang, True, False) for file, lang in files]
        
    def fill_fields(self, directory: str, granularity: str):
        lang2files = recognize_languages_dir(directory)
        files = transform_files_list(lang2files, granularity, None)
        self.files = [(get_full_path(file, directory), lang) for file, lang in files]
        self.files_data = self.extract_data_from_files(self.files)
        self.entities = self.extract_entities_from_files_data(self.files_data)
    
    def detect(self, directory: str, granularity: str) -> List[DetectionResult]:
        raise NotImplemented
        
        
class NaiveDetector(Detector):
    def detect(self, directory: str, threshold: float, granularity: str) -> DetectionResult:
        self.fill_fields(directory, granularity)
        clones = []
        for i in range(len(self.entities)):
            entity = self.entities[i]
            for j in range(i + 1, len(self.entities)):
                candidate = self.entities[j]
                if entity == candidate or entity.lang != candidate.lang:
                    continue
                
                sim = jaccard(entity.bag_of_tokens, candidate.bag_of_tokens)
                if sim > threshold:
                    clones.append((entity, candidate, sim))
        return DetectionResult(clones)
                        
naive_detector = NaiveDetector()
clones = naive_detector.detect(test_dir.parent, 0.65, "functions").clones
clones
file_data = get_data_from_file(test_dir/"test_file.java", "Java", True, True)
dir(file_data)
file_data
[obj for obj in file_data.objects if obj.object_type == ObjectTypes.FUNCTION]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09082918253735717, 'fraction_numerical': 0.0023439789041898623, 'mean_word_length': 3.0642857142857145, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '533241', 'license': 'MIT', 'n_tokens_mistral': 1029, 'n_tokens_neox': 1003, 'n_words': 224, 'path': 'experiments/naive algorithm.ipynb', 'repo_name': 'otzhora/duplication', 'size': '3413'}