<commit_before>//
//  NSSpellEngine_hunspell.m
//  NSSpellEngine_hunspell
//
//  Created by <NAME> on 8/29/11.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#import "NSSpellEngine_hunspell.h"
#import "NSSpellEngine_hunspellDictionary.h"
#import <Foundation/NSOrthography.h>

@implementation NSSpellEngine_hunspell

+(NSArray *)spellEngines {
   NSBundle *bundle=[NSBundle bundleForClass:self];
   NSString *directory=[[bundle resourcePath] stringByAppendingPathComponent:@"Spelling"];

   NSSpellEngine_hunspell *engine=[[[NSSpellEngine_hunspell alloc] initWithContentsOfFile:directory] autorelease];
   
   return [NSArray arrayWithObject: engine];
}

-initWithContentsOfFile:(NSString *)path {
   NSArray *contents=[[NSFileManager defaultManager] contentsOfDirectoryAtPath:path error:NULL];
   
   _dictionaries=[[NSMutableDictionary alloc] init];
      
   for(NSString *aff in contents){
       
    if([[aff pathExtension] isEqualToString:@"aff"]){
     NSString *affPath=[path stringByAppendingPathComponent:aff];
     NSSpellEngine_hunspellDictionary *dict=[[[NSSpellEngine_hunspellDictionary alloc] initWithContentsOfFile:affPath] autorelease];
     
     [_dictionaries setObject:dict forKey:[dict language]];
    }
        
   }
   
   return self;
}

-(NSString *)vendor {
   return @"Hunspell";
}

-(NSArray *)languages {
   NSMutableArray *result=[NSMutableArray array];
   
   for(NSSpellEngine_hunspellDictionary *dict in [_dictionaries allValues]){
    [result addObject:[dict language]];
   }
   
   [result sortUsingSelector:@selector(caseInsensitiveCompare:)];
   
   return result;
}

- (NSSpellEngine_hunspellDictionary *)_dictionaryForLanguage:(NSString *)language
{
	if (language == nil) {
		language = [[NSLocale currentLocale] localeIdentifier];
	}
	
	NSSpellEngine_hunspellDictionary *dict=[_dictionaries objectForKey:language];
	if (dict == nil) {
		// If the lang is "xx_YY", then try the first dict starting with "xx"
		// So "fr_CA", "fr_BE"... can use the french dictionary for example if no canadian or belgium
		// dictionary is available
		NSArray *elements = [language componentsSeparatedByString:@"_"];
		if ([elements count] >= 1) {
			NSString *langPrefix = [elements objectAtIndex:0];
			for (NSString *lang in _dictionaries.allKeys) {
				if ([lang hasPrefix:langPrefix]) {
					dict=[_dictionaries objectForKey:lang];
				}
			}
		}
	}
	if (dict == nil) {
		// Couldnt find any dictionary that seems interesting for the user
		// Fallback to the US one
		dict=[_dictionaries objectForKey:@"en_US"];
	}	
}

-(NSArray *)checkString:(NSString *)stringToCheck offset:(NSUInteger)offset types:(NSTextCheckingTypes)checkingTypes options:(NSDictionary *)options orthography:(NSOrthography *)orthography wordCount:(NSInteger *)wordCount {
   NSMutableArray *result=[NSMutableArray array];
   
   NSString *language=[orthography dominantLanguage];
   NSSpellEngine_hunspellDictionary *dict=[self _dictionaryForLanguage:language];
      
   NSUInteger length=[stringToCheck length];
   
   NSUInteger bufferCapacity=10,bufferOffset=offset,bufferIndex=0,bufferLength=0;
   unichar    buffer[bufferCapacity];
   
   NSUInteger wordCapacity=10,wordLength=0;
   unichar   *wordBuffer=(unichar *)NSZoneMalloc(NULL,sizeof(unichar)*wordCapacity);
   
   NSCharacterSet *letters=[NSCharacterSet letterCharacterSet];
   
   enum {
    STATE_WHITESPACE,
    STATE_WORD,
   } state=STATE_WHITESPACE;
   
   for(;bufferOffset+bufferIndex<=length;bufferIndex++){
    BOOL    appendToWord=NO;
    BOOL    checkWord=NO;
    unichar code;
        
    if(bufferOffset+bufferIndex==length)
     code=' ';
    else {<commit_msg>Missing return in _dictionaryForLanguage<commit_after>//
//  NSSpellEngine_hunspell.m
//  NSSpellEngine_hunspell
//
//  Created by <NAME> on 8/29/11.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#import "NSSpellEngine_hunspell.h"
#import "NSSpellEngine_hunspellDictionary.h"
#import <Foundation/NSOrthography.h>

@implementation NSSpellEngine_hunspell

+(NSArray *)spellEngines {
   NSBundle *bundle=[NSBundle bundleForClass:self];
   NSString *directory=[[bundle resourcePath] stringByAppendingPathComponent:@"Spelling"];

   NSSpellEngine_hunspell *engine=[[[NSSpellEngine_hunspell alloc] initWithContentsOfFile:directory] autorelease];
   
   return [NSArray arrayWithObject: engine];
}

-initWithContentsOfFile:(NSString *)path {
   NSArray *contents=[[NSFileManager defaultManager] contentsOfDirectoryAtPath:path error:NULL];
   
   _dictionaries=[[NSMutableDictionary alloc] init];
      
   for(NSString *aff in contents){
       
    if([[aff pathExtension] isEqualToString:@"aff"]){
     NSString *affPath=[path stringByAppendingPathComponent:aff];
     NSSpellEngine_hunspellDictionary *dict=[[[NSSpellEngine_hunspellDictionary alloc] initWithContentsOfFile:affPath] autorelease];
     
     [_dictionaries setObject:dict forKey:[dict language]];
    }
        
   }
   
   return self;
}

-(NSString *)vendor {
   return @"Hunspell";
}

-(NSArray *)languages {
   NSMutableArray *result=[NSMutableArray array];
   
   for(NSSpellEngine_hunspellDictionary *dict in [_dictionaries allValues]){
    [result addObject:[dict language]];
   }
   
   [result sortUsingSelector:@selector(caseInsensitiveCompare:)];
   
   return result;
}

- (NSSpellEngine_hunspellDictionary *)_dictionaryForLanguage:(NSString *)language
{
	if (language == nil) {
		language = [[NSLocale currentLocale] localeIdentifier];
	}
	NSSpellEngine_hunspellDictionary *dict=[_dictionaries objectForKey:language];
	if (dict == nil) {
		// If the lang is "xx_YY", then try the first dict starting with "xx"
		// So "fr_CA", "fr_BE"... can use the french dictionary for example if no canadian or belgium
		// dictionary is available
		NSArray *elements = [language componentsSeparatedByString:@"_"];
		if ([elements count] >= 1) {
			NSString *langPrefix = [elements objectAtIndex:0];
			for (NSString *lang in _dictionaries.allKeys) {
				if ([lang hasPrefix:langPrefix]) {
					dict=[_dictionaries objectForKey:lang];
				}
			}
		}
	}
	if (dict == nil) {
		// Couldnt find any dictionary that seems interesting for the user
		// Fallback to the US one
		dict=[_dictionaries objectForKey:@"en_US"];
	}	
	return [[dict retain] autorelease];
}

-(NSArray *)checkString:(NSString *)stringToCheck offset:(NSUInteger)offset types:(NSTextCheckingTypes)checkingTypes options:(NSDictionary *)options orthography:(NSOrthography *)orthography wordCount:(NSInteger *)wordCount {
   NSMutableArray *result=[NSMutableArray array];
   
   NSString *language=[orthography dominantLanguage];
   NSSpellEngine_hunspellDictionary *dict=[self _dictionaryForLanguage:language];
   NSUInteger length=[stringToCheck length];
   
   NSUInteger bufferCapacity=10,bufferOffset=offset,bufferIndex=0,bufferLength=0;
   unichar    buffer[bufferCapacity];
   
   NSUInteger wordCapacity=10,wordLength=0;
   unichar   *wordBuffer=(unichar *)NSZoneMalloc(NULL,sizeof(unichar)*wordCapacity);
   
   NSCharacterSet *letters=[NSCharacterSet letterCharacterSet];
   
   enum {
    STATE_WHITESPACE,
    STATE_WORD,
   } state=STATE_WHITESPACE;
   
   for(;bufferOffset+bufferIndex<=length;bufferIndex++){
    BOOL    appendToWord=NO;
    BOOL    checkWord=NO;
    unichar code;
        
    if(bufferOffset+bufferIndex==length)
     code=' ';
    else {
{'fraction_non_alphanumeric': 0.1102971102971103, 'fraction_numerical': 0.004884004884004884, 'mean_word_length': 4.832278481012659, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2487265', 'n_tokens_mistral': 2455, 'n_tokens_neox': 2284, 'n_words': 467}