<commit_before>/**
 * Copyright (c) 2017, Bongmi
 * All rights reserved
 * Author: <EMAIL>
 */

#import <Foundation/Foundation.h>
@class WRCharRange;

#define MAXLenCharRange 256u
@interface WRCharRangeNormalizeManager : NSObject{
 @public int table[MAXLenCharRange];
}
@property (nonatomic, strong, readwrite) NSMutableArray<WRCharRange *> *normalizedRanges;
- (instancetype)initWithRanges:(NSArray <WRCharRange *> *)ranges;
- (NSArray <WRCharRange *>*)decomposeRange:(WRCharRange *)range;
@end<commit_msg>mapper add decompose range list<commit_after>/**
 * Copyright (c) 2017, Bongmi
 * All rights reserved
 * Author: <EMAIL>
 */

#import <Foundation/Foundation.h>
@class WRCharRange;

#define MAXLenCharRange 256u
@interface WRCharRangeNormalizeMapper : NSObject {
 @public
  int table[MAXLenCharRange];
}
@property (nonatomic, strong, readwrite) NSMutableArray<WRCharRange *> *normalizedRanges;

- (instancetype)initWithRanges:(NSArray <WRCharRange *> *)ranges;
- (NSArray <NSNumber *> *)decomposeRange:(WRCharRange *)range;
- (NSArray <NSNumber *> *)decomposeRangeList:(NSArray <WRCharRange *>*)rangeList;

@end
{'fraction_non_alphanumeric': 0.1335125448028674, 'fraction_numerical': 0.012544802867383513, 'mean_word_length': 7.153284671532846, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1456231', 'n_tokens_mistral': 397, 'n_tokens_neox': 368, 'n_words': 82}