<commit_before>#import "TQHeaderParser.h"
#import "../../Runtime/TQBoxedObject.h"
#import "../CodeGen.h"
#import "../../Runtime/NSString+TQAdditions.h"
#import <objc/runtime.h>

using namespace llvm;

struct TQHeaderParserState {
    TQBridgedClassInfo *currentClass;
    NSMutableDictionary *functions;
    NSMutableDictionary *classes;
    NSMutableDictionary *literalConstants;
    NSMutableDictionary *constants;
    NSMutableDictionary *protocols;
};

#define EMPTY_PARSER_STATE ((struct TQHeaderParserState) { \
    nil, [NSMutableDictionary dictionary], [NSMutableDictionary dictionary], \
         [NSMutableDictionary dictionary], [NSMutableDictionary dictionary], \
         [NSMutableDictionary dictionary] })

static NSString *_prepareConstName(NSString *name)
{
    if([name hasPrefix:@"_"])
        return [NSString stringWithFormat:@"_%@", [[name substringFromIndex:1] stringByCapitalizingFirstLetter]];
    return [name stringByCapitalizingFirstLetter];
}

@interface TQHeaderParser ()
+ (const char *)_encodingForFunPtrCursor:(CXCursor)cursor;
+ (char *)_encodingForCursor:(CXCursor)cursor;
- (void)_parseTranslationUnit:(CXTranslationUnit)aTranslationUnit withState:(struct TQHeaderParserState *)aState;
- (void)_importParserState:(struct TQHeaderParserState *)aState;
- (BOOL)_writeParserState:(struct TQHeaderParserState *)aState toFile:(NSString *)aPath;
@end

@implementation TQHeaderParser
- (id)init
{
    if(!(self = [super init]))
        return nil;

    _functions        = [NSMutableDictionary new];
    _literalConstants = [NSMutableDictionary new];
    _constants        = [NSMutableDictionary new];
    _classes          = [NSMutableDictionary new];
    _protocols        = [NSMutableDictionary new];
    _index            = clang_createIndex(0, 1);

    return self;
}
- (id)initWithCoder:(NSCoder *)aCoder
{
    if(!(self = [self init]))
        return nil;
    _functions        = [[aCoder decodeObject] retain];
    _classes          = [[aCoder decodeObject] retain];
    _literalConstants = [[aCoder decodeObject] retain];
    _constants        = [[aCoder decodeObject] retain];
    _protocols        = [[aCoder decodeObject] retain];
    return self;
}

- (id)parseHeader:(NSString *)aPath
{
    NSRange frameworkRange = [aPath rangeOfString:@".framework/"];
    NSString *frameworksPath = nil;
    if(frameworkRange.location != NSNotFound)
        frameworksPath = [[aPath substringToIndex:NSMaxRange(frameworkRange)] stringByDeletingLastPathComponent];

    // We create a temp TQPCH file so that future compiles go faster
    if(![aPath isAbsolutePath])
        aPath = [[[NSFileManager defaultManager] currentDirectoryPath] stringByAppendingPathComponent:aPath];
    NSString *tempPath = [NSString stringWithFormat:@"/tmp/tranquil_pch/%@.tqpch", [aPath stringByDeletingPathExtension]];
    if([[NSFileManager defaultManager] fileExistsAtPath:tempPath])
        return [self parseTQPCH:tempPath];
    [[NSFileManager defaultManager] createDirectoryAtPath:[tempPath stringByDeletingLastPathComponent]
    withIntermediateDirectories:YES attributes:nil error:nil];
    const char *args[] = { "-x", "objective-c", frameworksPath ? [[@"-F" stringByAppendingString:frameworksPath] UTF8String] : nil };

    CXTranslationUnit translationUnit = clang_parseTranslationUnit(_index, [aPath fileSystemRepresentation], args, frameworksPath ? 3 : 2, NULL, 0,
                                                                   CXTranslationUnit_DetailedPreprocessingRecord|CXTranslationUnit_SkipFunctionBodies);
    if (!translationUnit) {
        TQLog(@"Couldn't parse header %@\n", aPath);
        return nil;
    }
    struct TQHeaderParserState parserState = EMPTY_PARSER_STATE;
    [self _parseTranslationUnit:translationUnit withState:&parserState];
    [self _importParserState:&parserState];
<commit_msg>Monkeypatched a search path that should be built in to libclang (and used to not be an issue, I'm hoping this will be fixed in llvm3.2 final)<commit_after>#import "TQHeaderParser.h"
#import "../../Runtime/TQBoxedObject.h"
#import "../CodeGen.h"
#import "../../Runtime/NSString+TQAdditions.h"
#import <objc/runtime.h>

using namespace llvm;

struct TQHeaderParserState {
    TQBridgedClassInfo *currentClass;
    NSMutableDictionary *functions;
    NSMutableDictionary *classes;
    NSMutableDictionary *literalConstants;
    NSMutableDictionary *constants;
    NSMutableDictionary *protocols;
};

#define EMPTY_PARSER_STATE ((struct TQHeaderParserState) { \
    nil, [NSMutableDictionary dictionary], [NSMutableDictionary dictionary], \
         [NSMutableDictionary dictionary], [NSMutableDictionary dictionary], \
         [NSMutableDictionary dictionary] })

static NSString *_prepareConstName(NSString *name)
{
    if([name hasPrefix:@"_"])
        return [NSString stringWithFormat:@"_%@", [[name substringFromIndex:1] stringByCapitalizingFirstLetter]];
    return [name stringByCapitalizingFirstLetter];
}

@interface TQHeaderParser ()
+ (const char *)_encodingForFunPtrCursor:(CXCursor)cursor;
+ (char *)_encodingForCursor:(CXCursor)cursor;
- (void)_parseTranslationUnit:(CXTranslationUnit)aTranslationUnit withState:(struct TQHeaderParserState *)aState;
- (void)_importParserState:(struct TQHeaderParserState *)aState;
- (BOOL)_writeParserState:(struct TQHeaderParserState *)aState toFile:(NSString *)aPath;
@end

@implementation TQHeaderParser
- (id)init
{
    if(!(self = [super init]))
        return nil;

    _functions        = [NSMutableDictionary new];
    _literalConstants = [NSMutableDictionary new];
    _constants        = [NSMutableDictionary new];
    _classes          = [NSMutableDictionary new];
    _protocols        = [NSMutableDictionary new];
    _index            = clang_createIndex(0, 1);

    return self;
}
- (id)initWithCoder:(NSCoder *)aCoder
{
    if(!(self = [self init]))
        return nil;
    _functions        = [[aCoder decodeObject] retain];
    _classes          = [[aCoder decodeObject] retain];
    _literalConstants = [[aCoder decodeObject] retain];
    _constants        = [[aCoder decodeObject] retain];
    _protocols        = [[aCoder decodeObject] retain];
    return self;
}

- (id)parseHeader:(NSString *)aPath
{
    NSRange frameworkRange = [aPath rangeOfString:@".framework/"];
    NSString *frameworksPath = nil;
    if(frameworkRange.location != NSNotFound)
        frameworksPath = [[aPath substringToIndex:NSMaxRange(frameworkRange)] stringByDeletingLastPathComponent];

    // We create a temp TQPCH file so that future compiles go faster
    if(![aPath isAbsolutePath])
        aPath = [[[NSFileManager defaultManager] currentDirectoryPath] stringByAppendingPathComponent:aPath];
    NSString *tempPath = [NSString stringWithFormat:@"/tmp/tranquil_pch/%@.tqpch", [aPath stringByDeletingPathExtension]];
    if([[NSFileManager defaultManager] fileExistsAtPath:tempPath])
        return [self parseTQPCH:tempPath];
    [[NSFileManager defaultManager] createDirectoryAtPath:[tempPath stringByDeletingLastPathComponent]
    withIntermediateDirectories:YES attributes:nil error:nil];
    const char *args[] = {
        "-x", "objective-c",
        "-I/usr/local/tranquil/llvm/lib/clang/3.2/include/", // TODO: Figure out why and if this is necessary, and if it is necessary, make it not hard-coded..
        frameworksPath ? [[@"-F" stringByAppendingString:frameworksPath] UTF8String] : nil
    };

    CXTranslationUnit translationUnit = clang_parseTranslationUnit(_index, [aPath fileSystemRepresentation], args, frameworksPath ? 4 : 3, NULL, 0,
                                                                   CXTranslationUnit_DetailedPreprocessingRecord|CXTranslationUnit_SkipFunctionBodies);
    if (!translationUnit) {
        TQLog(@"Couldn't parse header %@\n", aPath);
        return nil;
    }
    struct TQHeaderParserState parserState = EMPTY_PARSER_STATE;
    [self _parseTranslationUnit:translationUnit withState:&parserState];
    [self _importParserState:&parserState];

{'fraction_non_alphanumeric': 0.10803705558337506, 'fraction_numerical': 0.002253380070105158, 'mean_word_length': 4.235255570117955, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1687579', 'n_tokens_mistral': 2385, 'n_tokens_neox': 2248, 'n_words': 499}