<commit_before>//
//  WebcastsParser.m
//  CERN App
//
//  Created by <NAME> on 8/16/12.
//  Copyright (c) 2012 CERN. All rights reserved.
//

#include <cassert>

#import "WebcastsParser.h"

NSString * const webcastURL = @"http://webcast.web.cern.ch/webcast/";

@implementation WebcastsParser {
   NSString *htmlString;
   NSMutableData *asyncData;
   NSUInteger numParsersLoading;
   
   NSURLConnection *currentConnection;
}

@synthesize recentWebcasts, upcomingWebcasts;
@synthesize delegate, pendingHTMLStringLoad, pendingRecentWebcastsParse, pendingUpcomingWebcastsParse;

//________________________________________________________________________________________
- (id) init
{
   return self = [super init];
}

//________________________________________________________________________________________
- (void) loadHTMLString
{
   if (!pendingHTMLStringLoad) {
      pendingHTMLStringLoad = YES;
      asyncData = [[NSMutableData alloc] init];
      self.recentWebcasts = [[NSMutableArray alloc] init];
      self.upcomingWebcasts = [[NSMutableArray alloc] init];
      NSURLRequest *request = [NSURLRequest requestWithURL : [NSURL URLWithString : webcastURL]];
      currentConnection = [NSURLConnection connectionWithRequest : request delegate : self];
   }
}

//________________________________________________________________________________________
- (void) parseRecentWebcasts
{
   if (!htmlString) {
      pendingRecentWebcastsParse = YES;
      [self loadHTMLString];
      return;
   }

   NSScanner *scanner = [NSScanner scannerWithString:htmlString];
   numParsersLoading = 0;
   while ([scanner scanUpToString : @"<div class=\"recentEvents\"" intoString : nil]) {
      NSString * const beginningOfLink = @"<a href=\"https://cdsweb.cern.ch/record/";
      if ([scanner scanUpToString : beginningOfLink intoString : nil]) {
         [scanner scanString : beginningOfLink intoString : nil];
         NSString *webcastID = @"";
         [scanner scanUpToString : @"\"" intoString : &webcastID];
         NSString * const marcURLString = [NSString stringWithFormat : @"https://cdsweb.cern.ch/record/%@/export/xm?ln=en",
                                           webcastID];
         numParsersLoading++;
         CernMediaMARCParser * const parser = [[CernMediaMARCParser alloc] init];
         parser.delegate = self;
         parser.url = [NSURL URLWithString : marcURLString];
         parser.resourceTypes = @[@"mp40600", @"mp4mobile", @"jpgposterframe", @"jpgthumbnail", @"pngthumbnail"];
         [parser parse];
      }
   }

   pendingRecentWebcastsParse = NO;
}

//________________________________________________________________________________________
- (void) parseUpcomingWebcasts
{
   if (!htmlString) {
      pendingUpcomingWebcastsParse = YES;
      [self loadHTMLString];
      return;
   }
   
   NSScanner * const scanner = [NSScanner scannerWithString : htmlString];

   while ([scanner scanUpToString : @"<div class=\"upcomEvents timezoneChange\"" intoString : nil]) {
      NSMutableDictionary * const webcast = [NSMutableDictionary dictionary];
      // Extract the image URL
      if ([scanner scanUpToString : @"<img class=\"upcomImg\" src=\"" intoString : nil]) {
         [scanner scanString : @"<img class=\"upcomImg\" src=\"" intoString : NULL];
         NSString *imageURL = nil;
         [scanner scanUpToString : @"\"" intoString : &imageURL];
         if (imageURL)
            [webcast setValue : [NSURL URLWithString : imageURL] forKey : @"thumbnailURL"];
      }
      
      // Extract the webcast title
      if ([scanner scanUpToString : @"<div class=\"upcomEventTitle\" title=\"" intoString : nil]) {
         [scanner scanString : @"<div class=\"upcomEventTitle\" title=\"" intoString : nil];
         NSString *title = nil;
         [scanner scanUpToString : @"\"" intoString : &title];
         if (title)
            [webcast setValue : title forKey : @"title"];<commit_msg>Delete all previous webcast metadata as we start a new parse.<commit_after>//
//  WebcastsParser.m
//  CERN App
//
//  Created by <NAME> on 8/16/12.
//  Copyright (c) 2012 CERN. All rights reserved.
//

#include <cassert>

#import "WebcastsParser.h"

NSString * const webcastURL = @"http://webcast.web.cern.ch/webcast/";

@implementation WebcastsParser {
   NSString *htmlString;
   NSMutableData *asyncData;
   NSUInteger numParsersLoading;
   
   NSURLConnection *currentConnection;
}

@synthesize recentWebcasts, upcomingWebcasts;
@synthesize delegate, pendingHTMLStringLoad, pendingRecentWebcastsParse, pendingUpcomingWebcastsParse;

//________________________________________________________________________________________
- (id) init
{
   return self = [super init];
}

//________________________________________________________________________________________
- (void) loadHTMLString
{
   if (!pendingHTMLStringLoad) {
      pendingHTMLStringLoad = YES;
      asyncData = [[NSMutableData alloc] init];
      self.recentWebcasts = [[NSMutableArray alloc] init];
      self.upcomingWebcasts = [[NSMutableArray alloc] init];
      NSURLRequest *request = [NSURLRequest requestWithURL : [NSURL URLWithString : webcastURL]];
      currentConnection = [NSURLConnection connectionWithRequest : request delegate : self];
   }
}

//________________________________________________________________________________________
- (void) parseRecentWebcasts
{
   if (!htmlString) {
      pendingRecentWebcastsParse = YES;
      [self loadHTMLString];
      return;
   }

   NSScanner *scanner = [NSScanner scannerWithString:htmlString];
   numParsersLoading = 0;
   [recentWebcasts removeAllObjects];
   while ([scanner scanUpToString : @"<div class=\"recentEvents\"" intoString : nil]) {
      NSString * const beginningOfLink = @"<a href=\"https://cdsweb.cern.ch/record/";
      if ([scanner scanUpToString : beginningOfLink intoString : nil]) {
         [scanner scanString : beginningOfLink intoString : nil];
         NSString *webcastID = @"";
         [scanner scanUpToString : @"\"" intoString : &webcastID];
         NSString * const marcURLString = [NSString stringWithFormat : @"https://cdsweb.cern.ch/record/%@/export/xm?ln=en",
                                           webcastID];
         numParsersLoading++;
         CernMediaMARCParser * const parser = [[CernMediaMARCParser alloc] init];
         parser.delegate = self;
         parser.url = [NSURL URLWithString : marcURLString];
         parser.resourceTypes = @[@"mp40600", @"mp4mobile", @"jpgposterframe", @"jpgthumbnail", @"pngthumbnail"];
         [parser parse];
      }
   }

   pendingRecentWebcastsParse = NO;
}

//________________________________________________________________________________________
- (void) parseUpcomingWebcasts
{
   if (!htmlString) {
      pendingUpcomingWebcastsParse = YES;
      [self loadHTMLString];
      return;
   }
   
   NSScanner * const scanner = [NSScanner scannerWithString : htmlString];
   [upcomingWebcasts removeAllObjects];
   while ([scanner scanUpToString : @"<div class=\"upcomEvents timezoneChange\"" intoString : nil]) {
      NSMutableDictionary * const webcast = [NSMutableDictionary dictionary];
      // Extract the image URL
      if ([scanner scanUpToString : @"<img class=\"upcomImg\" src=\"" intoString : nil]) {
         [scanner scanString : @"<img class=\"upcomImg\" src=\"" intoString : NULL];
         NSString *imageURL = nil;
         [scanner scanUpToString : @"\"" intoString : &imageURL];
         if (imageURL)
            [webcast setValue : [NSURL URLWithString : imageURL] forKey : @"thumbnailURL"];
      }
      
      // Extract the webcast title
      if ([scanner scanUpToString : @"<div class=\"upcomEventTitle\" title=\"" intoString : nil]) {
         [scanner scanString : @"<div class=\"upcomEventTitle\" title=\"" intoString : nil];
         NSString *title = nil;
         [scanner scanUpToString : @"\"" intoString : &title];
         if (title)
            [webcast setValue : title forKey : @"title"];
{'fraction_non_alphanumeric': 0.11600550619446878, 'fraction_numerical': 0.004004505068201727, 'mean_word_length': 3.4974676420934157, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7530934', 'n_tokens_mistral': 2311, 'n_tokens_neox': 2128, 'n_words': 511}