<commit_before><?php

class Application_Model_Preference
{

    public static function SetValue($key, $value, $isUserValue = false){       
        try {
            $con = Propel::getConnection();

            //called from a daemon process
            if(!class_exists("Zend_Auth", false) || !Zend_Auth::getInstance()->hasIdentity()) {
                $id = NULL;
            }
            else {
                $auth = Zend_Auth::getInstance();
                $id = $auth->getIdentity()->id;
            }

            $key = pg_escape_string($key);
            $value = pg_escape_string($value);

            //Check if key already exists
            $sql = "SELECT COUNT(*) FROM cc_pref"
            ." WHERE keystr = '$key'";

            //For user specific preference, check if id matches as well
            if($isUserValue) {
                $sql .= " AND subjid = '$id'";
            }
            
            $result = $con->query($sql)->fetchColumn(0);

            if($value == "") {
                $value = "NULL";
            }else {
                $value = "'$value'";
            }
            
            if($result == 1) {
                // result found
                if(is_null($id) || !$isUserValue) {
                    // system pref
                    $sql = "UPDATE cc_pref"
                    ." SET subjid = NULL, valstr = $value"
                    ." WHERE keystr = '$key'";
                } else {
                    // user pref
                    $sql = "UPDATE cc_pref"
                    . " SET valstr = $value"
                    . " WHERE keystr = '$key' AND subjid = $id";
                }
            } else {
                // result not found
                if(is_null($id) || !$isUserValue) {
                    // system pref
                    $sql = "INSERT INTO cc_pref (keystr, valstr)"
                    ." VALUES ('$key', $value)";
                } else {
                    // user pref
                    $sql = "INSERT INTO cc_pref (subjid, keystr, valstr)"
                    ." VALUES ($id, '$key', $value)";
                }
            }

            $con->exec($sql);
        
        } catch (Exception $e){
            header('HTTP/1.0 503 Service Unavailable');
            Logging::log("Could not connect to database.");
            exit;
        }

    }

    public static function GetValue($key, $isUserValue = false){       
        try {
            $con = Propel::getConnection();

            //Check if key already exists
            $sql = "SELECT COUNT(*) FROM cc_pref"
            ." WHERE keystr = '$key'";
            //For user specific preference, check if id matches as well
            if ($isUserValue) {
                $auth = Zend_Auth::getInstance();
                if($auth->hasIdentity()) {
                    $id = $auth->getIdentity()->id;
                    $sql .= " AND subjid = '$id'";
                }
            }
            $result = $con->query($sql)->fetchColumn(0);
            if ($result == 0)
                return "";
            else {
                $sql = "SELECT valstr FROM cc_pref"
                ." WHERE keystr = '$key'";

                //For user specific preference, check if id matches as well
                if($isUserValue && $auth->hasIdentity()) {
                    $sql .= " AND subjid = '$id'";
                }

                $result = $con->query($sql)->fetchColumn(0);
                return ($result !== false) ? $result : "";
            }
        } catch (Exception $e) {
            header('HTTP/1.0 503 Service Unavailable');
            Logging::log("Could not connect to database.");
            exit;            
        }
    }

    public static function GetHeadTitle(){
        $title = self::GetValue("station_name");
        $defaultNamespace->title = $title;
        if (strlen($title) > 0)
            $title .= " - ";

        return $title."Airtime";
    }

    public static function SetHeadTitle($title, $view=null){
        self::SetValue("station_name", $title);

        // in case this is called from airtime-saas script
        if($view !== null){
            //set session variable to new station name so that html title is updated.
            //should probably do this in a view helper to keep this controller as minimal as possible.
            $view->headTitle()->exchangeArray(array()); //clear headTitle ArrayObject
            $view->headTitle(self::GetHeadTitle());
        }

        $eventType = "update_station_name";
        $md = array("station_name"=>$title);

        Application_Model_RabbitMq::SendMessageToPypo($eventType, $md);
    }

    /**
     * Set the furthest date that a never-ending show
     * should be populated until.
     *
     * @param DateTime $dateTime
     *        A row from cc_show_days table
     */
    public static function SetShowsPopulatedUntil($dateTime) {
        self::SetValue("shows_populated_until", $dateTime->format("Y-m-d H:i:s"));
    }

    /**
     * Get the furthest date that a never-ending show
     * should be populated until.
     *
     * Returns null if the value hasn't been set, otherwise returns
     * a DateTime object representing the date.
     *
     * @return DateTime (in UTC Timezone)
     */
    public static function GetShowsPopulatedUntil() {
        $date = self::GetValue("shows_populated_until");

        if ($date == ""){
            return null;
        } else {
            return new DateTime($date, new DateTimeZone("UTC"));
        }
    }

    public static function SetDefaultFade($fade) {
        self::SetValue("default_fade", $fade);
    }

    public static function GetDefaultFade() {
        $fade = self::GetValue("default_fade");
        
        if ($fade === "") {
            // the default value of the fade is 00.500000
            return "00.500000";
        } 
        
        $fade = number_format($fade, 6);
        //fades need 2 leading zeros for DateTime conversion
        $fade = str_pad($fade, 9, "0", STR_PAD_LEFT);
        return $fade;
    }

    public static function SetDefaultTransitionFade($fade) {
        self::SetValue("default_transition_fade", $fade);

        $eventType = "update_transition_fade";
        $md = array("transition_fade"=>$fade);
        Application_Model_RabbitMq::SendMessageToPypo($eventType, $md);
    }

    public static function GetDefaultTransitionFade() {
        $transition_fade = self::GetValue("default_transition_fade");
        if($transition_fade == ""){
            $transition_fade = "00.000000";
        }
        return $transition_fade;
    }

    public static function SetStreamLabelFormat($type){
        self::SetValue("stream_label_format", $type);

        $eventType = "update_stream_format";
        $md = array("stream_format"=>$type);

        Application_Model_RabbitMq::SendMessageToPypo($eventType, $md);
    }

    public static function GetStreamLabelFormat(){
        return self::getValue("stream_label_format");
    }

    public static function GetStationName(){
        return self::getValue("station_name");
    }

    public static function SetAutoUploadRecordedShowToSoundcloud($upload) {
        self::SetValue("soundcloud_auto_upload_recorded_show", $upload);
    }

    public static function GetAutoUploadRecordedShowToSoundcloud() {
        return self::GetValue("soundcloud_auto_upload_recorded_show");
    }

    public static function SetSoundCloudUser($user) {
        self::SetValue("soundcloud_user", $user);
    }

    public static function GetSoundCloudUser() {
        return self::GetValue("soundcloud_user");
    }

    public static function SetSoundCloudPassword($password) {
        if (strlen($password) > 0)
            self::SetValue("soundcloud_password", $password);
    }

    public static function GetSoundCloudPassword() {
        return self::GetValue("soundcloud_password");
    }

    public static function SetSoundCloudTags($tags) {
        self::SetValue("soundcloud_tags", $tags);
    }

    public static function GetSoundCloudTags() {
        return self::GetValue("soundcloud_tags");
    }

    public static function SetSoundCloudGenre($genre) {
        self::SetValue("soundcloud_genre", $genre);
    }

    public static function GetSoundCloudGenre() {
        return self::GetValue("soundcloud_genre");
    }

    public static function SetSoundCloudTrackType($track_type) {
        self::SetValue("soundcloud_tracktype", $track_type);
    }

    public static function GetSoundCloudTrackType() {
        return self::GetValue("soundcloud_tracktype");
    }

    public static function SetSoundCloudLicense($license) {
        self::SetValue("soundcloud_license", $license);
    }

    public static function GetSoundCloudLicense() {
        return self::GetValue("soundcloud_license");
    }

    public static function SetAllow3rdPartyApi($bool) {
        self::SetValue("third_party_api", $bool);
    }

    public static function GetAllow3rdPartyApi() {
        $val = self::GetValue("third_party_api");
        if (strlen($val) == 0){
            return "0";
        } else {
            return $val;
        }
    }

    public static function SetPhone($phone){
    	self::SetValue("phone", $phone);
    }

    public static function GetPhone(){
    	return self::GetValue("phone");
    }

	public static function SetEmail($email){
    	self::SetValue("email", $email);
    }

    public static function GetEmail(){
    	return self::GetValue("email");
    }

	public static function SetStationWebSite($site){
    	self::SetValue("station_website", $site);
    }

    public static function GetStationWebSite(){
    	return self::GetValue("station_website");
    }

	public static function SetSupportFeedback($feedback){
    	self::SetValue("support_feedback", $feedback);
    }

    public static function GetSupportFeedback(){
    	return self::GetValue("support_feedback");
    }

	public static function SetPublicise($publicise){
    	self::SetValue("publicise", $publicise);
    }

    public static function GetPublicise(){
    	return self::GetValue("publicise");
    }

	public static function SetRegistered($registered){
    	self::SetValue("registered", $registered);
    }

    public static function GetRegistered(){
    	return self::GetValue("registered");
    }

	public static function SetStationCountry($country){
    	self::SetValue("country", $country);
    }

    public static function GetStationCountry(){
    	return self::GetValue("country");
    }

	public static function SetStationCity($city){
    	self::SetValue("city", $city);
    }

	public static function GetStationCity(){
    	return self::GetValue("city");
    }

	public static function SetStationDescription($description){
    	self::SetValue("description", $description);
    }

	public static function GetStationDescription(){
    	return self::GetValue("description");
    }

    public static function SetTimezone($timezone){
        self::SetValue("timezone", $timezone);
        date_default_timezone_set($timezone);
        $md = array("timezone" => $timezone);
    }

    public static function GetTimezone(){
        return self::GetValue("timezone");
    }

    public static function SetStationLogo($imagePath){
    	if(!empty($imagePath)){
	    	$image = @file_get_contents($imagePath);
	    	$image = base64_encode($image);
	    	self::SetValue("logoImage", $image);
    	}
    }

	public static function GetStationLogo(){
    	return self::GetValue("logoImage");
    }

    public static function GetUniqueId(){
    	return self::GetValue("uniqueId");
    }

    public static function GetCountryList()
    {
    	$con = Propel::getConnection();
    	$sql = "SELECT * FROM cc_country";
    	$res =  $con->query($sql)->fetchAll();
    	$out = array();
    	$out[""] = "Select Country";
    	foreach($res as $r){
    		$out[$r["isocode"]] = $r["name"];
    	}
    	return $out;
    }

    public static function GetSystemInfo($returnArray=false, $p_testing=false)
    {
    	exec('/usr/bin/airtime-check-system --no-color', $output);
    	$output = preg_replace('/\s+/', ' ', $output);

    	$systemInfoArray = array();
    	foreach( $output as $key => &$out){
    		$info = explode('=', $out);
    		if(isset($info[1])){
    			$key = str_replace(' ', '_', trim($info[0]));
    			$key = strtoupper($key);
    			if ($key == 'WEB_SERVER' || $key == 'CPU' || $key == 'OS'  || $key == 'TOTAL_RAM' ||
    	                  $key == 'FREE_RAM' || $key == 'AIRTIME_VERSION' || $key == 'KERNAL_VERSION'  || 
    	                  $key == 'MACHINE_ARCHITECTURE' || $key == 'TOTAL_MEMORY_MBYTES' || $key == 'TOTAL_SWAP_MBYTES' ||
    	                  $key == 'PLAYOUT_ENGINE_CPU_PERC' ) {
    	            if($key == 'AIRTIME_VERSION'){
    	                // remove hash tag on the version string
    	                list($version, $dump) = explode('+', $info[1]);
    			        $systemInfoArray[$key] = $version;
    	            }else{
    	                $systemInfoArray[$key] = $info[1];
    	            }
    	        }
    		}
    	}
    	
    	$outputArray = array();

    	$outputArray['LIVE_DURATION'] = Application_Model_LiveLog::GetLiveShowDuration($p_testing);
    	$outputArray['SCHEDULED_DURATION'] = Application_Model_LiveLog::GetScheduledDuration($p_testing);
    	$outputArray['SOUNDCLOUD_ENABLED'] = self::GetUploadToSoundcloudOption();
        if ($outputArray['SOUNDCLOUD_ENABLED']) {
    	    $outputArray['NUM_SOUNDCLOUD_TRACKS_UPLOADED'] = Application_Model_StoredFile::getSoundCloudUploads();
        } else {
            $outputArray['NUM_SOUNDCLOUD_TRACKS_UPLOADED'] = NULL;
        }<commit_msg>CC-3863: Stats should not send version number with extra characters<commit_after><?php

class Application_Model_Preference
{

    public static function SetValue($key, $value, $isUserValue = false){       
        try {
            $con = Propel::getConnection();

            //called from a daemon process
            if(!class_exists("Zend_Auth", false) || !Zend_Auth::getInstance()->hasIdentity()) {
                $id = NULL;
            }
            else {
                $auth = Zend_Auth::getInstance();
                $id = $auth->getIdentity()->id;
            }

            $key = pg_escape_string($key);
            $value = pg_escape_string($value);

            //Check if key already exists
            $sql = "SELECT COUNT(*) FROM cc_pref"
            ." WHERE keystr = '$key'";

            //For user specific preference, check if id matches as well
            if($isUserValue) {
                $sql .= " AND subjid = '$id'";
            }
            
            $result = $con->query($sql)->fetchColumn(0);

            if($value == "") {
                $value = "NULL";
            }else {
                $value = "'$value'";
            }
            
            if($result == 1) {
                // result found
                if(is_null($id) || !$isUserValue) {
                    // system pref
                    $sql = "UPDATE cc_pref"
                    ." SET subjid = NULL, valstr = $value"
                    ." WHERE keystr = '$key'";
                } else {
                    // user pref
                    $sql = "UPDATE cc_pref"
                    . " SET valstr = $value"
                    . " WHERE keystr = '$key' AND subjid = $id";
                }
            } else {
                // result not found
                if(is_null($id) || !$isUserValue) {
                    // system pref
                    $sql = "INSERT INTO cc_pref (keystr, valstr)"
                    ." VALUES ('$key', $value)";
                } else {
                    // user pref
                    $sql = "INSERT INTO cc_pref (subjid, keystr, valstr)"
                    ." VALUES ($id, '$key', $value)";
                }
            }

            $con->exec($sql);
        
        } catch (Exception $e){
            header('HTTP/1.0 503 Service Unavailable');
            Logging::log("Could not connect to database.");
            exit;
        }

    }

    public static function GetValue($key, $isUserValue = false){       
        try {
            $con = Propel::getConnection();

            //Check if key already exists
            $sql = "SELECT COUNT(*) FROM cc_pref"
            ." WHERE keystr = '$key'";
            //For user specific preference, check if id matches as well
            if ($isUserValue) {
                $auth = Zend_Auth::getInstance();
                if($auth->hasIdentity()) {
                    $id = $auth->getIdentity()->id;
                    $sql .= " AND subjid = '$id'";
                }
            }
            $result = $con->query($sql)->fetchColumn(0);
            if ($result == 0)
                return "";
            else {
                $sql = "SELECT valstr FROM cc_pref"
                ." WHERE keystr = '$key'";

                //For user specific preference, check if id matches as well
                if($isUserValue && $auth->hasIdentity()) {
                    $sql .= " AND subjid = '$id'";
                }

                $result = $con->query($sql)->fetchColumn(0);
                return ($result !== false) ? $result : "";
            }
        } catch (Exception $e) {
            header('HTTP/1.0 503 Service Unavailable');
            Logging::log("Could not connect to database.");
            exit;            
        }
    }

    public static function GetHeadTitle(){
        $title = self::GetValue("station_name");
        $defaultNamespace->title = $title;
        if (strlen($title) > 0)
            $title .= " - ";

        return $title."Airtime";
    }

    public static function SetHeadTitle($title, $view=null){
        self::SetValue("station_name", $title);

        // in case this is called from airtime-saas script
        if($view !== null){
            //set session variable to new station name so that html title is updated.
            //should probably do this in a view helper to keep this controller as minimal as possible.
            $view->headTitle()->exchangeArray(array()); //clear headTitle ArrayObject
            $view->headTitle(self::GetHeadTitle());
        }

        $eventType = "update_station_name";
        $md = array("station_name"=>$title);

        Application_Model_RabbitMq::SendMessageToPypo($eventType, $md);
    }

    /**
     * Set the furthest date that a never-ending show
     * should be populated until.
     *
     * @param DateTime $dateTime
     *        A row from cc_show_days table
     */
    public static function SetShowsPopulatedUntil($dateTime) {
        self::SetValue("shows_populated_until", $dateTime->format("Y-m-d H:i:s"));
    }

    /**
     * Get the furthest date that a never-ending show
     * should be populated until.
     *
     * Returns null if the value hasn't been set, otherwise returns
     * a DateTime object representing the date.
     *
     * @return DateTime (in UTC Timezone)
     */
    public static function GetShowsPopulatedUntil() {
        $date = self::GetValue("shows_populated_until");

        if ($date == ""){
            return null;
        } else {
            return new DateTime($date, new DateTimeZone("UTC"));
        }
    }

    public static function SetDefaultFade($fade) {
        self::SetValue("default_fade", $fade);
    }

    public static function GetDefaultFade() {
        $fade = self::GetValue("default_fade");
        
        if ($fade === "") {
            // the default value of the fade is 00.500000
            return "00.500000";
        } 
        
        $fade = number_format($fade, 6);
        //fades need 2 leading zeros for DateTime conversion
        $fade = str_pad($fade, 9, "0", STR_PAD_LEFT);
        return $fade;
    }

    public static function SetDefaultTransitionFade($fade) {
        self::SetValue("default_transition_fade", $fade);

        $eventType = "update_transition_fade";
        $md = array("transition_fade"=>$fade);
        Application_Model_RabbitMq::SendMessageToPypo($eventType, $md);
    }

    public static function GetDefaultTransitionFade() {
        $transition_fade = self::GetValue("default_transition_fade");
        if($transition_fade == ""){
            $transition_fade = "00.000000";
        }
        return $transition_fade;
    }

    public static function SetStreamLabelFormat($type){
        self::SetValue("stream_label_format", $type);

        $eventType = "update_stream_format";
        $md = array("stream_format"=>$type);

        Application_Model_RabbitMq::SendMessageToPypo($eventType, $md);
    }

    public static function GetStreamLabelFormat(){
        return self::getValue("stream_label_format");
    }

    public static function GetStationName(){
        return self::getValue("station_name");
    }

    public static function SetAutoUploadRecordedShowToSoundcloud($upload) {
        self::SetValue("soundcloud_auto_upload_recorded_show", $upload);
    }

    public static function GetAutoUploadRecordedShowToSoundcloud() {
        return self::GetValue("soundcloud_auto_upload_recorded_show");
    }

    public static function SetSoundCloudUser($user) {
        self::SetValue("soundcloud_user", $user);
    }

    public static function GetSoundCloudUser() {
        return self::GetValue("soundcloud_user");
    }

    public static function SetSoundCloudPassword($password) {
        if (strlen($password) > 0)
            self::SetValue("soundcloud_password", $password);
    }

    public static function GetSoundCloudPassword() {
        return self::GetValue("soundcloud_password");
    }

    public static function SetSoundCloudTags($tags) {
        self::SetValue("soundcloud_tags", $tags);
    }

    public static function GetSoundCloudTags() {
        return self::GetValue("soundcloud_tags");
    }

    public static function SetSoundCloudGenre($genre) {
        self::SetValue("soundcloud_genre", $genre);
    }

    public static function GetSoundCloudGenre() {
        return self::GetValue("soundcloud_genre");
    }

    public static function SetSoundCloudTrackType($track_type) {
        self::SetValue("soundcloud_tracktype", $track_type);
    }

    public static function GetSoundCloudTrackType() {
        return self::GetValue("soundcloud_tracktype");
    }

    public static function SetSoundCloudLicense($license) {
        self::SetValue("soundcloud_license", $license);
    }

    public static function GetSoundCloudLicense() {
        return self::GetValue("soundcloud_license");
    }

    public static function SetAllow3rdPartyApi($bool) {
        self::SetValue("third_party_api", $bool);
    }

    public static function GetAllow3rdPartyApi() {
        $val = self::GetValue("third_party_api");
        if (strlen($val) == 0){
            return "0";
        } else {
            return $val;
        }
    }

    public static function SetPhone($phone){
    	self::SetValue("phone", $phone);
    }

    public static function GetPhone(){
    	return self::GetValue("phone");
    }

	public static function SetEmail($email){
    	self::SetValue("email", $email);
    }

    public static function GetEmail(){
    	return self::GetValue("email");
    }

	public static function SetStationWebSite($site){
    	self::SetValue("station_website", $site);
    }

    public static function GetStationWebSite(){
    	return self::GetValue("station_website");
    }

	public static function SetSupportFeedback($feedback){
    	self::SetValue("support_feedback", $feedback);
    }

    public static function GetSupportFeedback(){
    	return self::GetValue("support_feedback");
    }

	public static function SetPublicise($publicise){
    	self::SetValue("publicise", $publicise);
    }

    public static function GetPublicise(){
    	return self::GetValue("publicise");
    }

	public static function SetRegistered($registered){
    	self::SetValue("registered", $registered);
    }

    public static function GetRegistered(){
    	return self::GetValue("registered");
    }

	public static function SetStationCountry($country){
    	self::SetValue("country", $country);
    }

    public static function GetStationCountry(){
    	return self::GetValue("country");
    }

	public static function SetStationCity($city){
    	self::SetValue("city", $city);
    }

	public static function GetStationCity(){
    	return self::GetValue("city");
    }

	public static function SetStationDescription($description){
    	self::SetValue("description", $description);
    }

	public static function GetStationDescription(){
    	return self::GetValue("description");
    }

    public static function SetTimezone($timezone){
        self::SetValue("timezone", $timezone);
        date_default_timezone_set($timezone);
        $md = array("timezone" => $timezone);
    }

    public static function GetTimezone(){
        return self::GetValue("timezone");
    }

    public static function SetStationLogo($imagePath){
    	if(!empty($imagePath)){
	    	$image = @file_get_contents($imagePath);
	    	$image = base64_encode($image);
	    	self::SetValue("logoImage", $image);
    	}
    }

	public static function GetStationLogo(){
    	return self::GetValue("logoImage");
    }

    public static function GetUniqueId(){
    	return self::GetValue("uniqueId");
    }

    public static function GetCountryList()
    {
    	$con = Propel::getConnection();
    	$sql = "SELECT * FROM cc_country";
    	$res =  $con->query($sql)->fetchAll();
    	$out = array();
    	$out[""] = "Select Country";
    	foreach($res as $r){
    		$out[$r["isocode"]] = $r["name"];
    	}
    	return $out;
    }

    public static function GetSystemInfo($returnArray=false, $p_testing=false)
    {
    	exec('/usr/bin/airtime-check-system --no-color', $output);
    	$output = preg_replace('/\s+/', ' ', $output);

    	$systemInfoArray = array();
    	foreach( $output as $key => &$out){
    		$info = explode('=', $out);
    		if(isset($info[1])){
    			$key = str_replace(' ', '_', trim($info[0]));
    			$key = strtoupper($key);
    			if ($key == 'WEB_SERVER' || $key == 'CPU' || $key == 'OS'  || $key == 'TOTAL_RAM' ||
    	                  $key == 'FREE_RAM' || $key == 'AIRTIME_VERSION' || $key == 'KERNAL_VERSION'  || 
    	                  $key == 'MACHINE_ARCHITECTURE' || $key == 'TOTAL_MEMORY_MBYTES' || $key == 'TOTAL_SWAP_MBYTES' ||
    	                  $key == 'PLAYOUT_ENGINE_CPU_PERC' ) {
    	            if($key == 'AIRTIME_VERSION'){
    	                // remove hash tag on the version string
    	                $version = explode('+', $info[1]);
    			        $systemInfoArray[$key] = $version[0];
    	            }else{
    	                $systemInfoArray[$key] = $info[1];
    	            }
    	        }
    		}
    	}
    	
    	$outputArray = array();

    	$outputArray['LIVE_DURATION'] = Application_Model_LiveLog::GetLiveShowDuration($p_testing);
    	$outputArray['SCHEDULED_DURATION'] = Application_Model_LiveLog::GetScheduledDuration($p_testing);
    	$outputArray['SOUNDCLOUD_ENABLED'] = self::GetUploadToSoundcloudOption();
        if ($outputArray['SOUNDCLOUD_ENABLED']) {
    	    $outputArray['NUM_SOUNDCLOUD_TRACKS_UPLOADED'] = Application_Model_StoredFile::getSoundCloudUploads();
        } else {
            $outputArray['NUM_SOUNDCLOUD_TRACKS_UPLOADED'] = NULL;
        }
{'fraction_non_alphanumeric': 0.13098457715079306, 'fraction_numerical': 0.004202909144068416, 'mean_word_length': 2.164816099930604, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7477775', 'n_tokens_mistral': 8026, 'n_tokens_neox': 7583, 'n_words': 1682}