<commit_before><?php

define('INSTALLDIR', realpath(dirname(__FILE__) . '/..'));
define('LACONICA', true);

require_once(INSTALLDIR . '/lib/common.php');
require_once(INSTALLDIR . '/lib/util.php');

$output_paths = parse_args();

standard_map();
notices_map();
user_map();
index_map();

# ------------------------------------------------------------------------------
# Main functions: get data out and turn them into sitemaps
# ------------------------------------------------------------------------------

# Generate index sitemap of all other sitemaps.
function index_map() {
	global $output_paths;
	$output_dir = $output_paths['output_dir'];
	$output_url = $output_paths['output_url'];

	foreach (glob("$output_dir*.xml") as $file_name) {

		# Just the file name please.
		$file_name = preg_replace("|$output_dir|", '', $file_name);

		$index_urls .= sitemap(
						   array(
								 'url' => $output_url . $file_name,
								 'changefreq' => 'daily'
								 )
						   );
	}

	write_file($output_paths['index_file'], sitemapindex($index_urls));
}

# Generate sitemap of standard site elements.
function standard_map() {
	global $output_paths;

	$standard_map_urls .= url(
							  array(
									'url' => common_local_url('public'),
									'changefreq' => 'daily',
									'priority' => '1',
									)
							  );

	$standard_map_urls .= url(
							  array(
									'url' => common_local_url('publicrss'),
									'changefreq' => 'daily',
									'priority' => '0.3',
									)
							  );

	$docs = array('about', 'faq', 'contact', 'im', 'openid', 'openmublog', 'privacy', 'source');

	foreach($docs as $title) {
		$standard_map_urls .= url(
								  array(
										'url' => common_local_url('doc', array('title' => $title)),
										'changefreq' => 'monthly',
										'priority'   => '0.2',
										)
								  );
	}

	$urlset_path = $output_paths['output_dir'] . 'standard.xml';

	write_file($urlset_path, urlset($standard_map_urls));
}

# Generate sitemaps of all notices.
function notices_map() {
	global $output_paths;

	$notices = DB_DataObject::factory('notice');

	$notices->query('SELECT id, uri, url, modified FROM notice');

	$notice_count = 0;
	$map_count = 1;

	while ($notices->fetch()) {

		# Maximum 50,000 URLs per sitemap file.
		if ($notice_count == 50000) {
			$notice_count = 0;
			$map_count++;
		}

		# remote notices have an URL
		
		if (!$notices->url && $notices->uri) {
			$notice = array(
						'url'        => ($notices->uri) ? $notices->uri : common_local_url('shownotice', array('notice' => $notices->id)),
						'lastmod'    => common_date_w3dtf($notices->modified),
						'changefreq' => 'never',
						'priority'   => '1',
						);

			$notice_list[$map_count] .= url($notice);
			$notice_count++;
		}
	}

	# Make full sitemaps from the lists and save them.
	array_to_map($notice_list, 'notice');<commit_msg>only put notices that are local in the sitemap<commit_after><?php

define('INSTALLDIR', realpath(dirname(__FILE__) . '/..'));
define('LACONICA', true);

require_once(INSTALLDIR . '/lib/common.php');
require_once(INSTALLDIR . '/lib/util.php');

$output_paths = parse_args();

standard_map();
notices_map();
user_map();
index_map();

# ------------------------------------------------------------------------------
# Main functions: get data out and turn them into sitemaps
# ------------------------------------------------------------------------------

# Generate index sitemap of all other sitemaps.
function index_map() {
	global $output_paths;
	$output_dir = $output_paths['output_dir'];
	$output_url = $output_paths['output_url'];

	foreach (glob("$output_dir*.xml") as $file_name) {

		# Just the file name please.
		$file_name = preg_replace("|$output_dir|", '', $file_name);

		$index_urls .= sitemap(
						   array(
								 'url' => $output_url . $file_name,
								 'changefreq' => 'daily'
								 )
						   );
	}

	write_file($output_paths['index_file'], sitemapindex($index_urls));
}

# Generate sitemap of standard site elements.
function standard_map() {
	global $output_paths;

	$standard_map_urls .= url(
							  array(
									'url' => common_local_url('public'),
									'changefreq' => 'daily',
									'priority' => '1',
									)
							  );

	$standard_map_urls .= url(
							  array(
									'url' => common_local_url('publicrss'),
									'changefreq' => 'daily',
									'priority' => '0.3',
									)
							  );

	$docs = array('about', 'faq', 'contact', 'im', 'openid', 'openmublog', 'privacy', 'source');

	foreach($docs as $title) {
		$standard_map_urls .= url(
								  array(
										'url' => common_local_url('doc', array('title' => $title)),
										'changefreq' => 'monthly',
										'priority'   => '0.2',
										)
								  );
	}

	$urlset_path = $output_paths['output_dir'] . 'standard.xml';

	write_file($urlset_path, urlset($standard_map_urls));
}

# Generate sitemaps of all notices.
function notices_map() {
	global $output_paths;

	$notices = DB_DataObject::factory('notice');

	$notices->query('SELECT id, uri, url, modified FROM notice where is_local = 1');

	$notice_count = 0;
	$map_count = 1;

	while ($notices->fetch()) {

		# Maximum 50,000 URLs per sitemap file.
		if ($notice_count == 50000) {
			$notice_count = 0;
			$map_count++;
		}

		# remote notices have an URL
		
		if (!$notices->url && $notices->uri) {
			$notice = array(
						'url'        => ($notices->uri) ? $notices->uri : common_local_url('shownotice', array('notice' => $notices->id)),
						'lastmod'    => common_date_w3dtf($notices->modified),
						'changefreq' => 'never',
						'priority'   => '1',
						);

			$notice_list[$map_count] .= url($notice);
			$notice_count++;
		}
	}

	# Make full sitemaps from the lists and save them.
	array_to_map($notice_list, 'notice');
{'fraction_non_alphanumeric': 0.21694856748360372, 'fraction_numerical': 0.007076285812909907, 'mean_word_length': 2.566153846153846, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6136786', 'n_tokens_mistral': 2393, 'n_tokens_neox': 1864, 'n_words': 390}