<commit_before><?php

class Generate
{
    public function domainWorker()
    {
        $domainSting = "";
        $lines = Array();
        $handle = fopen(__DIR__ . "/domains.txt", "r");

        if (!$handle) {
            throw new \RuntimeException('Error opening file domains.txt');
        }

        while (($line = fgets($handle)) !== false) {
            $line = preg_quote(trim(preg_replace('/\s\s+/', ' ', $line)));
            if (empty($line)) {
                continue;
            }
            array_push($lines, $line);
        }
        fclose($handle);

        // Sort the array
        sort($lines);

        // Removes duplicate values from the array
        $uniqueLines = array_unique($lines);

        foreach ($uniqueLines as $line) {
            $domainSting .= str_replace("\\", "", $line) . "\n";
        }

        file_put_contents("domains.txt", $domainSting);

        // Return the lines for later usage
        return $lines;
    }

    public function createApache($date, $lines)
    {

        $file = '../.htaccess';

        $data = "# https://github.com/Stevie-Ray/apache-nginx-referral-spam-blacklist
# Updated " . $date . "\n
<IfModule mod_rewrite.c>\n
RewriteEngine On\n\n";

        foreach ($lines as $line) {
            $data .= "RewriteCond %{HTTP_REFERER} ^http(s)?://(www.)?.*" . $line . ".*$ [NC,OR]\n";
        }

        $data .= "</IfModule>

# Apache 2.2
<IfModule !mod_authz_core.c>
    <IfModule mod_authz_host.c>
        Order allow,deny
        Allow from all
        Deny from env=spambot
    </IfModule>
</IfModule>

# Apache 2.4
<IfModule mod_authz_core.c>
    <RequireAll>
        Require all granted
        Require not env spambot
    </RequireAll>
</IfModule>";

        // Write the contents back to the
        file_put_contents($file, $data);
    }

    public function createNginx($date, $lines)
    {
        $file = '../referral-spam.conf';

        $data = "# https://github.com/Stevie-Ray/apache-nginx-referral-spam-blacklist
# Updated " . $date . "
#
# /etc/nginx/referral-spam.conf
#
# With referral-spam.conf in /etc/nginx, include it globally from within /etc/nginx/nginx.conf:
#
#     include referral-spam.conf;
#
# Add the following to each /etc/nginx/site-available/your-site.conf that needs protection:
#
#      server {
#        if (\$bad_referer) {
#          return 444;
#        }
#      }
#
map \$http_referer \$bad_referer {
\tdefault 0;\n\n";

        foreach ($lines as $line) {
            $data .= "\t\"~*" . $line . "\" 1;\n";
        }

        $data .= "\n}";

        // Write the contents back to the
        file_put_contents($file, $data);
    }
}

date_default_timezone_set('UTC');
$date = date('Y-m-d H:i:s');

$generator = new Generate();

$lines = $generator->domainWorker();

$generator->createApache($date, $lines);
$generator->createNginx($date, $lines);<commit_msg>#61 thanks @Gamesh, moved preg_quote added SORT_STRING flag<commit_after><?php
class Generate
{
    /**
     * @return array
     */
    public function domainWorker()
    {
        $domainsFile = __DIR__ . "/domains.txt";
        $handle = fopen($domainsFile, "r");
        if (! $handle) {
            throw new \RuntimeException('Error opening file ' . $domainsFile);
        }
        $lines = array();
        while (($line = fgets($handle)) !== false) {
            $line = trim(preg_replace('/\s\s+/', ' ', $line));
            if (empty($line)) {
                continue;
            }
            $lines[] = $line;
        }
        fclose($handle);
        $uniqueLines = array_unique($lines, SORT_STRING);
        sort($uniqueLines, SORT_STRING);
        file_put_contents($domainsFile, implode("\n", $uniqueLines));
        return $lines;
    }
    /**
     * @param string $date
     * @param array  $lines
     */
    public function createApache($date, array $lines)
    {
        $file = __DIR__ . '/../.htaccess';
        $data = "# https://github.com/Stevie-Ray/apache-nginx-referral-spam-blacklist
# Updated " . $date . "\n
<IfModule mod_rewrite.c>\n
RewriteEngine On\n\n";
        foreach ($lines as $line) {
            $data .= "RewriteCond %{HTTP_REFERER} ^http(s)?://(www.)?.*" . preg_quote($line) . ".*$ [NC,OR]\n";
        }
        $data .= "</IfModule>
# Apache 2.2
<IfModule !mod_authz_core.c>
    <IfModule mod_authz_host.c>
        Order allow,deny
        Allow from all
        Deny from env=spambot
    </IfModule>
</IfModule>
# Apache 2.4
<IfModule mod_authz_core.c>
    <RequireAll>
        Require all granted
        Require not env spambot
    </RequireAll>
</IfModule>";
        file_put_contents($file, $data);
    }
    /**
     * @param string $date
     * @param array  $lines
     */
    public function createNginx($date, array $lines)
    {
        $file = __DIR__ . '/../referral-spam.conf';
        $data = "# https://github.com/Stevie-Ray/apache-nginx-referral-spam-blacklist
# Updated " . $date . "
#
# /etc/nginx/referral-spam.conf
#
# With referral-spam.conf in /etc/nginx, include it globally from within /etc/nginx/nginx.conf:
#
#     include referral-spam.conf;
#
# Add the following to each /etc/nginx/site-available/your-site.conf that needs protection:
#
#      server {
#        if (\$bad_referer) {
#          return 444;
#        }
#      }
#
map \$http_referer \$bad_referer {
\tdefault 0;\n\n";
        foreach ($lines as $line) {
            $data .= "\t\"~*" . preg_quote($line) . "\" 1;\n";
        }
        $data .= "\n}";
        file_put_contents($file, $data);
    }
}
date_default_timezone_set('UTC');
$date = date('Y-m-d H:i:s');
$generator = new Generate();
$lines = $generator->domainWorker();
$generator->createApache($date, $lines);
$generator->createNginx($date, $lines);
{'fraction_non_alphanumeric': 0.1664338106880894, 'fraction_numerical': 0.0034928396786587496, 'mean_word_length': 2.1553719008264465, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5677286', 'n_tokens_mistral': 1947, 'n_tokens_neox': 1882, 'n_words': 397}