<commit_before>import logging
from datetime import datetime

from cwpoliticl.database.base.mysql_db import MysqlDatabase
from cwpoliticl.items import CacheItem
from cwpoliticl.utils.crawl_utils import CrawlUtils


class CacheDatabase(MysqlDatabase):
    def __init__(self, host, port, user, passwd, db, collection_name):
        super(CacheDatabase, self).__init__(host, port, user, passwd, db, collection_name)

    def save_cache(self, item, index=1):
        logging.debug("process the cache item at position: {}".format(index - 1))

        # sql = """ SELECT id FROM {} WHERE model_id = '{}' """.format(self.collection_name, item['model_id'])
        # _cache_id = self._get_row_id(sql, "Caches")

        # if _cache_id:
        #     logging.debug("  item exist {} from {} on the cache database".format(item["model_id"], item["url_from"]))
        # else:
        self._insert_for_cache(item)

    def _insert_for_cache(self, item):
        _excep = None
        _connection = self.get_client()
        _cursor = _connection.cursor()

        sql = "INSERT INTO " + self.collection_name + " (url, url_from, created_at) VALUES(%s,%s,%s)"

        try:
            # Execute the SQL command
            _cursor.execute(sql, (item['url'], item['url_from'], item['created_at']))
            # Commit your changes in the database
            _connection.commit()
        except Exception, e:
            _excep = e
            # Rollback in case there is any error
            _connection.rollback()
        finally:
            _cursor.close()
            _connection.close()

        if _excep:
            logging.debug("  mysql: insert the ads_caches row failure, {}".format(_excep))
        else:
            logging.debug(
                "  mysql: insert {} into the {} from the {} successfully".format(item['model_id'], self.collection_name,
                                                                                 item['url_from']))

    def get_oldest_row(self, _last, url_from):
        logging.debug("Get the oldest row")

        if _last:
            logging.debug("  1. delete the row by the last url: {}".format(_last))
            self._delete_cache_row(_last, url_from)

        # Query the oldest cache item.
        return self._find_oldest_for_cache()

    def _delete_cache_row(self, _last, url_from):
        _excep = None
        _deleted_count = 0

        # 1. Parse the url and get the unique model_id.
        model_id = CrawlUtils.get_model_id_by_url_from(_last, url_from)
        logging.debug("  2. get the last url's model_id: {}".format(model_id))

        # Query the deleted item count, must be equal to 1.
        sql = """ SELECT 1 FROM {} WHERE model_id = '{}'""".format(self.collection_name, model_id)
        count = self._get_count(sql, self.collection_name)
        logging.debug("  3. found the deleted item count: {} by model_id".format(count))
        if count:

            _connection = self.get_client()
            _cursor = _connection.cursor()
            sql = """ DELETE FROM {} WHERE model_id = '{}'""".format(self.collection_name, model_id)
            try:
                # Execute the SQL command
                _cursor.execute(sql)
                # Commit your changes in the database
                _connection.commit()
                _deleted_count = _cursor.rowcount
            except Exception, e:
                _excep = e
                # Rollback in case there is any error
                _connection.rollback()
            finally:
                _cursor.close()
                _connection.close()

        if _excep:
            logging.debug(
                "  mysql: delete the oldest cache row, model_id: {}, from the {} failure, {}".format(
                    model_id, url_from, _excep))
        else:
            logging.debug(
                "  4. deleted cache row, model_id: {}, deleted count: {}, from the {} successfully"
                    .format(model_id, _deleted_count, url_from))

    def _get_cache_total_count(self):
        _count = 0
        _connection = self.get_client()
        _cursor = _connection.cursor()

        sql = """ SELECT * FROM {} """.format(self.collection_name)<commit_msg>cwpoliticl.<commit_after>import logging
from datetime import datetime

from cwpoliticl.database.base.mysql_db import MysqlDatabase
from cwpoliticl.items import CacheItem
from cwpoliticl.utils.crawl_utils import CrawlUtils


class CacheDatabase(MysqlDatabase):
    def __init__(self, host, port, user, passwd, db, collection_name):
        super(CacheDatabase, self).__init__(host, port, user, passwd, db, collection_name)

    def save_cache(self, item, index=1):
        logging.debug("process the cache item at position: {}".format(index - 1))

        # sql = """ SELECT id FROM {} WHERE model_id = '{}' """.format(self.collection_name, item['model_id'])
        # _cache_id = self._get_row_id(sql, "Caches")

        # if _cache_id:
        #     logging.debug("  item exist {} from {} on the cache database".format(item["model_id"], item["url_from"]))
        # else:
        self._insert_for_cache(item)

    def _insert_for_cache(self, item):
        _excep = None
        _connection = self.get_client()
        _cursor = _connection.cursor()

        sql = "INSERT INTO " + self.collection_name + " (url, url_from, created_at) VALUES(%s,%s,%s)"

        try:
            # Execute the SQL command
            _cursor.execute(sql, (item['url'], item['url_from'], item['created_at']))
            # Commit your changes in the database
            _connection.commit()
        except Exception, e:
            _excep = e
            # Rollback in case there is any error
            _connection.rollback()
            logging.debug("  mysql: insert the ads_caches row failure, {}".format(_excep))
        finally:
            _cursor.close()
            _connection.close()

    def get_oldest_row(self, _last, url_from):
        logging.debug("Get the oldest row")

        if _last:
            logging.debug("  1. delete the row by the last url: {}".format(_last))
            self._delete_cache_row(_last, url_from)

        # Query the oldest cache item.
        return self._find_oldest_for_cache()

    def _delete_cache_row(self, _last, url_from):
        _excep = None
        _deleted_count = 0

        # 1. Parse the url and get the unique model_id.
        model_id = CrawlUtils.get_model_id_by_url_from(_last, url_from)
        logging.debug("  2. get the last url's model_id: {}".format(model_id))

        # Query the deleted item count, must be equal to 1.
        sql = """ SELECT 1 FROM {} WHERE model_id = '{}'""".format(self.collection_name, model_id)
        count = self._get_count(sql, self.collection_name)
        logging.debug("  3. found the deleted item count: {} by model_id".format(count))
        if count:

            _connection = self.get_client()
            _cursor = _connection.cursor()
            sql = """ DELETE FROM {} WHERE model_id = '{}'""".format(self.collection_name, model_id)
            try:
                # Execute the SQL command
                _cursor.execute(sql)
                # Commit your changes in the database
                _connection.commit()
                _deleted_count = _cursor.rowcount
            except Exception, e:
                _excep = e
                # Rollback in case there is any error
                _connection.rollback()
                logging.debug("  mysql: delete the oldest cache row, from the {} failure, {}".format(url_from, _excep))
            finally:
                _cursor.close()
                _connection.close()

    def _get_cache_total_count(self):
        _count = 0
        _connection = self.get_client()
        _cursor = _connection.cursor()

        sql = """ SELECT * FROM {} """.format(self.collection_name)
{'fraction_non_alphanumeric': 0.1068332483426823, 'fraction_numerical': 0.0026772055073941868, 'mean_word_length': 2.1430288461538463, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2878046', 'n_tokens_mistral': 2267, 'n_tokens_neox': 2189, 'n_words': 598}