<commit_before>#    EVE-Central.com Codebase
#    Copyright (C) 2006-2009 StackFoundry LLC and <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.


import random

def upload_suggest(db, region, rettype = "names"):

    cur = db.cursor()
    cur.execute("SELECT DISTINCT ON(types.typename) types.typename,types.typeid FROM types, current_market WHERE types.typeid = current_market.typeid AND current_market.regionid = %s AND current_market.reportedtime < (NOW() - interval '2 days') AND current_market.reportedtime > (NOW() - interval '12 days') GROUP BY types.typename,types.typeid,current_market.reportedtime LIMIT 200 OFFSET 0", [region])
    l = []

    r = cur.fetchone()
    while r:
        if rettype == "names":
            l.append(r[0])
        elif rettype == "ids":
            l.append(r[1])
        else:
            tup = (r[0], r[1])
            l.append(tup)

        r = cur.fetchone()
    random.shuffle(l)

    return l[0:20]<commit_msg>Improved SQL query for upload suggest. We want to get items we don't have yet, not only types we already have in the database that need a refreshment. This should lead to a wider selection of upload suggestions. The old one used to repeat after a few runs.<commit_after>#    EVE-Central.com Codebase
#    Copyright (C) 2006-2009 StackFoundry LLC and <NAME>
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.


import random

def upload_suggest(db, region, rettype = "names"):

    cur = db.cursor()
    cur.execute("SELECT types.typename,types.typeid FROM types WHERE types.published = 1 AND types.marketgroup > 0 AND types.typeid NOT IN (SELECT DISTINCT ON (current_market.typeid) current_market.typeid FROM current_market WHERE current_market.reportedtime < (NOW() - interval '1 days') AND current_market.regionid = %s ) ORDER BY RANDOM() LIMIT 20", [region])
    l = []

    r = cur.fetchone()
    while r:
        if rettype == "names":
            l.append(r[0])
        elif rettype == "ids":
            l.append(r[1])
        else:
            tup = (r[0], r[1])
            l.append(tup)

        r = cur.fetchone()

    return l
{'fraction_non_alphanumeric': 0.08891523414344991, 'fraction_numerical': 0.012151748666271488, 'mean_word_length': 3.0467625899280577, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3483309', 'n_tokens_mistral': 1001, 'n_tokens_neox': 944, 'n_words': 402}