<commit_before>#![cfg(feature = "postgresql")]

//! PostgreSQL backend,
//! needs citext, hstore and smlar.

extern crate postgres;
extern crate crypto;

use self::crypto::scrypt::{scrypt_simple,scrypt_check,ScryptParams};

use self::postgres::{Connection, TlsMode, Result as SQLResult};
use self::postgres::rows::Row;

pub struct Db(Connection);

use super::{Image,Tag,AnyWith,ImageBuilder,VoteImageError,parse_tag};
use super::super::CONFIG;

impl Default for Db { // Damn clippy
    fn default() -> Self {
        Self::new()
    }
}

lazy_static! {
    static ref SCRYPT_PARAMS: ScryptParams = ScryptParams::new(10, 8, 1); // 10 is really faster than 14
}

impl Db {
    pub fn new() -> Self {
        let conn = Connection::connect(format!("postgres://{}:{}@localhost", config!("postgres-login"), config!("postgres-password")), TlsMode::None).unwrap();
        conn.batch_execute("CREATE EXTENSION IF NOT EXISTS citext;
                            CREATE EXTENSION IF NOT EXISTS hstore;
                            CREATE EXTENSION IF NOT EXISTS smlar;

                            CREATE TABLE IF NOT EXISTS images(
                                id SERIAL PRIMARY KEY,
                                name TEXT NOT NULL UNIQUE,
                                tags TEXT[] NOT NULL,
                                uploader TEXT,
                                score INT NOT NULL DEFAULT 0,

                                got_from TEXT,
                                original_link TEXT,
                                rating CHAR
                            );

                            CREATE TABLE IF NOT EXISTS users(
                                id SERIAL PRIMARY KEY,
                                name CITEXT UNIQUE NOT NULL,
                                pass TEXT NOT NULL,
                                votes HSTORE
                            );").unwrap();
        Db(conn)
    }

    /// Add image, generate name from tags & id
    pub fn add_with_tags_name(&self, tags: &[String], ext: &str, uploader: &str) -> SQLResult<String> {
        let lastnum = self.0.query("SELECT id FROM images ORDER BY id DESC LIMIT 1", &[])?.get(0).get::<_, i32>("id");

        let name = format!("{}_{}.{}", lastnum + 1  , tags.join("_").replace("'","''"),ext);
        self.add_image(&ImageBuilder::new(&name, tags).uploader(uploader).finalize())?;
        Ok(name)
    }

    /// Add image
    pub fn add_image(&self, image: &ImageBuilder) -> SQLResult<()> {
        self.0.execute("INSERT into images (name,
                                            tags,
                                            got_from,
                                            original_link,
                                            rating,
                                            uploader,
                                            score)
                       VALUES ($1,
                               $2,
                               $3,
                               $4,
                               $5,
                               $6,
                               $7) ON CONFLICT (name) DO UPDATE SET tags = $2, score = $7",
        &[&image.name,&image.tags,&image.got_from, &image.original_link,&image.rating.map(|x| x.to_string()), &image.uploader, &image.score]).unwrap();
        Ok(())
    }

    pub fn get_image(&self, id: i32) -> SQLResult<Option<Image>> {
        let rows = self.0.query("SELECT * FROM images WHERE id = $1", &[&id])?;
        Ok(if !rows.is_empty() {
            Some(Db::extract_image(rows.get(0)))
        } else {
            None
        })
    }

    pub fn get_images<T: Into<Option<i32>>>(&self, take: T, skip: usize) -> SQLResult<Vec<Image>>{
        let take = match take.into() {
            Some(x) => x.to_string(),
            None    => "ALL".to_string()
        };

        Ok(self.0.query(&format!("SELECT * FROM images ORDER BY id DESC LIMIT {} OFFSET {}", take, skip as i32),&[])?
           .iter().fold(Vec::new(), |mut acc, row| {
               acc.push(Db::extract_image(row));
               acc
           }))
    }

    /// Search images by tags
    pub fn by_tags<T: Into<Option<i32>>>(&self, take: T, skip: usize, tags: &[String]) -> SQLResult<Vec<Image>> {
        let tags = tags.iter().map(|x| parse_tag(x)).collect::<Vec<_>>();
        let order = tags.iter().filter_map(|x| {
            match *x {
                Tag::OrderBy(ref by, ref ascdesc) => {
                    Some(format!("{:?}", by).to_lowercase() + " " + &format!("{:?}", ascdesc).to_uppercase())
                },
                _   => None
            }
        }).collect::<Vec<_>>();
        let order = match order.last() {
            Some(t) => t,
            None    => "id DESC"
        };

        let q = tags.clone().iter().map(|t| { // TODO: make it better?
            match *t {
                Tag::Include(ref incl) => format!(r"tags @> ARRAY['{}']", incl),
                Tag::Exclude(ref excl) => format!(r"NOT tags @> ARRAY['{}']", excl),
                Tag::AnyWith(ref x) => match *x {
                    AnyWith::Before(ref bef) => format!(r"(SELECT bool_or(tag ~ '^{}') FROM unnest(tags) t (tag))", bef),
                    AnyWith::After(ref aft) => format!(r"(SELECT bool_or(tag ~ '{}$') FROM unnest(tags) t (tag))", aft),
                },
                Tag::Rating(ref r) => {
                    let mut s = "(".to_string();
                    for tg in r {
                        s.push_str(&format!("rating = '{}' OR ", tg))
                    }
                    let _ = (0..4).inspect(|_| {s.pop(); }).collect::<Vec<_>>();
                    s.push_str(")");

                    s
                },

                Tag::From(ref f) => {
                    let mut s = "(".to_string();
                    for tg in f {
                        s.push_str(&format!("got_from = '{}' OR ", tg))
                    }
                    let _ = (0..4).inspect(|_| {s.pop(); }).collect::<Vec<_>>();
                    s.push_str(")");

                    s
                },

                Tag::Uploader(ref u) => {
                    let mut s = "(".to_string();
                    for tg in u {
                        s.push_str(&format!("uploader = '{}' OR ", tg))
                    }
                    let _ = (0..4).inspect(|_| {s.pop(); }).collect::<Vec<_>>();
                    s.push_str(")");

                    s
                },
                Tag::OrderBy(_,_) => {String::new()}
            }
        }).filter(|x| !x.is_empty()).collect::<Vec<_>>().join(" AND ");
        let q = if !q.is_empty() {
            format!("WHERE {}", q)
        } else {
            String::new()
        };

        let take = match take.into() {
            Some(x) => x.to_string(),
            None    => "ALL".to_string()
        };<commit_msg>Better query generator for tags<commit_after>#![cfg(feature = "postgresql")]

//! PostgreSQL backend,
//! needs citext, hstore and smlar.

extern crate postgres;
extern crate crypto;

use self::crypto::scrypt::{scrypt_simple,scrypt_check,ScryptParams};

use self::postgres::{Connection, TlsMode, Result as SQLResult};
use self::postgres::rows::Row;

pub struct Db(Connection);

use super::{Image,Tag,AnyWith,ImageBuilder,VoteImageError,parse_tag};
use super::super::CONFIG;

impl Default for Db { // Damn clippy
    fn default() -> Self {
        Self::new()
    }
}

lazy_static! {
    static ref SCRYPT_PARAMS: ScryptParams = ScryptParams::new(10, 8, 1); // 10 is really faster than 14
}

impl Db {
    pub fn new() -> Self {
        let conn = Connection::connect(format!("postgres://{}:{}@localhost", config!("postgres-login"), config!("postgres-password")), TlsMode::None).unwrap();
        conn.batch_execute("CREATE EXTENSION IF NOT EXISTS citext;
                            CREATE EXTENSION IF NOT EXISTS hstore;
                            CREATE EXTENSION IF NOT EXISTS smlar;

                            CREATE TABLE IF NOT EXISTS images(
                                id SERIAL PRIMARY KEY,
                                name TEXT NOT NULL UNIQUE,
                                tags TEXT[] NOT NULL,
                                uploader TEXT,
                                score INT NOT NULL DEFAULT 0,

                                got_from TEXT,
                                original_link TEXT,
                                rating CHAR
                            );

                            CREATE TABLE IF NOT EXISTS users(
                                id SERIAL PRIMARY KEY,
                                name CITEXT UNIQUE NOT NULL,
                                pass TEXT NOT NULL,
                                votes HSTORE
                            );").unwrap();
        Db(conn)
    }

    /// Add image, generate name from tags & id
    pub fn add_with_tags_name(&self, tags: &[String], ext: &str, uploader: &str) -> SQLResult<String> {
        let lastnum = self.0.query("SELECT id FROM images ORDER BY id DESC LIMIT 1", &[])?.get(0).get::<_, i32>("id");

        let name = format!("{}_{}.{}", lastnum + 1  , tags.join("_").replace("'","''"),ext);
        self.add_image(&ImageBuilder::new(&name, tags).uploader(uploader).finalize())?;
        Ok(name)
    }

    /// Add image
    pub fn add_image(&self, image: &ImageBuilder) -> SQLResult<()> {
        self.0.execute("INSERT into images (name,
                                            tags,
                                            got_from,
                                            original_link,
                                            rating,
                                            uploader,
                                            score)
                       VALUES ($1,
                               $2,
                               $3,
                               $4,
                               $5,
                               $6,
                               $7) ON CONFLICT (name) DO UPDATE SET tags = $2, score = $7",
        &[&image.name,&image.tags,&image.got_from, &image.original_link,&image.rating.map(|x| x.to_string()), &image.uploader, &image.score]).unwrap();
        Ok(())
    }

    pub fn get_image(&self, id: i32) -> SQLResult<Option<Image>> {
        let rows = self.0.query("SELECT * FROM images WHERE id = $1", &[&id])?;
        Ok(if !rows.is_empty() {
            Some(Db::extract_image(rows.get(0)))
        } else {
            None
        })
    }

    pub fn get_images<T: Into<Option<i32>>>(&self, take: T, skip: usize) -> SQLResult<Vec<Image>>{
        let take = match take.into() {
            Some(x) => x.to_string(),
            None    => "ALL".to_string()
        };

        Ok(self.0.query(&format!("SELECT * FROM images ORDER BY id DESC LIMIT {} OFFSET {}", take, skip as i32),&[])?
           .iter().fold(Vec::new(), |mut acc, row| {
               acc.push(Db::extract_image(row));
               acc
           }))
    }

    /// Search images by tags
    pub fn by_tags<T: Into<Option<i32>>>(&self, take: T, skip: usize, tags: &[String]) -> SQLResult<Vec<Image>> {
        let tags = tags.iter().map(|x| parse_tag(x)).collect::<Vec<_>>();
        let order = tags.iter().filter_map(|x| {
            match *x {
                Tag::OrderBy(ref by, ref ascdesc) => {
                    Some(format!("{:?}", by).to_lowercase() + " " + &format!("{:?}", ascdesc).to_uppercase())
                },
                _   => None
            }
        }).collect::<Vec<_>>();
        let order = match order.last() {
            Some(t) => t,
            None    => "id DESC"
        };

        /// Join tags for tags that can be separated with comma, e.g. rating or uploader
        fn join_tags(kind: &str, values: &[String]) -> String {
            values.iter().map(|s| format!("{} = '{}'", kind, s)).collect::<Vec<_>>().join(" OR ")
        }

        let q = tags.iter().map(|t| { // TODO: do something with OrderBy
            match *t {
                Tag::Include(ref incl)  => format!(r"tags @> ARRAY['{}']", incl),
                Tag::Exclude(ref excl)  => format!(r"NOT tags @> ARRAY['{}']", excl),
                Tag::AnyWith(ref x)     => match *x {
                    AnyWith::Before(ref bef) => format!(r"(SELECT bool_or(tag ~ '^{}') FROM unnest(tags) t (tag))", bef),
                    AnyWith::After(ref aft) => format!(r"(SELECT bool_or(tag ~ '{}$') FROM unnest(tags) t (tag))", aft),
                },
                Tag::Rating(ref r)      => join_tags("rating", r),
                Tag::From(ref f)        => join_tags("got_from", f),
                Tag::Uploader(ref u)    => join_tags("uploader", u),
                Tag::OrderBy(_,_)       => String::new() // <- This one
            }
        }).filter(|x| !x.is_empty()).collect::<Vec<_>>().join(" AND ");
        let q = if !q.is_empty() { format!("WHERE {}", q) } else { String::new() };

        let take = match take.into() {
            Some(x) => x.to_string(),
            None    => "ALL".to_string()
        };
{'fraction_non_alphanumeric': 0.15331650218039936, 'fraction_numerical': 0.006120419248718537, 'mean_word_length': 1.3767272727272728, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1259715', 'n_tokens_mistral': 3792, 'n_tokens_neox': 3637, 'n_words': 873}