<commit_before>/*
 * Copyright 2013-2019 Erudika. https://erudika.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * For issues and patches go to: https://github.com/erudika
 */
package com.erudika.scoold.utils;

import com.erudika.para.Para;
import com.erudika.para.client.ParaClient;
import com.erudika.para.core.App;
import com.erudika.para.core.ParaObject;
import com.erudika.para.core.Sysprop;
import com.erudika.para.core.User;
import com.erudika.para.core.utils.ParaObjectUtils;
import com.erudika.para.email.Emailer;
import com.erudika.para.utils.Config;
import com.erudika.para.utils.Pager;
import com.erudika.para.utils.Utils;
import com.erudika.para.validation.ValidationUtils;
import static com.erudika.scoold.ScooldServer.*;
import com.erudika.scoold.core.Comment;
import com.erudika.scoold.core.Feedback;
import com.erudika.scoold.core.Post;
import static com.erudika.scoold.core.Post.ALL_MY_SPACES;
import static com.erudika.scoold.core.Post.DEFAULT_SPACE;
import com.erudika.scoold.core.Profile;
import static com.erudika.scoold.core.Profile.Badge.ENTHUSIAST;
import static com.erudika.scoold.core.Profile.Badge.TEACHER;
import com.erudika.scoold.core.Question;
import com.erudika.scoold.core.Reply;
import com.erudika.scoold.core.Report;
import com.erudika.scoold.core.Revision;
import com.erudika.scoold.core.UnapprovedQuestion;
import com.erudika.scoold.core.UnapprovedReply;
import static com.erudika.scoold.utils.HttpUtils.getCookieValue;
import com.typesafe.config.ConfigObject;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Scanner;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Named;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.validation.ConstraintViolation;
import javax.ws.rs.WebApplicationException;
import org.apache.commons.lang3.RegExUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Component;

/**
 *
 * @author <NAME> [<EMAIL>]
 */
@Component
@Named
public final class ScooldUtils {

	private static final Logger logger = LoggerFactory.getLogger(ScooldUtils.class);
	private static final Map<String, String> EMAIL_TEMPLATES = new ConcurrentHashMap<String, String>();
	private static final Set<String> APPROVED_DOMAINS = new HashSet<>();
	private static final Set<String> ADMINS = new HashSet<>();
	private static final String EMAIL_ALERTS_PREFIX = "email-alerts" + Config.SEPARATOR;

	private static Set<String> coreTypes;
	static {
		coreTypes = new HashSet<>(Arrays.asList(Utils.type(Comment.class),
				Utils.type(Feedback.class),
				Utils.type(Profile.class),
				Utils.type(Question.class),
				Utils.type(Reply.class),
				Utils.type(Report.class),
				Utils.type(Revision.class),
				Utils.type(UnapprovedQuestion.class),
				Utils.type(UnapprovedReply.class)));
	}

	private ParaClient pc;
	private LanguageUtils langutils;
	private static ScooldUtils instance;
	@Inject private Emailer emailer;

	@Inject
	public ScooldUtils(ParaClient pc, LanguageUtils langutils) {
		this.pc = pc;
		this.langutils = langutils;
	}

	public ParaClient getParaClient() {
		return pc;
	}

	public LanguageUtils getLangutils() {
		return langutils;
	}

	public static ScooldUtils getInstance() {
		return instance;
	}

	static void setInstance(ScooldUtils instance) {
		ScooldUtils.instance = instance;
	}

	static {
		// multiple domains/admins are now allowed only in Scoold PRO
		String approvedDomains = Config.getConfigParam("approved_domains_for_signups", "");
		if (!StringUtils.isBlank(approvedDomains)) {
			APPROVED_DOMAINS.add(approvedDomains);
		}
		// multiple admins are now allowed only in Scoold PRO
		String admins = Config.getConfigParam("admins", "");
		if (!StringUtils.isBlank(admins)) {
			ADMINS.add(admins);
		}
	}

	public static void tryConnectToPara(Callable<Boolean> callable) {
		retryConnection(callable, 0);
	}

	private static void retryConnection(Callable<Boolean> callable, int retryCount) {
		try {
			if (!callable.call()) {
				throw new Exception();
			}
		} catch (Exception e) {
			int maxRetries = Config.getConfigInt("connection_retries_max", 10);
			int retryInterval = Config.getConfigInt("connection_retry_interval_sec", 10);
			int count = ++retryCount;
			logger.error("No connection to Para backend. Retrying connection in {}s (attempt {} of {})...",
					retryInterval, count, maxRetries);
			if (maxRetries < 0 || retryCount < maxRetries) {
				Para.asyncExecute(new Runnable() {
					public void run() {
						try {
							Thread.sleep(retryInterval * 1000L);
						} catch (InterruptedException ex) {
							logger.error(null, ex);
							Thread.currentThread().interrupt();
						}
						retryConnection(callable, count);
					}
				});
			}
		}
	}

	public ParaObject checkAuth(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
		Profile authUser = null;
		boolean isApiRequest = isApiRequest(req);
		if (isApiRequest) {
			return checkApiAuth(req);
		} else if (HttpUtils.getStateParam(AUTH_COOKIE, req) != null &&
				!StringUtils.endsWithAny(req.getRequestURI(), ".js", ".css", ".svg", ".png", ".jpg")) {
			User u = pc.me(HttpUtils.getStateParam(AUTH_COOKIE, req));
			if (u != null && isEmailDomainApproved(u.getEmail())) {
				authUser = getOrCreateProfile(u, req);
				authUser.setUser(u);
				if (promoteOrDemoteUser(authUser, u) || updateProfilePictureAndName(authUser, u)) {
					authUser.update();
				}
			} else {
				clearSession(req, res);
				if (u != null) {
					logger.warn("Attempted signin from an unknown domain: {}", u.getEmail());
				} else {
					logger.info("Invalid JWT found in cookie {}.", AUTH_COOKIE);
				}
				res.sendRedirect(SIGNINLINK + "?code=3&error=true");
				return null;
			}
		}
		return authUser;
	}

	private ParaObject checkApiAuth(HttpServletRequest req) {
		if (req.getRequestURI().equals(CONTEXT_PATH + "/api")) {
			return null;
		}
		String superToken = StringUtils.removeStart(req.getHeader(HttpHeaders.AUTHORIZATION), "Bearer ");
		if (StringUtils.isBlank(superToken)) {
			throw new WebApplicationException(401);
		}
		ParaObject app = pc.me(superToken);
		if (app == null || !(app instanceof App)) {
			throw new WebApplicationException(401);
		}
		return app;
	}

	private boolean promoteOrDemoteUser(Profile authUser, User u) {
		if (authUser != null) {
			if (!isAdmin(authUser) && isRecognizedAsAdmin(u)) {
				logger.info("User '{}' with id={} promoted to admin.", u.getName(), authUser.getId());
				authUser.setGroups(User.Groups.ADMINS.toString());
				return true;
			} else if (isAdmin(authUser) && !isRecognizedAsAdmin(u)) {
				logger.info("User '{}' with id={} demoted to regular user.", u.getName(), authUser.getId());
				authUser.setGroups(User.Groups.USERS.toString());
				return true;
			}
		}
		return false;
	}

	private Profile getOrCreateProfile(User u, HttpServletRequest req) {
		Profile authUser = pc.read(Profile.id(u.getId()));
		if (authUser == null) {
			authUser = new Profile(u.getId(), u.getName());
			authUser.setOriginalName(u.getName());
			authUser.setPicture(u.getPicture());
			authUser.setAppid(u.getAppid());
			authUser.setCreatorid(u.getId());
			authUser.setTimestamp(u.getTimestamp());
			authUser.setGroups(isRecognizedAsAdmin(u)
					? User.Groups.ADMINS.toString() : u.getGroups());
			// auto-assign spaces to new users
			String space = Config.getConfigParam("auto_assign_spaces", "");
			if (!StringUtils.isBlank(space) && !isDefaultSpace(space)) {
				Sysprop s = pc.read(getSpaceId(space));
				if (s != null) {
					authUser.getSpaces().add(s.getId() + Config.SEPARATOR + s.getName());
				}
			}<commit_msg>fixed a possible NPE when assigning spaces to user<commit_after>/*
 * Copyright 2013-2019 Erudika. https://erudika.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * For issues and patches go to: https://github.com/erudika
 */
package com.erudika.scoold.utils;

import com.erudika.para.Para;
import com.erudika.para.client.ParaClient;
import com.erudika.para.core.App;
import com.erudika.para.core.ParaObject;
import com.erudika.para.core.Sysprop;
import com.erudika.para.core.User;
import com.erudika.para.core.utils.ParaObjectUtils;
import com.erudika.para.email.Emailer;
import com.erudika.para.utils.Config;
import com.erudika.para.utils.Pager;
import com.erudika.para.utils.Utils;
import com.erudika.para.validation.ValidationUtils;
import static com.erudika.scoold.ScooldServer.*;
import com.erudika.scoold.core.Comment;
import com.erudika.scoold.core.Feedback;
import com.erudika.scoold.core.Post;
import static com.erudika.scoold.core.Post.ALL_MY_SPACES;
import static com.erudika.scoold.core.Post.DEFAULT_SPACE;
import com.erudika.scoold.core.Profile;
import static com.erudika.scoold.core.Profile.Badge.ENTHUSIAST;
import static com.erudika.scoold.core.Profile.Badge.TEACHER;
import com.erudika.scoold.core.Question;
import com.erudika.scoold.core.Reply;
import com.erudika.scoold.core.Report;
import com.erudika.scoold.core.Revision;
import com.erudika.scoold.core.UnapprovedQuestion;
import com.erudika.scoold.core.UnapprovedReply;
import static com.erudika.scoold.utils.HttpUtils.getCookieValue;
import com.typesafe.config.ConfigObject;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Scanner;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import javax.inject.Inject;
import javax.inject.Named;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.validation.ConstraintViolation;
import javax.ws.rs.WebApplicationException;
import org.apache.commons.lang3.RegExUtils;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.HttpHeaders;
import org.springframework.stereotype.Component;

/**
 *
 * @author <NAME> [<EMAIL>]
 */
@Component
@Named
public final class ScooldUtils {

	private static final Logger logger = LoggerFactory.getLogger(ScooldUtils.class);
	private static final Map<String, String> EMAIL_TEMPLATES = new ConcurrentHashMap<String, String>();
	private static final Set<String> APPROVED_DOMAINS = new HashSet<>();
	private static final Set<String> ADMINS = new HashSet<>();
	private static final String EMAIL_ALERTS_PREFIX = "email-alerts" + Config.SEPARATOR;

	private static Set<String> coreTypes;
	static {
		coreTypes = new HashSet<>(Arrays.asList(Utils.type(Comment.class),
				Utils.type(Feedback.class),
				Utils.type(Profile.class),
				Utils.type(Question.class),
				Utils.type(Reply.class),
				Utils.type(Report.class),
				Utils.type(Revision.class),
				Utils.type(UnapprovedQuestion.class),
				Utils.type(UnapprovedReply.class)));
	}

	private ParaClient pc;
	private LanguageUtils langutils;
	private static ScooldUtils instance;
	@Inject private Emailer emailer;

	@Inject
	public ScooldUtils(ParaClient pc, LanguageUtils langutils) {
		this.pc = pc;
		this.langutils = langutils;
	}

	public ParaClient getParaClient() {
		return pc;
	}

	public LanguageUtils getLangutils() {
		return langutils;
	}

	public static ScooldUtils getInstance() {
		return instance;
	}

	static void setInstance(ScooldUtils instance) {
		ScooldUtils.instance = instance;
	}

	static {
		// multiple domains/admins are now allowed only in Scoold PRO
		String approvedDomains = Config.getConfigParam("approved_domains_for_signups", "");
		if (!StringUtils.isBlank(approvedDomains)) {
			APPROVED_DOMAINS.add(approvedDomains);
		}
		// multiple admins are now allowed only in Scoold PRO
		String admins = Config.getConfigParam("admins", "");
		if (!StringUtils.isBlank(admins)) {
			ADMINS.add(admins);
		}
	}

	public static void tryConnectToPara(Callable<Boolean> callable) {
		retryConnection(callable, 0);
	}

	private static void retryConnection(Callable<Boolean> callable, int retryCount) {
		try {
			if (!callable.call()) {
				throw new Exception();
			}
		} catch (Exception e) {
			int maxRetries = Config.getConfigInt("connection_retries_max", 10);
			int retryInterval = Config.getConfigInt("connection_retry_interval_sec", 10);
			int count = ++retryCount;
			logger.error("No connection to Para backend. Retrying connection in {}s (attempt {} of {})...",
					retryInterval, count, maxRetries);
			if (maxRetries < 0 || retryCount < maxRetries) {
				Para.asyncExecute(new Runnable() {
					public void run() {
						try {
							Thread.sleep(retryInterval * 1000L);
						} catch (InterruptedException ex) {
							logger.error(null, ex);
							Thread.currentThread().interrupt();
						}
						retryConnection(callable, count);
					}
				});
			}
		}
	}

	public ParaObject checkAuth(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
		Profile authUser = null;
		boolean isApiRequest = isApiRequest(req);
		if (isApiRequest) {
			return checkApiAuth(req);
		} else if (HttpUtils.getStateParam(AUTH_COOKIE, req) != null &&
				!StringUtils.endsWithAny(req.getRequestURI(), ".js", ".css", ".svg", ".png", ".jpg")) {
			User u = pc.me(HttpUtils.getStateParam(AUTH_COOKIE, req));
			if (u != null && isEmailDomainApproved(u.getEmail())) {
				authUser = getOrCreateProfile(u, req);
				authUser.setUser(u);
				if (promoteOrDemoteUser(authUser, u) || updateProfilePictureAndName(authUser, u)) {
					authUser.update();
				}
			} else {
				clearSession(req, res);
				if (u != null) {
					logger.warn("Attempted signin from an unknown domain: {}", u.getEmail());
				} else {
					logger.info("Invalid JWT found in cookie {}.", AUTH_COOKIE);
				}
				res.sendRedirect(SIGNINLINK + "?code=3&error=true");
				return null;
			}
		}
		return authUser;
	}

	private ParaObject checkApiAuth(HttpServletRequest req) {
		if (req.getRequestURI().equals(CONTEXT_PATH + "/api")) {
			return null;
		}
		String superToken = StringUtils.removeStart(req.getHeader(HttpHeaders.AUTHORIZATION), "Bearer ");
		if (StringUtils.isBlank(superToken)) {
			throw new WebApplicationException(401);
		}
		ParaObject app = pc.me(superToken);
		if (app == null || !(app instanceof App)) {
			throw new WebApplicationException(401);
		}
		return app;
	}

	private boolean promoteOrDemoteUser(Profile authUser, User u) {
		if (authUser != null) {
			if (!isAdmin(authUser) && isRecognizedAsAdmin(u)) {
				logger.info("User '{}' with id={} promoted to admin.", u.getName(), authUser.getId());
				authUser.setGroups(User.Groups.ADMINS.toString());
				return true;
			} else if (isAdmin(authUser) && !isRecognizedAsAdmin(u)) {
				logger.info("User '{}' with id={} demoted to regular user.", u.getName(), authUser.getId());
				authUser.setGroups(User.Groups.USERS.toString());
				return true;
			}
		}
		return false;
	}

	private Profile getOrCreateProfile(User u, HttpServletRequest req) {
		Profile authUser = pc.read(Profile.id(u.getId()));
		if (authUser == null) {
			authUser = new Profile(u.getId(), u.getName());
			authUser.setUser(u);
			authUser.setOriginalName(u.getName());
			authUser.setPicture(u.getPicture());
			authUser.setAppid(u.getAppid());
			authUser.setCreatorid(u.getId());
			authUser.setTimestamp(u.getTimestamp());
			authUser.setGroups(isRecognizedAsAdmin(u)
					? User.Groups.ADMINS.toString() : u.getGroups());
			// auto-assign spaces to new users
			String space = Config.getConfigParam("auto_assign_spaces", "");
			if (!StringUtils.isBlank(space) && !isDefaultSpace(space)) {
				Sysprop s = pc.read(getSpaceId(space));
				if (s != null) {
					authUser.getSpaces().add(s.getId() + Config.SEPARATOR + s.getName());
				}
			}
{'fraction_non_alphanumeric': 0.1249503489757703, 'fraction_numerical': 0.003858593882993815, 'mean_word_length': 4.538654934003771, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 27, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1367999', 'n_tokens_mistral': 6303, 'n_tokens_neox': 5696, 'n_words': 1241}