<commit_before># -*- coding: utf-8 -*-
"""Various utilities."""

import json
import logging
import os
import random
import string
import time
from collections import defaultdict
from datetime import date, datetime, timedelta
from itertools import filterfalse, groupby
from urllib.parse import quote, urlencode, urlparse

import chardet
import emails
import flask
import requests
import yaml
from flask import request, url_for
from flask_login import current_user
from html2text import html2text
from itsdangerous import (BadSignature, SignatureExpired, TimedJSONWebSignatureSerializer)
from jinja2 import Template
from peewee import JOIN, SQL
from yaml.dumper import Dumper
from yaml.representer import SafeRepresenter

from . import app, orcid_client, rq
from .models import (AFFILIATION_TYPES, Affiliation, AffiliationRecord, FundingInvitees,
                     FundingRecord, Log, OrcidToken, Organisation, OrgInvitation, PartialDate,
                     PeerReviewExternalId, PeerReviewInvitee, PeerReviewRecord, Role, Task,
                     TaskType, User, UserInvitation, UserOrg, WorkInvitees, WorkRecord, get_val)

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

EDU_CODES = {"student", "edu"}
EMP_CODES = {"faculty", "staff", "emp"}

ENV = app.config.get("ENV")
EXTERNAL_SP = app.config.get("EXTERNAL_SP")


def get_next_url():
    """Retrieve and sanitize next/return URL."""
    _next = request.args.get("next") or request.args.get("_next") or request.args.get("url")

    if _next and ("orcidhub.org.nz" in _next or _next.startswith("/") or "127.0" in _next
                  or "c9users.io" in _next):
        return _next
    return None


def is_valid_url(url):
    """Validate URL (expexted to have a path)."""
    try:
        result = urlparse(url)
        return result.scheme and result.netloc and (result.path or result.path == '')
    except:
        return False<commit_msg>fixed callback url for 'localhost'<commit_after># -*- coding: utf-8 -*-
"""Various utilities."""

import json
import logging
import os
import random
import string
import time
from collections import defaultdict
from datetime import date, datetime, timedelta
from itertools import filterfalse, groupby
from urllib.parse import quote, urlencode, urlparse

import chardet
import emails
import flask
import requests
import yaml
from flask import request, url_for
from flask_login import current_user
from html2text import html2text
from itsdangerous import (BadSignature, SignatureExpired, TimedJSONWebSignatureSerializer)
from jinja2 import Template
from peewee import JOIN, SQL
from yaml.dumper import Dumper
from yaml.representer import SafeRepresenter

from . import app, orcid_client, rq
from .models import (AFFILIATION_TYPES, Affiliation, AffiliationRecord, FundingInvitees,
                     FundingRecord, Log, OrcidToken, Organisation, OrgInvitation, PartialDate,
                     PeerReviewExternalId, PeerReviewInvitee, PeerReviewRecord, Role, Task,
                     TaskType, User, UserInvitation, UserOrg, WorkInvitees, WorkRecord, get_val)

logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler())

EDU_CODES = {"student", "edu"}
EMP_CODES = {"faculty", "staff", "emp"}

ENV = app.config.get("ENV")
EXTERNAL_SP = app.config.get("EXTERNAL_SP")


def get_next_url():
    """Retrieve and sanitize next/return URL."""
    _next = request.args.get("next") or request.args.get("_next") or request.args.get("url")

    if _next and ("orcidhub.org.nz" in _next or _next.startswith("/") or "127.0" in _next
                  or "localhost" in _next or "c9users.io" in _next):
        return _next
    return None


def is_valid_url(url):
    """Validate URL (expexted to have a path)."""
    try:
        result = urlparse(url)
        return result.scheme and result.netloc and (result.path or result.path == '')
    except:
        return False
{'fraction_non_alphanumeric': 0.09298818798693138, 'fraction_numerical': 0.004523749685850716, 'mean_word_length': 4.08301404853129, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '612315', 'n_tokens_mistral': 1210, 'n_tokens_neox': 1165, 'n_words': 377}