<commit_before>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package ra

import (
	"crypto/x509"
	"errors"
	"fmt"
	"net"
	"net/mail"
	"reflect"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/net/publicsuffix"
	"github.com/letsencrypt/boulder/probs"

	"github.com/letsencrypt/boulder/bdns"
	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
)

// DefaultAuthorizationLifetime is the 10 month default authorization lifetime.
// When used with a 90-day cert lifetime, this allows creation of certs that will
// cover a whole year, plus a grace period of a month.
// TODO(jsha): Read from a config file.
const DefaultAuthorizationLifetime = 300 * 24 * time.Hour

// DefaultPendingAuthorizationLifetime is one week.  If you can't respond to a
// challenge this quickly, then you need to request a new challenge.
// TODO(rlb): Read from a config file
const DefaultPendingAuthorizationLifetime = 7 * 24 * time.Hour

// RegistrationAuthorityImpl defines an RA.
//
// NOTE: All of the fields in RegistrationAuthorityImpl need to be
// populated, or there is a risk of panic.
type RegistrationAuthorityImpl struct {
	CA          core.CertificateAuthority
	VA          core.ValidationAuthority
	SA          core.StorageAuthority
	PA          core.PolicyAuthority
	stats       statsd.Statter
	DNSResolver bdns.DNSResolver
	clk         clock.Clock
	log         *blog.AuditLogger
	dc          *DomainCheck
	// How long before a newly created authorization expires.
	authorizationLifetime        time.Duration
	pendingAuthorizationLifetime time.Duration
	rlPolicies                   cmd.RateLimitConfig
	tiMu                         *sync.RWMutex
	totalIssuedCache             int
	lastIssuedCount              *time.Time
	maxContactsPerReg            int
}

// NewRegistrationAuthorityImpl constructs a new RA object.
func NewRegistrationAuthorityImpl(clk clock.Clock, logger *blog.AuditLogger, stats statsd.Statter, dc *DomainCheck, policies cmd.RateLimitConfig, maxContactsPerReg int) *RegistrationAuthorityImpl {
	ra := &RegistrationAuthorityImpl{
		stats: stats,
		clk:   clk,
		log:   logger,
		dc:    dc,
		authorizationLifetime:        DefaultAuthorizationLifetime,
		pendingAuthorizationLifetime: DefaultPendingAuthorizationLifetime,
		rlPolicies:                   policies,
		tiMu:                         new(sync.RWMutex),
		maxContactsPerReg:            maxContactsPerReg,
	}
	return ra
}

const (
	unparseableEmailDetail = "not a valid e-mail address"
	emptyDNSResponseDetail = "empty DNS response"
)

func validateEmail(address string, resolver bdns.DNSResolver) (prob *probs.ProblemDetails) {
	_, err := mail.ParseAddress(address)
	if err != nil {
		return &probs.ProblemDetails{
			Type:   probs.InvalidEmailProblem,
			Detail: unparseableEmailDetail,
		}
	}
	splitEmail := strings.SplitN(address, "@", -1)
	domain := strings.ToLower(splitEmail[len(splitEmail)-1])
	var resultMX []string
	var resultA []net.IP
	resultMX, err = resolver.LookupMX(domain)

	if err == nil && len(resultMX) == 0 {
		resultA, err = resolver.LookupHost(domain)
		if err == nil && len(resultA) == 0 {
			return &probs.ProblemDetails{
				Type:   probs.InvalidEmailProblem,
				Detail: emptyDNSResponseDetail,
			}
		}
	}
	if err != nil {
		dnsProblem := bdns.ProblemDetailsFromDNSError(err)
		return &probs.ProblemDetails{
			Type:   probs.InvalidEmailProblem,
			Detail: dnsProblem.Detail,
		}
	}

	return nil
}

type certificateRequestEvent struct {
	ID                  string    `json:",omitempty"`
	Requester           int64     `json:",omitempty"`
	SerialNumber        string    `json:",omitempty"`
	RequestMethod       string    `json:",omitempty"`
	VerificationMethods []string  `json:",omitempty"`
	VerifiedFields      []string  `json:",omitempty"`
	CommonName          string    `json:",omitempty"`
	Names               []string  `json:",omitempty"`
	NotBefore           time.Time `json:",omitempty"`
	NotAfter            time.Time `json:",omitempty"`
	RequestTime         time.Time `json:",omitempty"`
	ResponseTime        time.Time `json:",omitempty"`
	Error               string    `json:",omitempty"`
}

var issuanceCountCacheLife = 1 * time.Minute

// issuanceCountInvalid checks if the current issuance count is invalid either
// because it hasn't been set yet or because it has expired. This method expects
// that the caller holds either a R or W ra.tiMu lock.
func (ra *RegistrationAuthorityImpl) issuanceCountInvalid(now time.Time) bool {
	return ra.lastIssuedCount == nil || ra.lastIssuedCount.Add(issuanceCountCacheLife).Before(now)
}

func (ra *RegistrationAuthorityImpl) getIssuanceCount() (int, error) {
	ra.tiMu.RLock()
	if ra.issuanceCountInvalid(ra.clk.Now()) {
		ra.tiMu.RUnlock()
		return ra.setIssuanceCount()
	}
	count := ra.totalIssuedCache
	ra.tiMu.RUnlock()
	return count, nil
}

func (ra *RegistrationAuthorityImpl) setIssuanceCount() (int, error) {
	ra.tiMu.Lock()
	defer ra.tiMu.Unlock()

	now := ra.clk.Now()
	if ra.issuanceCountInvalid(now) {
		count, err := ra.SA.CountCertificatesRange(
			now.Add(-ra.rlPolicies.TotalCertificates.Window.Duration),
			now,
		)
		if err != nil {
			return 0, err
		}
		ra.totalIssuedCache = int(count)
		ra.lastIssuedCount = &now
	}
	return ra.totalIssuedCache, nil
}

// noRegistrationID is used for the regID parameter to GetThreshold when no
// registration-based overrides are necessary.
const noRegistrationID = -1

func (ra *RegistrationAuthorityImpl) checkRegistrationLimit(ip net.IP) error {
	limit := ra.rlPolicies.RegistrationsPerIP
	if limit.Enabled() {
		now := ra.clk.Now()
		count, err := ra.SA.CountRegistrationsByIP(ip, limit.WindowBegin(now), now)
		if err != nil {
			return err
		}
		if count >= limit.GetThreshold(ip.String(), noRegistrationID) {
			return core.RateLimitedError("Too many registrations from this IP")
		}
	}
	return nil
}

// NewRegistration constructs a new Registration from a request.
func (ra *RegistrationAuthorityImpl) NewRegistration(init core.Registration) (reg core.Registration, err error) {
	if err = core.GoodKey(init.Key.Key); err != nil {
		return core.Registration{}, core.MalformedRequestError(fmt.Sprintf("Invalid public key: %s", err.Error()))
	}
	if err = ra.checkRegistrationLimit(init.InitialIP); err != nil {
		return core.Registration{}, err
	}

	reg = core.Registration{
		Key: init.Key,
	}
	reg.MergeUpdate(init)

	// This field isn't updatable by the end user, so it isn't copied by
	// MergeUpdate. But we need to fill it in for new registrations.
	reg.InitialIP = init.InitialIP

	err = ra.validateContacts(reg.Contact)
	if err != nil {
		return
	}

	// Store the authorization object, then return it
	reg, err = ra.SA.NewRegistration(reg)
	if err != nil {
		// InternalServerError since the user-data was validated before being
		// passed to the SA.
		err = core.InternalServerError(err.Error())
	}

	ra.stats.Inc("RA.NewRegistrations", 1, 1.0)
	return
}

func (ra *RegistrationAuthorityImpl) validateContacts(contacts []*core.AcmeURL) (err error) {
	if ra.maxContactsPerReg > 0 && len(contacts) > ra.maxContactsPerReg {
		return core.MalformedRequestError(fmt.Sprintf("Too many contacts provided: %d > %d",
			len(contacts), ra.maxContactsPerReg))
	}

	for _, contact := range contacts {
		if contact == nil {
			return core.MalformedRequestError("Invalid contact")
		}
		switch contact.Scheme {
		case "tel":
			continue
		case "mailto":
			start := ra.clk.Now()
			ra.stats.Inc("RA.ValidateEmail.Calls", 1, 1.0)
			problem := validateEmail(contact.Opaque, ra.DNSResolver)
			ra.stats.TimingDuration("RA.ValidateEmail.Latency", ra.clk.Now().Sub(start), 1.0)
			if problem != nil {
				ra.stats.Inc("RA.ValidateEmail.Errors", 1, 1.0)
				return problem
			}
			ra.stats.Inc("RA.ValidateEmail.Successes", 1, 1.0)
		default:
			err = core.MalformedRequestError(fmt.Sprintf("Contact method %s is not supported", contact.Scheme))
			return
		}
	}

	return
}

func checkPendingAuthorizationLimit(sa core.StorageGetter, limit *cmd.RateLimitPolicy, regID int64) error {
	if limit.Enabled() {
		count, err := sa.CountPendingAuthorizations(regID)
		if err != nil {
			return err
		}
		// Most rate limits have a key for overrides, but there is no meaningful key
		// here.
		noKey := ""
		if count > limit.GetThreshold(noKey, regID) {
			return core.RateLimitedError("Too many currently pending authorizations.")
		}
	}
	return nil
}

// NewAuthorization constuct a new Authz from a request. Values (domains) in
// request.Identifier will be lowercased before storage.
func (ra *RegistrationAuthorityImpl) NewAuthorization(request core.Authorization, regID int64) (authz core.Authorization, err error) {
	reg, err := ra.SA.GetRegistration(regID)
	if err != nil {
		err = core.MalformedRequestError(fmt.Sprintf("Invalid registration ID: %d", regID))
		return authz, err
	}

	identifier := request.Identifier
	identifier.Value = strings.ToLower(identifier.Value)

	// Check that the identifier is present and appropriate<commit_msg>fix pending authz rate limit check<commit_after>// Copyright 2014 ISRG.  All rights reserved
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package ra

import (
	"crypto/x509"
	"errors"
	"fmt"
	"net"
	"net/mail"
	"reflect"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/cactus/go-statsd-client/statsd"
	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/jmhodges/clock"
	"github.com/letsencrypt/boulder/Godeps/_workspace/src/github.com/letsencrypt/net/publicsuffix"
	"github.com/letsencrypt/boulder/probs"

	"github.com/letsencrypt/boulder/bdns"
	"github.com/letsencrypt/boulder/cmd"
	"github.com/letsencrypt/boulder/core"
	blog "github.com/letsencrypt/boulder/log"
)

// DefaultAuthorizationLifetime is the 10 month default authorization lifetime.
// When used with a 90-day cert lifetime, this allows creation of certs that will
// cover a whole year, plus a grace period of a month.
// TODO(jsha): Read from a config file.
const DefaultAuthorizationLifetime = 300 * 24 * time.Hour

// DefaultPendingAuthorizationLifetime is one week.  If you can't respond to a
// challenge this quickly, then you need to request a new challenge.
// TODO(rlb): Read from a config file
const DefaultPendingAuthorizationLifetime = 7 * 24 * time.Hour

// RegistrationAuthorityImpl defines an RA.
//
// NOTE: All of the fields in RegistrationAuthorityImpl need to be
// populated, or there is a risk of panic.
type RegistrationAuthorityImpl struct {
	CA          core.CertificateAuthority
	VA          core.ValidationAuthority
	SA          core.StorageAuthority
	PA          core.PolicyAuthority
	stats       statsd.Statter
	DNSResolver bdns.DNSResolver
	clk         clock.Clock
	log         *blog.AuditLogger
	dc          *DomainCheck
	// How long before a newly created authorization expires.
	authorizationLifetime        time.Duration
	pendingAuthorizationLifetime time.Duration
	rlPolicies                   cmd.RateLimitConfig
	tiMu                         *sync.RWMutex
	totalIssuedCache             int
	lastIssuedCount              *time.Time
	maxContactsPerReg            int
}

// NewRegistrationAuthorityImpl constructs a new RA object.
func NewRegistrationAuthorityImpl(clk clock.Clock, logger *blog.AuditLogger, stats statsd.Statter, dc *DomainCheck, policies cmd.RateLimitConfig, maxContactsPerReg int) *RegistrationAuthorityImpl {
	ra := &RegistrationAuthorityImpl{
		stats: stats,
		clk:   clk,
		log:   logger,
		dc:    dc,
		authorizationLifetime:        DefaultAuthorizationLifetime,
		pendingAuthorizationLifetime: DefaultPendingAuthorizationLifetime,
		rlPolicies:                   policies,
		tiMu:                         new(sync.RWMutex),
		maxContactsPerReg:            maxContactsPerReg,
	}
	return ra
}

const (
	unparseableEmailDetail = "not a valid e-mail address"
	emptyDNSResponseDetail = "empty DNS response"
)

func validateEmail(address string, resolver bdns.DNSResolver) (prob *probs.ProblemDetails) {
	_, err := mail.ParseAddress(address)
	if err != nil {
		return &probs.ProblemDetails{
			Type:   probs.InvalidEmailProblem,
			Detail: unparseableEmailDetail,
		}
	}
	splitEmail := strings.SplitN(address, "@", -1)
	domain := strings.ToLower(splitEmail[len(splitEmail)-1])
	var resultMX []string
	var resultA []net.IP
	resultMX, err = resolver.LookupMX(domain)

	if err == nil && len(resultMX) == 0 {
		resultA, err = resolver.LookupHost(domain)
		if err == nil && len(resultA) == 0 {
			return &probs.ProblemDetails{
				Type:   probs.InvalidEmailProblem,
				Detail: emptyDNSResponseDetail,
			}
		}
	}
	if err != nil {
		dnsProblem := bdns.ProblemDetailsFromDNSError(err)
		return &probs.ProblemDetails{
			Type:   probs.InvalidEmailProblem,
			Detail: dnsProblem.Detail,
		}
	}

	return nil
}

type certificateRequestEvent struct {
	ID                  string    `json:",omitempty"`
	Requester           int64     `json:",omitempty"`
	SerialNumber        string    `json:",omitempty"`
	RequestMethod       string    `json:",omitempty"`
	VerificationMethods []string  `json:",omitempty"`
	VerifiedFields      []string  `json:",omitempty"`
	CommonName          string    `json:",omitempty"`
	Names               []string  `json:",omitempty"`
	NotBefore           time.Time `json:",omitempty"`
	NotAfter            time.Time `json:",omitempty"`
	RequestTime         time.Time `json:",omitempty"`
	ResponseTime        time.Time `json:",omitempty"`
	Error               string    `json:",omitempty"`
}

var issuanceCountCacheLife = 1 * time.Minute

// issuanceCountInvalid checks if the current issuance count is invalid either
// because it hasn't been set yet or because it has expired. This method expects
// that the caller holds either a R or W ra.tiMu lock.
func (ra *RegistrationAuthorityImpl) issuanceCountInvalid(now time.Time) bool {
	return ra.lastIssuedCount == nil || ra.lastIssuedCount.Add(issuanceCountCacheLife).Before(now)
}

func (ra *RegistrationAuthorityImpl) getIssuanceCount() (int, error) {
	ra.tiMu.RLock()
	if ra.issuanceCountInvalid(ra.clk.Now()) {
		ra.tiMu.RUnlock()
		return ra.setIssuanceCount()
	}
	count := ra.totalIssuedCache
	ra.tiMu.RUnlock()
	return count, nil
}

func (ra *RegistrationAuthorityImpl) setIssuanceCount() (int, error) {
	ra.tiMu.Lock()
	defer ra.tiMu.Unlock()

	now := ra.clk.Now()
	if ra.issuanceCountInvalid(now) {
		count, err := ra.SA.CountCertificatesRange(
			now.Add(-ra.rlPolicies.TotalCertificates.Window.Duration),
			now,
		)
		if err != nil {
			return 0, err
		}
		ra.totalIssuedCache = int(count)
		ra.lastIssuedCount = &now
	}
	return ra.totalIssuedCache, nil
}

// noRegistrationID is used for the regID parameter to GetThreshold when no
// registration-based overrides are necessary.
const noRegistrationID = -1

func (ra *RegistrationAuthorityImpl) checkRegistrationLimit(ip net.IP) error {
	limit := ra.rlPolicies.RegistrationsPerIP
	if limit.Enabled() {
		now := ra.clk.Now()
		count, err := ra.SA.CountRegistrationsByIP(ip, limit.WindowBegin(now), now)
		if err != nil {
			return err
		}
		if count >= limit.GetThreshold(ip.String(), noRegistrationID) {
			return core.RateLimitedError("Too many registrations from this IP")
		}
	}
	return nil
}

// NewRegistration constructs a new Registration from a request.
func (ra *RegistrationAuthorityImpl) NewRegistration(init core.Registration) (reg core.Registration, err error) {
	if err = core.GoodKey(init.Key.Key); err != nil {
		return core.Registration{}, core.MalformedRequestError(fmt.Sprintf("Invalid public key: %s", err.Error()))
	}
	if err = ra.checkRegistrationLimit(init.InitialIP); err != nil {
		return core.Registration{}, err
	}

	reg = core.Registration{
		Key: init.Key,
	}
	reg.MergeUpdate(init)

	// This field isn't updatable by the end user, so it isn't copied by
	// MergeUpdate. But we need to fill it in for new registrations.
	reg.InitialIP = init.InitialIP

	err = ra.validateContacts(reg.Contact)
	if err != nil {
		return
	}

	// Store the authorization object, then return it
	reg, err = ra.SA.NewRegistration(reg)
	if err != nil {
		// InternalServerError since the user-data was validated before being
		// passed to the SA.
		err = core.InternalServerError(err.Error())
	}

	ra.stats.Inc("RA.NewRegistrations", 1, 1.0)
	return
}

func (ra *RegistrationAuthorityImpl) validateContacts(contacts []*core.AcmeURL) (err error) {
	if ra.maxContactsPerReg > 0 && len(contacts) > ra.maxContactsPerReg {
		return core.MalformedRequestError(fmt.Sprintf("Too many contacts provided: %d > %d",
			len(contacts), ra.maxContactsPerReg))
	}

	for _, contact := range contacts {
		if contact == nil {
			return core.MalformedRequestError("Invalid contact")
		}
		switch contact.Scheme {
		case "tel":
			continue
		case "mailto":
			start := ra.clk.Now()
			ra.stats.Inc("RA.ValidateEmail.Calls", 1, 1.0)
			problem := validateEmail(contact.Opaque, ra.DNSResolver)
			ra.stats.TimingDuration("RA.ValidateEmail.Latency", ra.clk.Now().Sub(start), 1.0)
			if problem != nil {
				ra.stats.Inc("RA.ValidateEmail.Errors", 1, 1.0)
				return problem
			}
			ra.stats.Inc("RA.ValidateEmail.Successes", 1, 1.0)
		default:
			err = core.MalformedRequestError(fmt.Sprintf("Contact method %s is not supported", contact.Scheme))
			return
		}
	}

	return
}

func checkPendingAuthorizationLimit(sa core.StorageGetter, limit *cmd.RateLimitPolicy, regID int64) error {
	if limit.Enabled() {
		count, err := sa.CountPendingAuthorizations(regID)
		if err != nil {
			return err
		}
		// Most rate limits have a key for overrides, but there is no meaningful key
		// here.
		noKey := ""
		if count >= limit.GetThreshold(noKey, regID) {
			return core.RateLimitedError("Too many currently pending authorizations.")
		}
	}
	return nil
}

// NewAuthorization constuct a new Authz from a request. Values (domains) in
// request.Identifier will be lowercased before storage.
func (ra *RegistrationAuthorityImpl) NewAuthorization(request core.Authorization, regID int64) (authz core.Authorization, err error) {
	reg, err := ra.SA.GetRegistration(regID)
	if err != nil {
		err = core.MalformedRequestError(fmt.Sprintf("Invalid registration ID: %d", regID))
		return authz, err
	}

	identifier := request.Identifier
	identifier.Value = strings.ToLower(identifier.Value)

	// Check that the identifier is present and appropriate
{'fraction_non_alphanumeric': 0.10719940884619444, 'fraction_numerical': 0.0053837221577113905, 'mean_word_length': 3.6736556487419834, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6585425', 'n_tokens_mistral': 6311, 'n_tokens_neox': 5892, 'n_words': 1586}