<commit_before>// Copyright 2020 Google Inc. All Rights Reserved.

package models

import (
	"errors"
	"fmt"
	"regexp"
	"time"

	rpc "apigov.dev/flame/rpc"
	ptypes "github.com/golang/protobuf/ptypes"
)

// File ...
type File struct {
	ProjectID   string    // Uniquely identifies a project.
	ProductID   string    // Uniquely identifies a product within a project.
	VersionID   string    // Uniquely identifies a version within a product.
	SpecID      string    // Uniquely identifies a spec within a version.
	FileID      string    // Uniquely identifies a file within a spec.
	DisplayName string    // A human-friendly name.
	Description string    // A detailed description.
	CreateTime  time.Time // Creation time.
	UpdateTime  time.Time // Time of last change.
	FileName    string    // Name of file.
	SizeInBytes int32     // Size of the file.
	Hash        string    // A hash of the file.
	SourceURI   string    // The original source URI of the file.
	Contents    []byte    `datastore:",noindex"` // The contents of the file.
}

// ParseParentSpec ...
func ParseParentSpec(parent string) ([]string, error) {
	r := regexp.MustCompile("^projects/" + nameRegex +
		"/products/" + nameRegex +
		"/versions/" + nameRegex +
		"/specs/" + nameRegex +
		"$")
	m := r.FindAllStringSubmatch(parent, -1)
	if m == nil {
		return nil, fmt.Errorf("invalid spec '%s'", parent)
	}
	return m[0], nil
}

// NewFileFromParentAndFileID returns an initialized file for a specified parent and fileID.
func NewFileFromParentAndFileID(parent string, fileID string) (*File, error) {
	r := regexp.MustCompile("^projects/" + nameRegex +
		"/products/" + nameRegex +
		"/versions/" + nameRegex +
		"/specs/" + nameRegex + "$")
	m := r.FindAllStringSubmatch(parent, -1)
	if m == nil {
		return nil, fmt.Errorf("invalid parent '%s'", parent)
	}
	if err := validateID(fileID); err != nil {
		return nil, err
	}
	file := &File{}
	file.ProjectID = m[0][1]
	file.ProductID = m[0][2]
	file.VersionID = m[0][3]
	file.SpecID = m[0][4]
	file.FileID = fileID
	return file, nil
}

// NewFileFromResourceName parses resource names and returns an initialized file.
func NewFileFromResourceName(name string) (*File, error) {
	file := &File{}
	r := regexp.MustCompile("^projects/" + nameRegex + "/products/" + nameRegex + "/versions/" + nameRegex + "/specs/" + nameRegex + "/files/" + nameRegex + "$")
	m := r.FindAllStringSubmatch(name, -1)
	if m == nil {
		return nil, errors.New("invalid file name")
	}
	file.ProjectID = m[0][1]
	file.ProductID = m[0][2]
	file.VersionID = m[0][3]
	file.SpecID = m[0][4]
	file.FileID = m[0][5]
	return file, nil
}

// NewFileFromMessage returns an initialized file from a message.
func NewFileFromMessage(message *rpc.File) (*File, error) {
	file, err := NewFileFromResourceName(message.GetName())
	if err != nil {
		return nil, err
	}
	file.DisplayName = message.GetDisplayName()
	file.Description = message.GetDescription()
	//file.Availability = message.GetAvailability()
	//file.RecommendedVersion = message.GetRecommendedVersion()
	return file, nil
}

// ResourceName generates the resource name of a file.
func (file *File) ResourceName() string {
	return fmt.Sprintf("projects/%s/products/%s/versions/%s/specs/%s/files/%s",
		file.ProjectID, file.ProductID, file.VersionID, file.SpecID, file.FileID)
}

// Message returns a message representing a file.
func (file *File) Message(view rpc.FileView) (message *rpc.File, err error) {
	message = &rpc.File{}
	message.Name = file.ResourceName()
	message.DisplayName = file.DisplayName
	message.Description = file.Description
	message.CreateTime, err = ptypes.TimestampProto(file.CreateTime)
	message.UpdateTime, err = ptypes.TimestampProto(file.UpdateTime)
	if view == rpc.FileView_FULL {
		message.Contents = file.Contents
	}
	return message, err
}

// Update modifies a file using the contents of a message.
func (file *File) Update(message *rpc.File) error {
	file.Contents = message.GetContents()
	file.UpdateTime = file.CreateTime
	return nil
}<commit_msg>Compute hash and size for uploaded files.<commit_after>// Copyright 2020 Google Inc. All Rights Reserved.

package models

import (
	"crypto/sha1"
	"errors"
	"fmt"
	"regexp"
	"time"

	rpc "apigov.dev/flame/rpc"
	ptypes "github.com/golang/protobuf/ptypes"
)

// File ...
type File struct {
	ProjectID   string    // Uniquely identifies a project.
	ProductID   string    // Uniquely identifies a product within a project.
	VersionID   string    // Uniquely identifies a version within a product.
	SpecID      string    // Uniquely identifies a spec within a version.
	FileID      string    // Uniquely identifies a file within a spec.
	DisplayName string    // A human-friendly name.
	Description string    // A detailed description.
	CreateTime  time.Time // Creation time.
	UpdateTime  time.Time // Time of last change.
	FileName    string    // Name of file.
	SizeInBytes int32     // Size of the file.
	Hash        string    // A hash of the file.
	SourceURI   string    // The original source URI of the file.
	Contents    []byte    `datastore:",noindex"` // The contents of the file.
}

// ParseParentSpec ...
func ParseParentSpec(parent string) ([]string, error) {
	r := regexp.MustCompile("^projects/" + nameRegex +
		"/products/" + nameRegex +
		"/versions/" + nameRegex +
		"/specs/" + nameRegex +
		"$")
	m := r.FindAllStringSubmatch(parent, -1)
	if m == nil {
		return nil, fmt.Errorf("invalid spec '%s'", parent)
	}
	return m[0], nil
}

// NewFileFromParentAndFileID returns an initialized file for a specified parent and fileID.
func NewFileFromParentAndFileID(parent string, fileID string) (*File, error) {
	r := regexp.MustCompile("^projects/" + nameRegex +
		"/products/" + nameRegex +
		"/versions/" + nameRegex +
		"/specs/" + nameRegex + "$")
	m := r.FindAllStringSubmatch(parent, -1)
	if m == nil {
		return nil, fmt.Errorf("invalid parent '%s'", parent)
	}
	if err := validateID(fileID); err != nil {
		return nil, err
	}
	file := &File{}
	file.ProjectID = m[0][1]
	file.ProductID = m[0][2]
	file.VersionID = m[0][3]
	file.SpecID = m[0][4]
	file.FileID = fileID
	return file, nil
}

// NewFileFromResourceName parses resource names and returns an initialized file.
func NewFileFromResourceName(name string) (*File, error) {
	file := &File{}
	r := regexp.MustCompile("^projects/" + nameRegex + "/products/" + nameRegex + "/versions/" + nameRegex + "/specs/" + nameRegex + "/files/" + nameRegex + "$")
	m := r.FindAllStringSubmatch(name, -1)
	if m == nil {
		return nil, errors.New("invalid file name")
	}
	file.ProjectID = m[0][1]
	file.ProductID = m[0][2]
	file.VersionID = m[0][3]
	file.SpecID = m[0][4]
	file.FileID = m[0][5]
	return file, nil
}

// NewFileFromMessage returns an initialized file from a message.
func NewFileFromMessage(message *rpc.File) (*File, error) {
	file, err := NewFileFromResourceName(message.GetName())
	if err != nil {
		return nil, err
	}
	file.DisplayName = message.GetDisplayName()
	file.Description = message.GetDescription()
	return file, nil
}

// ResourceName generates the resource name of a file.
func (file *File) ResourceName() string {
	return fmt.Sprintf("projects/%s/products/%s/versions/%s/specs/%s/files/%s",
		file.ProjectID, file.ProductID, file.VersionID, file.SpecID, file.FileID)
}

// Message returns a message representing a file.
func (file *File) Message(view rpc.FileView) (message *rpc.File, err error) {
	message = &rpc.File{}
	message.Name = file.ResourceName()
	message.DisplayName = file.DisplayName
	message.Description = file.Description
	message.SizeBytes = file.SizeInBytes
	message.Hash = file.Hash
	message.CreateTime, err = ptypes.TimestampProto(file.CreateTime)
	message.UpdateTime, err = ptypes.TimestampProto(file.UpdateTime)
	if view == rpc.FileView_FULL {
		message.Contents = file.Contents
	}
	return message, err
}

// Update modifies a file using the contents of a message.
func (file *File) Update(message *rpc.File) error {
	contents := message.GetContents()
	file.Contents = contents
	file.SizeInBytes = int32(len(contents))
	h := sha1.New()
	h.Write(contents)
	bs := h.Sum(nil)
	file.Hash = fmt.Sprintf("%x", bs)
	file.UpdateTime = file.CreateTime
	return nil
}
{'fraction_non_alphanumeric': 0.1333659610913985, 'fraction_numerical': 0.007341245564664138, 'mean_word_length': 3.670857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '294900', 'n_tokens_mistral': 2726, 'n_tokens_neox': 2650, 'n_words': 771}