<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stream

import (
	"fmt"
	"math"
	"reflect"

	"github.com/golang/protobuf/proto"
	"github.com/google/gapid/core/data/binary"
	"github.com/google/gapid/core/math/f16"
	"github.com/google/gapid/core/math/f32"
	"github.com/google/gapid/core/math/f64"
	"github.com/google/gapid/core/math/u64"
)

type buf struct {
	bytes     []byte
	component *Component
	offset    uint32 // in bits
	stride    uint32 // in bits
}

func (b buf) clone() buf {
	b.component = b.component.Clone()
	return b
}

type mapping struct {
	dst, src buf
}

// Convert converts count elements from data in format src to dst.
// Components are matched based on channel and semantic information.
// Components found in src that are not in dst are ignored.
// Certain components found in dst that are not in src are filled with default
// values (Y=0, Z=0, W=1, Alpha=1).
// Component order and datatypes can be changed.
func Convert(dst, src *Format, data []byte) ([]byte, error) {
	if dst == src || reflect.DeepEqual(dst, src) {
		return data, nil
	}

	dstStride, srcStride := dst.Stride(), src.Stride()
	count := len(data) / srcStride
	out := make([]byte, dst.Size(count))
	mappings := make([]mapping, len(dst.Components))
	srcOffsets := src.BitOffsets()
	dstOffset := uint32(0)

	// Fill in the mappings slice with direct component matches.
	for i, d := range dst.Components {
		m := &mappings[i]
		m.dst = buf{out, d, dstOffset, uint32(dstStride) * 8}

		s, err := src.Component(d.Channel)
		if err != nil {
			return nil, err
		}
		if s != nil {
			m.src = buf{data, s, srcOffsets[s], uint32(srcStride) * 8}
		}
		dstOffset += d.DataType.Bits()

		if err := m.convertCurve(count); err != nil {
			return nil, err
		}
	}

	if src.Channels().Contains(Channel_SharedExponent) && !dst.Channels().Contains(Channel_SharedExponent) {
		return convertSharedExponent(dst, src, data)
	}

	// Some components can be implicitly added (alpha, Y, Z, W).
	mappings = resolveImplicitMappings(count, mappings, src, data)

	// Do the conversion work.
	for _, m := range mappings {
		if m.src.component == nil {
			return nil, fmt.Errorf("Channel %v not found in source format: %v",
				m.dst.component.Channel, src)
		}
		if err := m.conv(count); err != nil {
			return nil, err
		}

	}
	return out, nil
}

func resolveImplicitMappings(count int, mappings []mapping, srcFmt *Format, srcData []byte) []mapping {
	for i := range mappings {
		m := &mappings[i]
		if m.src.component != nil {
			continue
		}
		srcChannels := srcFmt.Channels()
		switch m.dst.component.Channel {
		case Channel_Alpha:
			if srcChannels.ContainsColor() || srcChannels.ContainsDepth() {
				m.src = buf1Norm
			}
		case Channel_W:
			if srcChannels.ContainsVector() {
				m.src = buf1Norm
			}
		case Channel_Y, Channel_Z:
			if srcChannels.ContainsVector() {
				m.src = buf0
			}
		case Channel_Red, Channel_Green, Channel_Blue:
			if c, _ := srcFmt.Component(Channel_Gray, Channel_Luminance); c != nil {
				// Missing red, green or blue but have a gray or luminance channel. Use that.
				m.src = buf{srcData, c, srcFmt.BitOffsets()[c], uint32(srcFmt.Stride()) * 8}
			} else if c, _ := srcFmt.Component(Channel_Depth); c != nil {
				// Convert depth to RGB.
				m.src = buf{srcData, c, srcFmt.BitOffsets()[c], uint32(srcFmt.Stride()) * 8}
			} else if srcChannels.ContainsColor() {
				m.src = buf0
			}
		case Channel_Luminance:
			if c := srcFmt.GetSingleComponent(func(c *Component) bool { return c.Channel.IsColor() }); c != nil {
				// A format with a single color channel is equivalent to a luninance format.
				m.src = buf{srcData, c, srcFmt.BitOffsets()[c], uint32(srcFmt.Stride()) * 8}
			} else if c, _ := srcFmt.Component(Channel_Depth); c != nil {
				// Convert depth to luminance.
				m.src = buf{srcData, c, srcFmt.BitOffsets()[c], uint32(srcFmt.Stride()) * 8}
			}
			// TODO: RGB->Luminance conversion (#276)
		}
	}
	return mappings
}

var (
	buf0     = buf{[]byte{0}, &Component{DataType: &U1, Sampling: Linear}, 0, 0}
	buf1Norm = buf{[]byte{1}, &Component{DataType: &U1, Sampling: LinearNormalized}, 0, 0}
)<commit_msg>Correctly deal with GPU written depth_stencil textures.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package stream

import (
	"fmt"
	"math"
	"reflect"

	"github.com/golang/protobuf/proto"
	"github.com/google/gapid/core/data/binary"
	"github.com/google/gapid/core/math/f16"
	"github.com/google/gapid/core/math/f32"
	"github.com/google/gapid/core/math/f64"
	"github.com/google/gapid/core/math/u64"
)

type buf struct {
	bytes     []byte
	component *Component
	offset    uint32 // in bits
	stride    uint32 // in bits
}

func (b buf) clone() buf {
	b.component = b.component.Clone()
	return b
}

type mapping struct {
	dst, src buf
}

// Convert converts count elements from data in format src to dst.
// Components are matched based on channel and semantic information.
// Components found in src that are not in dst are ignored.
// Certain components found in dst that are not in src are filled with default
// values (Y=0, Z=0, W=1, Alpha=1).
// Component order and datatypes can be changed.
func Convert(dst, src *Format, data []byte) ([]byte, error) {
	if dst == src || reflect.DeepEqual(dst, src) {
		return data, nil
	}

	dstStride, srcStride := dst.Stride(), src.Stride()
	count := len(data) / srcStride
	out := make([]byte, dst.Size(count))
	mappings := make([]mapping, len(dst.Components))
	srcOffsets := src.BitOffsets()
	dstOffset := uint32(0)

	// Fill in the mappings slice with direct component matches.
	for i, d := range dst.Components {
		m := &mappings[i]
		m.dst = buf{out, d, dstOffset, uint32(dstStride) * 8}

		s, err := src.Component(d.Channel)
		if err != nil {
			return nil, err
		}
		if s != nil {
			m.src = buf{data, s, srcOffsets[s], uint32(srcStride) * 8}
		}
		dstOffset += d.DataType.Bits()

		if err := m.convertCurve(count); err != nil {
			return nil, err
		}
	}

	if src.Channels().Contains(Channel_SharedExponent) && !dst.Channels().Contains(Channel_SharedExponent) {
		return convertSharedExponent(dst, src, data)
	}

	// Some components can be implicitly added (alpha, Y, Z, W).
	mappings = resolveImplicitMappings(count, mappings, src, data)

	// Do the conversion work.
	for _, m := range mappings {
		if m.src.component == nil {
			return nil, fmt.Errorf("Channel %v not found in source format: %v",
				m.dst.component.Channel, src)
		}
		if err := m.conv(count); err != nil {
			return nil, err
		}

	}
	return out, nil
}

func resolveImplicitMappings(count int, mappings []mapping, srcFmt *Format, srcData []byte) []mapping {
	for i := range mappings {
		m := &mappings[i]
		if m.src.component != nil {
			continue
		}
		srcChannels := srcFmt.Channels()
		switch m.dst.component.Channel {
		case Channel_Alpha:
			if srcChannels.ContainsColor() || srcChannels.ContainsDepth() {
				m.src = buf1Norm
			}
		case Channel_W:
			if srcChannels.ContainsVector() {
				m.src = buf1Norm
			}
		case Channel_Y, Channel_Z:
			if srcChannels.ContainsVector() {
				m.src = buf0
			}
		case Channel_Red, Channel_Green, Channel_Blue:
			if c, _ := srcFmt.Component(Channel_Gray, Channel_Luminance); c != nil {
				// Missing red, green or blue but have a gray or luminance channel. Use that.
				m.src = buf{srcData, c, srcFmt.BitOffsets()[c], uint32(srcFmt.Stride()) * 8}
			} else if c, _ := srcFmt.Component(Channel_Depth); c != nil {
				// Convert depth to RGB.
				m.src = buf{srcData, c, srcFmt.BitOffsets()[c], uint32(srcFmt.Stride()) * 8}
			} else if srcChannels.ContainsColor() {
				m.src = buf0
			}
		case Channel_Luminance:
			if c := srcFmt.GetSingleComponent(func(c *Component) bool { return c.Channel.IsColor() }); c != nil {
				// A format with a single color channel is equivalent to a luninance format.
				m.src = buf{srcData, c, srcFmt.BitOffsets()[c], uint32(srcFmt.Stride()) * 8}
			} else if c, _ := srcFmt.Component(Channel_Depth); c != nil {
				// Convert depth to luminance.
				m.src = buf{srcData, c, srcFmt.BitOffsets()[c], uint32(srcFmt.Stride()) * 8}
			}
			// TODO: RGB->Luminance conversion (#276)
		case Channel_Stencil:
			// This is to work around our limitation of not being able to read stencil data.
			// If any conversion requests a stencil component, but the source doesn't have it,
			// just return a bunch-o-zeros.
			m.src = buf0
		}
	}
	return mappings
}

var (
	buf0     = buf{[]byte{0}, &Component{DataType: &U1, Sampling: Linear}, 0, 0}
	buf1Norm = buf{[]byte{1}, &Component{DataType: &U1, Sampling: LinearNormalized}, 0, 0}
)
{'fraction_non_alphanumeric': 0.13334022964725353, 'fraction_numerical': 0.01293058860039309, 'mean_word_length': 3.445057471264368, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '317090', 'n_tokens_mistral': 3588, 'n_tokens_neox': 3242, 'n_words': 1014}