431 lines
11 KiB
Go
431 lines
11 KiB
Go
|
// Copyright 2011 Google Inc. All rights reserved.
|
||
|
|
// Use of this source code is governed by the Apache 2.0
|
||
|
|
// license that can be found in the LICENSE file.
|
||
|
|
|
||
|
|
package datastore
|
||
|
|
|
||
|
|
import (
|
||
|
|
"fmt"
|
||
|
|
"reflect"
|
||
|
|
"strings"
|
||
|
|
"time"
|
||
|
|
|
||
|
|
"github.com/golang/protobuf/proto"
|
||
|
|
"google.golang.org/appengine"
|
||
|
|
pb "google.golang.org/appengine/internal/datastore"
|
||
|
|
)
|
||
|
|
|
||
|
|
var (
|
||
|
|
typeOfBlobKey = reflect.TypeOf(appengine.BlobKey(""))
|
||
|
|
typeOfByteSlice = reflect.TypeOf([]byte(nil))
|
||
|
|
typeOfByteString = reflect.TypeOf(ByteString(nil))
|
||
|
|
typeOfGeoPoint = reflect.TypeOf(appengine.GeoPoint{})
|
||
|
|
typeOfTime = reflect.TypeOf(time.Time{})
|
||
|
|
typeOfKeyPtr = reflect.TypeOf(&Key{})
|
||
|
|
typeOfEntityPtr = reflect.TypeOf(&Entity{})
|
||
|
|
)
|
||
|
|
|
||
|
|
// typeMismatchReason returns a string explaining why the property p could not
|
||
|
|
// be stored in an entity field of type v.Type().
|
||
|
|
func typeMismatchReason(pValue interface{}, v reflect.Value) string {
|
||
|
|
entityType := "empty"
|
||
|
|
switch pValue.(type) {
|
||
|
|
case int64:
|
||
|
|
entityType = "int"
|
||
|
|
case bool:
|
||
|
|
entityType = "bool"
|
||
|
|
case string:
|
||
|
|
entityType = "string"
|
||
|
|
case float64:
|
||
|
|
entityType = "float"
|
||
|
|
case *Key:
|
||
|
|
entityType = "*datastore.Key"
|
||
|
|
case time.Time:
|
||
|
|
entityType = "time.Time"
|
||
|
|
case appengine.BlobKey:
|
||
|
|
entityType = "appengine.BlobKey"
|
||
|
|
case appengine.GeoPoint:
|
||
|
|
entityType = "appengine.GeoPoint"
|
||
|
|
case ByteString:
|
||
|
|
entityType = "datastore.ByteString"
|
||
|
|
case []byte:
|
||
|
|
entityType = "[]byte"
|
||
|
|
}
|
||
|
|
return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type())
|
||
|
|
}
|
||
|
|
|
||
|
|
type propertyLoader struct {
|
||
|
|
// m holds the number of times a substruct field like "Foo.Bar.Baz" has
|
||
|
|
// been seen so far. The map is constructed lazily.
|
||
|
|
m map[string]int
|
||
|
|
}
|
||
|
|
|
||
|
|
func (l *propertyLoader) load(codec *structCodec, structValue reflect.Value, p Property, requireSlice bool) string {
|
||
|
|
var v reflect.Value
|
||
|
|
var sliceIndex int
|
||
|
|
|
||
|
|
name := p.Name
|
||
|
|
for name != "" {
|
||
|
|
// First we try to find a field with name matching
|
||
|
|
// the value of 'name' exactly.
|
||
|
|
decoder, ok := codec.fields[name]
|
||
|
|
if ok {
|
||
|
|
name = ""
|
||
|
|
} else {
|
||
|
|
// Now try for legacy flattened nested field (named eg. "A.B.C.D").
|
||
|
|
|
||
|
|
parent := name
|
||
|
|
child := ""
|
||
|
|
|
||
|
|
// Cut off the last field (delimited by ".") and find its parent
|
||
|
|
// in the codec.
|
||
|
|
// eg. for name "A.B.C.D", split off "A.B.C" and try to
|
||
|
|
// find a field in the codec with this name.
|
||
|
|
// Loop again with "A.B", etc.
|
||
|
|
for !ok {
|
||
|
|
i := strings.LastIndex(parent, ".")
|
||
|
|
if i < 0 {
|
||
|
|
return "no such struct field"
|
||
|
|
}
|
||
|
|
if i == len(name)-1 {
|
||
|
|
return "field name cannot end with '.'"
|
||
|
|
}
|
||
|
|
parent, child = name[:i], name[i+1:]
|
||
|
|
decoder, ok = codec.fields[parent]
|
||
|
|
}
|
||
|
|
|
||
|
|
name = child
|
||
|
|
}
|
||
|
|
|
||
|
|
v = initField(structValue, decoder.path)
|
||
|
|
if !v.IsValid() {
|
||
|
|
return "no such struct field"
|
||
|
|
}
|
||
|
|
if !v.CanSet() {
|
||
|
|
return "cannot set struct field"
|
||
|
|
}
|
||
|
|
|
||
|
|
if decoder.structCodec != nil {
|
||
|
|
codec = decoder.structCodec
|
||
|
|
structValue = v
|
||
|
|
}
|
||
|
|
|
||
|
|
if v.Kind() == reflect.Slice && v.Type() != typeOfByteSlice {
|
||
|
|
if l.m == nil {
|
||
|
|
l.m = make(map[string]int)
|
||
|
|
}
|
||
|
|
sliceIndex = l.m[p.Name]
|
||
|
|
l.m[p.Name] = sliceIndex + 1
|
||
|
|
for v.Len() <= sliceIndex {
|
||
|
|
v.Set(reflect.Append(v, reflect.New(v.Type().Elem()).Elem()))
|
||
|
|
}
|
||
|
|
structValue = v.Index(sliceIndex)
|
||
|
|
requireSlice = false
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
var slice reflect.Value
|
||
|
|
if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
|
||
|
|
slice = v
|
||
|
|
v = reflect.New(v.Type().Elem()).Elem()
|
||
|
|
} else if requireSlice {
|
||
|
|
return "multiple-valued property requires a slice field type"
|
||
|
|
}
|
||
|
|
|
||
|
|
// Convert indexValues to a Go value with a meaning derived from the
|
||
|
|
// destination type.
|
||
|
|
pValue := p.Value
|
||
|
|
if iv, ok := pValue.(indexValue); ok {
|
||
|
|
meaning := pb.Property_NO_MEANING
|
||
|
|
switch v.Type() {
|
||
|
|
case typeOfBlobKey:
|
||
|
|
meaning = pb.Property_BLOBKEY
|
||
|
|
case typeOfByteSlice:
|
||
|
|
meaning = pb.Property_BLOB
|
||
|
|
case typeOfByteString:
|
||
|
|
meaning = pb.Property_BYTESTRING
|
||
|
|
case typeOfGeoPoint:
|
||
|
|
meaning = pb.Property_GEORSS_POINT
|
||
|
|
case typeOfTime:
|
||
|
|
meaning = pb.Property_GD_WHEN
|
||
|
|
case typeOfEntityPtr:
|
||
|
|
meaning = pb.Property_ENTITY_PROTO
|
||
|
|
}
|
||
|
|
var err error
|
||
|
|
pValue, err = propValue(iv.value, meaning)
|
||
|
|
if err != nil {
|
||
|
|
return err.Error()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if errReason := setVal(v, pValue); errReason != "" {
|
||
|
|
// Set the slice back to its zero value.
|
||
|
|
if slice.IsValid() {
|
||
|
|
slice.Set(reflect.Zero(slice.Type()))
|
||
|
|
}
|
||
|
|
return errReason
|
||
|
|
}
|
||
|
|
|
||
|
|
if slice.IsValid() {
|
||
|
|
slice.Index(sliceIndex).Set(v)
|
||
|
|
}
|
||
|
|
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
|
||
|
|
// setVal sets v to the value pValue.
|
||
|
|
func setVal(v reflect.Value, pValue interface{}) string {
|
||
|
|
switch v.Kind() {
|
||
|
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||
|
|
x, ok := pValue.(int64)
|
||
|
|
if !ok && pValue != nil {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
if v.OverflowInt(x) {
|
||
|
|
return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
|
||
|
|
}
|
||
|
|
v.SetInt(x)
|
||
|
|
case reflect.Bool:
|
||
|
|
x, ok := pValue.(bool)
|
||
|
|
if !ok && pValue != nil {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
v.SetBool(x)
|
||
|
|
case reflect.String:
|
||
|
|
switch x := pValue.(type) {
|
||
|
|
case appengine.BlobKey:
|
||
|
|
v.SetString(string(x))
|
||
|
|
case ByteString:
|
||
|
|
v.SetString(string(x))
|
||
|
|
case string:
|
||
|
|
v.SetString(x)
|
||
|
|
default:
|
||
|
|
if pValue != nil {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
}
|
||
|
|
case reflect.Float32, reflect.Float64:
|
||
|
|
x, ok := pValue.(float64)
|
||
|
|
if !ok && pValue != nil {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
if v.OverflowFloat(x) {
|
||
|
|
return fmt.Sprintf("value %v overflows struct field of type %v", x, v.Type())
|
||
|
|
}
|
||
|
|
v.SetFloat(x)
|
||
|
|
case reflect.Ptr:
|
||
|
|
x, ok := pValue.(*Key)
|
||
|
|
if !ok && pValue != nil {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
if _, ok := v.Interface().(*Key); !ok {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
v.Set(reflect.ValueOf(x))
|
||
|
|
case reflect.Struct:
|
||
|
|
switch v.Type() {
|
||
|
|
case typeOfTime:
|
||
|
|
x, ok := pValue.(time.Time)
|
||
|
|
if !ok && pValue != nil {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
v.Set(reflect.ValueOf(x))
|
||
|
|
case typeOfGeoPoint:
|
||
|
|
x, ok := pValue.(appengine.GeoPoint)
|
||
|
|
if !ok && pValue != nil {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
v.Set(reflect.ValueOf(x))
|
||
|
|
default:
|
||
|
|
ent, ok := pValue.(*Entity)
|
||
|
|
if !ok {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
|
||
|
|
// Recursively load nested struct
|
||
|
|
pls, err := newStructPLS(v.Addr().Interface())
|
||
|
|
if err != nil {
|
||
|
|
return err.Error()
|
||
|
|
}
|
||
|
|
|
||
|
|
// if ent has a Key value and our struct has a Key field,
|
||
|
|
// load the Entity's Key value into the Key field on the struct.
|
||
|
|
if ent.Key != nil && pls.codec.keyField != -1 {
|
||
|
|
|
||
|
|
pls.v.Field(pls.codec.keyField).Set(reflect.ValueOf(ent.Key))
|
||
|
|
}
|
||
|
|
|
||
|
|
err = pls.Load(ent.Properties)
|
||
|
|
if err != nil {
|
||
|
|
return err.Error()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
case reflect.Slice:
|
||
|
|
x, ok := pValue.([]byte)
|
||
|
|
if !ok {
|
||
|
|
if y, yok := pValue.(ByteString); yok {
|
||
|
|
x, ok = []byte(y), true
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if !ok && pValue != nil {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
if v.Type().Elem().Kind() != reflect.Uint8 {
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
v.SetBytes(x)
|
||
|
|
default:
|
||
|
|
return typeMismatchReason(pValue, v)
|
||
|
|
}
|
||
|
|
return ""
|
||
|
|
}
|
||
|
|
|
||
|
|
// initField is similar to reflect's Value.FieldByIndex, in that it
|
||
|
|
// returns the nested struct field corresponding to index, but it
|
||
|
|
// initialises any nil pointers encountered when traversing the structure.
|
||
|
|
func initField(val reflect.Value, index []int) reflect.Value {
|
||
|
|
for _, i := range index[:len(index)-1] {
|
||
|
|
val = val.Field(i)
|
||
|
|
if val.Kind() == reflect.Ptr {
|
||
|
|
if val.IsNil() {
|
||
|
|
val.Set(reflect.New(val.Type().Elem()))
|
||
|
|
}
|
||
|
|
val = val.Elem()
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return val.Field(index[len(index)-1])
|
||
|
|
}
|
||
|
|
|
||
|
|
// loadEntity loads an EntityProto into PropertyLoadSaver or struct pointer.
|
||
|
|
func loadEntity(dst interface{}, src *pb.EntityProto) (err error) {
|
||
|
|
ent, err := protoToEntity(src)
|
||
|
|
if err != nil {
|
||
|
|
return err
|
||
|
|
}
|
||
|
|
if e, ok := dst.(PropertyLoadSaver); ok {
|
||
|
|
return e.Load(ent.Properties)
|
||
|
|
}
|
||
|
|
return LoadStruct(dst, ent.Properties)
|
||
|
|
}
|
||
|
|
|
||
|
|
func (s structPLS) Load(props []Property) error {
|
||
|
|
var fieldName, reason string
|
||
|
|
var l propertyLoader
|
||
|
|
for _, p := range props {
|
||
|
|
if errStr := l.load(s.codec, s.v, p, p.Multiple); errStr != "" {
|
||
|
|
// We don't return early, as we try to load as many properties as possible.
|
||
|
|
// It is valid to load an entity into a struct that cannot fully represent it.
|
||
|
|
// That case returns an error, but the caller is free to ignore it.
|
||
|
|
fieldName, reason = p.Name, errStr
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if reason != "" {
|
||
|
|
return &ErrFieldMismatch{
|
||
|
|
StructType: s.v.Type(),
|
||
|
|
FieldName: fieldName,
|
||
|
|
Reason: reason,
|
||
|
|
}
|
||
|
|
}
|
||
|
|
return nil
|
||
|
|
}
|
||
|
|
|
||
|
|
func protoToEntity(src *pb.EntityProto) (*Entity, error) {
|
||
|
|
props, rawProps := src.Property, src.RawProperty
|
||
|
|
outProps := make([]Property, 0, len(props)+len(rawProps))
|
||
|
|
for {
|
||
|
|
var (
|
||
|
|
x *pb.Property
|
||
|
|
noIndex bool
|
||
|
|
)
|
||
|
|
if len(props) > 0 {
|
||
|
|
x, props = props[0], props[1:]
|
||
|
|
} else if len(rawProps) > 0 {
|
||
|
|
x, rawProps = rawProps[0], rawProps[1:]
|
||
|
|
noIndex = true
|
||
|
|
} else {
|
||
|
|
break
|
||
|
|
}
|
||
|
|
|
||
|
|
var value interface{}
|
||
|
|
if x.Meaning != nil && *x.Meaning == pb.Property_INDEX_VALUE {
|
||
|
|
value = indexValue{x.Value}
|
||
|
|
} else {
|
||
|
|
var err error
|
||
|
|
value, err = propValue(x.Value, x.GetMeaning())
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
}
|
||
|
|
outProps = append(outProps, Property{
|
||
|
|
Name: x.GetName(),
|
||
|
|
Value: value,
|
||
|
|
NoIndex: noIndex,
|
||
|
|
Multiple: x.GetMultiple(),
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
var key *Key
|
||
|
|
if src.Key != nil {
|
||
|
|
// Ignore any error, since nested entity values
|
||
|
|
// are allowed to have an invalid key.
|
||
|
|
key, _ = protoToKey(src.Key)
|
||
|
|
}
|
||
|
|
return &Entity{key, outProps}, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// propValue returns a Go value that combines the raw PropertyValue with a
|
||
|
|
// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time.
|
||
|
|
func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) {
|
||
|
|
switch {
|
||
|
|
case v.Int64Value != nil:
|
||
|
|
if m == pb.Property_GD_WHEN {
|
||
|
|
return fromUnixMicro(*v.Int64Value), nil
|
||
|
|
} else {
|
||
|
|
return *v.Int64Value, nil
|
||
|
|
}
|
||
|
|
case v.BooleanValue != nil:
|
||
|
|
return *v.BooleanValue, nil
|
||
|
|
case v.StringValue != nil:
|
||
|
|
if m == pb.Property_BLOB {
|
||
|
|
return []byte(*v.StringValue), nil
|
||
|
|
} else if m == pb.Property_BLOBKEY {
|
||
|
|
return appengine.BlobKey(*v.StringValue), nil
|
||
|
|
} else if m == pb.Property_BYTESTRING {
|
||
|
|
return ByteString(*v.StringValue), nil
|
||
|
|
} else if m == pb.Property_ENTITY_PROTO {
|
||
|
|
var ent pb.EntityProto
|
||
|
|
err := proto.Unmarshal([]byte(*v.StringValue), &ent)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
return protoToEntity(&ent)
|
||
|
|
} else {
|
||
|
|
return *v.StringValue, nil
|
||
|
|
}
|
||
|
|
case v.DoubleValue != nil:
|
||
|
|
return *v.DoubleValue, nil
|
||
|
|
case v.Referencevalue != nil:
|
||
|
|
key, err := referenceValueToKey(v.Referencevalue)
|
||
|
|
if err != nil {
|
||
|
|
return nil, err
|
||
|
|
}
|
||
|
|
return key, nil
|
||
|
|
case v.Pointvalue != nil:
|
||
|
|
// NOTE: Strangely, latitude maps to X, longitude to Y.
|
||
|
|
return appengine.GeoPoint{Lat: v.Pointvalue.GetX(), Lng: v.Pointvalue.GetY()}, nil
|
||
|
|
}
|
||
|
|
return nil, nil
|
||
|
|
}
|
||
|
|
|
||
|
|
// indexValue is a Property value that is created when entities are loaded from
|
||
|
|
// an index, such as from a projection query.
|
||
|
|
//
|
||
|
|
// Such Property values do not contain all of the metadata required to be
|
||
|
|
// faithfully represented as a Go value, and are instead represented as an
|
||
|
|
// opaque indexValue. Load the properties into a concrete struct type (e.g. by
|
||
|
|
// passing a struct pointer to Iterator.Next) to reconstruct actual Go values
|
||
|
|
// of type int, string, time.Time, etc.
|
||
|
|
type indexValue struct {
|
||
|
|
value *pb.PropertyValue
|
||
|
|
}
|