fast-path.go.tmpl (17091B)
1 // +build !notfastpath 2 // +build !codec.notfastpath 3 4 // Copyright (c) 2012-2020 Ugorji Nwoke. All rights reserved. 5 // Use of this source code is governed by a MIT license found in the LICENSE file. 6 7 // Code generated from fast-path.go.tmpl - DO NOT EDIT. 8 9 package codec 10 11 // Fast path functions try to create a fast path encode or decode implementation 12 // for common maps and slices. 13 // 14 // We define the functions and register them in this single file 15 // so as not to pollute the encode.go and decode.go, and create a dependency in there. 16 // This file can be omitted without causing a build failure. 17 // 18 // The advantage of fast paths is: 19 // - Many calls bypass reflection altogether 20 // 21 // Currently support 22 // - slice of all builtin types (numeric, bool, string, []byte) 23 // - maps of builtin types to builtin or interface{} type, EXCEPT FOR 24 // keys of type uintptr, int8/16/32, uint16/32, float32/64, bool, interface{} 25 // AND values of type type int8/16/32, uint16/32 26 // This should provide adequate "typical" implementations. 27 // 28 // Note that fast track decode functions must handle values for which an address cannot be obtained. 29 // For example: 30 // m2 := map[string]int{} 31 // p2 := []interface{}{m2} 32 // // decoding into p2 will bomb if fast track functions do not treat like unaddressable. 33 // 34 35 {{/* 36 fastpathEncMapStringUint64R (called by fastpath...switch) 37 EncMapStringUint64V (called by codecgen) 38 39 fastpathEncSliceBoolR: (called by fastpath...switch) (checks f.ti.mbs and calls one of them below) 40 EncSliceBoolV (also called by codecgen) 41 EncAsMapSliceBoolV (delegate when mapbyslice=true) 42 43 fastpathDecSliceIntfR (called by fastpath...switch) (calls Y or N below depending on if it can be updated) 44 DecSliceIntfX (called by codecgen) (calls Y below) 45 DecSliceIntfY (delegate when slice CAN be updated) 46 DecSliceIntfN (delegate when slice CANNOT be updated e.g. from array or non-addressable slice) 47 48 fastpathDecMap...R (called by fastpath...switch) (calls L or X? below) 49 DecMap...X (called by codecgen) 50 DecMap...L (delegated to by both above) 51 */ -}} 52 53 import ( 54 "reflect" 55 "sort" 56 ) 57 58 const fastpathEnabled = true 59 60 {{/* 61 const fastpathMapBySliceErrMsg = "mapBySlice requires even slice length, but got %v" 62 */ -}} 63 64 type fastpathT struct {} 65 66 var fastpathTV fastpathT 67 68 type fastpathE struct { 69 {{/* rtid uintptr */ -}} 70 rt reflect.Type 71 encfn func(*Encoder, *codecFnInfo, reflect.Value) 72 decfn func(*Decoder, *codecFnInfo, reflect.Value) 73 } 74 75 type fastpathA [{{ .FastpathLen }}]fastpathE 76 type fastpathARtid [{{ .FastpathLen }}]uintptr 77 78 var fastpathAv fastpathA 79 var fastpathAvRtid fastpathARtid 80 81 type fastpathAslice struct{} 82 83 func (fastpathAslice) Len() int { return {{ .FastpathLen }} } 84 func (fastpathAslice) Less(i, j int) bool { 85 return fastpathAvRtid[uint(i)] < fastpathAvRtid[uint(j)] 86 } 87 func (fastpathAslice) Swap(i, j int) { 88 fastpathAvRtid[uint(i)], fastpathAvRtid[uint(j)] = fastpathAvRtid[uint(j)], fastpathAvRtid[uint(i)] 89 fastpathAv[uint(i)], fastpathAv[uint(j)] = fastpathAv[uint(j)], fastpathAv[uint(i)] 90 } 91 92 func fastpathAvIndex(rtid uintptr) int { 93 // use binary search to grab the index (adapted from sort/search.go) 94 // Note: we use goto (instead of for loop) so this can be inlined. 95 // h, i, j := 0, 0, {{ .FastpathLen }} 96 var h, i uint 97 var j uint = {{ .FastpathLen }} 98 LOOP: 99 if i < j { 100 h = (i + j) >> 1 // avoid overflow when computing h // h = i + (j-i)/2 101 if fastpathAvRtid[h] < rtid { 102 i = h + 1 103 } else { 104 j = h 105 } 106 goto LOOP 107 } 108 if i < {{ .FastpathLen }} && fastpathAvRtid[i] == rtid { 109 return int(i) 110 } 111 return -1 112 } 113 114 115 // due to possible initialization loop error, make fastpath in an init() 116 func init() { 117 var i uint = 0 118 fn := func(v interface{}, 119 fe func(*Encoder, *codecFnInfo, reflect.Value), 120 fd func(*Decoder, *codecFnInfo, reflect.Value)) { 121 xrt := reflect.TypeOf(v) 122 xptr := rt2id(xrt) 123 fastpathAvRtid[i] = xptr 124 fastpathAv[i] = fastpathE{xrt, fe, fd} 125 i++ 126 } 127 {{/* do not register []byte in fast-path */}} 128 {{range .Values}}{{if not .Primitive}}{{if not .MapKey -}} 129 fn([]{{ .Elem }}(nil), (*Encoder).{{ .MethodNamePfx "fastpathEnc" false }}R, (*Decoder).{{ .MethodNamePfx "fastpathDec" false }}R) 130 {{end}}{{end}}{{end}} 131 132 {{range .Values}}{{if not .Primitive}}{{if .MapKey -}} 133 fn(map[{{ .MapKey }}]{{ .Elem }}(nil), (*Encoder).{{ .MethodNamePfx "fastpathEnc" false }}R, (*Decoder).{{ .MethodNamePfx "fastpathDec" false }}R) 134 {{end}}{{end}}{{end}} 135 136 sort.Sort(fastpathAslice{}) 137 } 138 139 // -- encode 140 141 // -- -- fast path type switch 142 func fastpathEncodeTypeSwitch(iv interface{}, e *Encoder) bool { 143 switch v := iv.(type) { 144 {{range .Values}}{{if not .Primitive}}{{if not .MapKey -}} 145 case []{{ .Elem }}: 146 fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, e) 147 case *[]{{ .Elem }}: 148 if *v == nil { 149 e.e.EncodeNil() 150 } else { 151 fastpathTV.{{ .MethodNamePfx "Enc" false }}V(*v, e) 152 } 153 {{end}}{{end}}{{end -}} 154 155 {{range .Values}}{{if not .Primitive}}{{if .MapKey -}} 156 case map[{{ .MapKey }}]{{ .Elem }}: 157 fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, e) 158 case *map[{{ .MapKey }}]{{ .Elem }}: 159 if *v == nil { 160 e.e.EncodeNil() 161 } else { 162 fastpathTV.{{ .MethodNamePfx "Enc" false }}V(*v, e) 163 } 164 {{end}}{{end}}{{end -}} 165 166 default: 167 _ = v // workaround https://github.com/golang/go/issues/12927 seen in go1.4 168 return false 169 } 170 return true 171 } 172 173 // -- -- fast path functions 174 {{range .Values}}{{if not .Primitive}}{{if not .MapKey -}} 175 func (e *Encoder) {{ .MethodNamePfx "fastpathEnc" false }}R(f *codecFnInfo, rv reflect.Value) { 176 var v []{{ .Elem }} 177 if rv.Kind() == reflect.Array { 178 rvGetSlice4Array(rv, &v) 179 } else { 180 v = rv2i(rv).([]{{ .Elem }}) 181 } 182 if f.ti.mbs { 183 fastpathTV.{{ .MethodNamePfx "EncAsMap" false }}V(v, e) 184 } else { 185 fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, e) 186 } 187 } 188 func (fastpathT) {{ .MethodNamePfx "Enc" false }}V(v []{{ .Elem }}, e *Encoder) { 189 {{/* if v == nil { e.e.EncodeNil(); return } */ -}} 190 {{ if eq .Elem "uint8" "byte" -}} 191 e.e.EncodeStringBytesRaw(v) 192 {{ else -}} 193 e.arrayStart(len(v)) 194 for j := range v { 195 e.arrayElem() 196 {{ encmd .Elem "v[j]"}} 197 } 198 e.arrayEnd() 199 {{ end -}} 200 } 201 func (fastpathT) {{ .MethodNamePfx "EncAsMap" false }}V(v []{{ .Elem }}, e *Encoder) { 202 {{/* if v == nil { e.e.EncodeNil() } else */ -}} 203 e.haltOnMbsOddLen(len(v)) 204 {{/* 205 if len(v)&1 != 0 { // similar to &1==1 or %2 == 1 206 e.errorf(fastpathMapBySliceErrMsg, len(v)) 207 } 208 */ -}} 209 e.mapStart(len(v) >> 1) // e.mapStart(len(v) / 2) 210 for j := range v { 211 if j&1 == 0 { // if j%2 == 0 { 212 e.mapElemKey() 213 } else { 214 e.mapElemValue() 215 } 216 {{ encmd .Elem "v[j]"}} 217 } 218 e.mapEnd() 219 } 220 {{end}}{{end}}{{end -}} 221 222 {{range .Values}}{{if not .Primitive}}{{if .MapKey -}} 223 func (e *Encoder) {{ .MethodNamePfx "fastpathEnc" false }}R(f *codecFnInfo, rv reflect.Value) { 224 fastpathTV.{{ .MethodNamePfx "Enc" false }}V(rv2i(rv).(map[{{ .MapKey }}]{{ .Elem }}), e) 225 } 226 func (fastpathT) {{ .MethodNamePfx "Enc" false }}V(v map[{{ .MapKey }}]{{ .Elem }}, e *Encoder) { 227 {{/* if v == nil { e.e.EncodeNil(); return } */ -}} 228 e.mapStart(len(v)) 229 if e.h.Canonical { {{/* need to figure out .NoCanonical */}} 230 {{if eq .MapKey "interface{}"}}{{/* out of band */ -}} 231 var mksv []byte = make([]byte, 0, len(v)*16) // temporary byte slice for the encoding 232 e2 := NewEncoderBytes(&mksv, e.hh) 233 v2 := make([]bytesIntf, len(v)) 234 var i, l uint {{/* put loop variables outside. seems currently needed for better perf */}} 235 var vp *bytesIntf 236 for k2 := range v { 237 l = uint(len(mksv)) 238 e2.MustEncode(k2) 239 vp = &v2[i] 240 vp.v = mksv[l:] 241 vp.i = k2 242 i++ 243 } 244 sort.Sort(bytesIntfSlice(v2)) 245 for j := range v2 { 246 e.mapElemKey() 247 e.asis(v2[j].v) 248 e.mapElemValue() 249 e.encode(v[v2[j].i]) 250 } {{else}}{{ $x := sorttype .MapKey true}}v2 := make([]{{ $x }}, len(v)) 251 var i uint 252 for k := range v { 253 v2[i] = {{if eq $x .MapKey}}k{{else}}{{ $x }}(k){{end}} 254 i++ 255 } 256 sort.Sort({{ sorttype .MapKey false}}(v2)) 257 for _, k2 := range v2 { 258 e.mapElemKey() 259 {{if eq .MapKey "string"}} e.e.EncodeString(k2) {{else}}{{ $y := printf "%s(k2)" .MapKey }}{{if eq $x .MapKey }}{{ $y = "k2" }}{{end}}{{ encmd .MapKey $y }}{{end}} 260 e.mapElemValue() 261 {{ $y := printf "v[%s(k2)]" .MapKey }}{{if eq $x .MapKey }}{{ $y = "v[k2]" }}{{end}}{{ encmd .Elem $y }} 262 } {{end}} 263 } else { 264 for k2, v2 := range v { 265 e.mapElemKey() 266 {{if eq .MapKey "string"}} e.e.EncodeString(k2) {{else}}{{ encmd .MapKey "k2"}}{{end}} 267 e.mapElemValue() 268 {{ encmd .Elem "v2"}} 269 } 270 } 271 e.mapEnd() 272 } 273 {{end}}{{end}}{{end -}} 274 275 // -- decode 276 277 // -- -- fast path type switch 278 func fastpathDecodeTypeSwitch(iv interface{}, d *Decoder) bool { 279 var changed bool 280 var containerLen int 281 switch v := iv.(type) { 282 {{range .Values}}{{if not .Primitive}}{{if not .MapKey -}} 283 case []{{ .Elem }}: 284 fastpathTV.{{ .MethodNamePfx "Dec" false }}N(v, d) 285 case *[]{{ .Elem }}: 286 var v2 []{{ .Elem }} 287 if v2, changed = fastpathTV.{{ .MethodNamePfx "Dec" false }}Y(*v, d); changed { 288 *v = v2 289 } 290 {{end}}{{end}}{{end -}} 291 {{range .Values}}{{if not .Primitive}}{{if .MapKey }}{{/* 292 // maps only change if nil, and in that case, there's no point copying 293 */ -}} 294 case map[{{ .MapKey }}]{{ .Elem }}: 295 containerLen = d.mapStart(d.d.ReadMapStart()) 296 if containerLen != containerLenNil { 297 if containerLen != 0 { 298 fastpathTV.{{ .MethodNamePfx "Dec" false }}L(v, containerLen, d) 299 } 300 d.mapEnd() 301 } 302 case *map[{{ .MapKey }}]{{ .Elem }}: 303 {{/* 304 containerLen = d.mapStart(d.d.ReadMapStart()) 305 if containerLen == 0 { 306 d.mapEnd() 307 } else if containerLen == containerLenNil { 308 *v = nil 309 } else { 310 if *v == nil { 311 *v = make(map[{{ .MapKey }}]{{ .Elem }}, decInferLen(containerLen, d.h.MaxInitLen, {{ .Size }})) 312 } 313 fastpathTV.{{ .MethodNamePfx "Dec" false }}L(*v, containerLen, d) 314 } 315 // consider delegating fully to X - encoding *map is uncommon, so ok to pay small function call cost 316 */ -}} 317 fastpathTV.{{ .MethodNamePfx "Dec" false }}X(v, d) 318 {{end}}{{end}}{{end -}} 319 default: 320 _ = v // workaround https://github.com/golang/go/issues/12927 seen in go1.4 321 return false 322 } 323 return true 324 } 325 326 func fastpathDecodeSetZeroTypeSwitch(iv interface{}) bool { 327 switch v := iv.(type) { 328 {{range .Values}}{{if not .Primitive}}{{if not .MapKey -}} 329 case *[]{{ .Elem }}: 330 *v = nil 331 {{end}}{{end}}{{end}} 332 {{range .Values}}{{if not .Primitive}}{{if .MapKey -}} 333 case *map[{{ .MapKey }}]{{ .Elem }}: 334 *v = nil 335 {{end}}{{end}}{{end}} 336 default: 337 _ = v // workaround https://github.com/golang/go/issues/12927 seen in go1.4 338 return false 339 } 340 return true 341 } 342 343 // -- -- fast path functions 344 {{range .Values}}{{if not .Primitive}}{{if not .MapKey -}} 345 {{/* 346 Slices can change if they 347 - did not come from an array 348 - are addressable (from a ptr) 349 - are settable (e.g. contained in an interface{}) 350 */}} 351 func (d *Decoder) {{ .MethodNamePfx "fastpathDec" false }}R(f *codecFnInfo, rv reflect.Value) { 352 {{/* 353 // seqTypeArray=true means that we are not getting a pointer, so no need to check that. 354 if f.seq != seqTypeArray && rv.Kind() == reflect.Ptr { 355 */ -}} 356 var v []{{ .Elem }} 357 switch rv.Kind() { 358 case reflect.Ptr: 359 vp := rv2i(rv).(*[]{{ .Elem }}) 360 var changed bool 361 if v, changed = fastpathTV.{{ .MethodNamePfx "Dec" false }}Y(*vp, d); changed { 362 *vp = v 363 } 364 case reflect.Array: 365 rvGetSlice4Array(rv, &v) 366 fastpathTV.{{ .MethodNamePfx "Dec" false }}N(v, d) 367 default: 368 fastpathTV.{{ .MethodNamePfx "Dec" false }}N(rv2i(rv).([]{{ .Elem }}), d) 369 } 370 } 371 func (f fastpathT) {{ .MethodNamePfx "Dec" false }}X(vp *[]{{ .Elem }}, d *Decoder) { 372 if v, changed := f.{{ .MethodNamePfx "Dec" false }}Y(*vp, d); changed { *vp = v } 373 } 374 func (fastpathT) {{ .MethodNamePfx "Dec" false }}Y(v []{{ .Elem }}, d *Decoder) (v2 []{{ .Elem }}, changed bool) { 375 {{ if eq .Elem "uint8" "byte" -}} 376 switch d.d.ContainerType() { 377 case valueTypeNil, valueTypeMap: 378 break 379 default: 380 v2 = d.decodeBytesInto(v[:len(v):len(v)]) 381 changed = !(len(v2) > 0 && len(v2) == len(v) && &v2[0] == &v[0]) // not same slice 382 return 383 } 384 {{ end -}} 385 slh, containerLenS := d.decSliceHelperStart() 386 if slh.IsNil { 387 if v == nil { return } 388 return nil, true 389 } 390 if containerLenS == 0 { 391 if v == nil { v = []{{ .Elem }}{} } else if len(v) != 0 { v = v[:0] } 392 slh.End() 393 return v, true 394 } 395 hasLen := containerLenS > 0 396 var xlen int 397 if hasLen { 398 if containerLenS > cap(v) { 399 xlen = decInferLen(containerLenS, d.h.MaxInitLen, {{ .Size }}) 400 if xlen <= cap(v) { 401 v = v[:uint(xlen)] 402 } else { 403 v = make([]{{ .Elem }}, uint(xlen)) 404 } 405 changed = true 406 } else if containerLenS != len(v) { 407 v = v[:containerLenS] 408 changed = true 409 } 410 } 411 var j int 412 for j = 0; d.containerNext(j, containerLenS, hasLen); j++ { 413 if j == 0 && len(v) == 0 { // means hasLen == false 414 xlen = decInferLen(containerLenS, d.h.MaxInitLen, {{ .Size }}) {{/* xlen = decDefSliceCap */}} 415 v = make([]{{ .Elem }}, uint(xlen)) 416 changed = true 417 } 418 {{/* // if indefinite, etc, then expand the slice if necessary */ -}} 419 if j >= len(v) { 420 v = append(v, {{ zerocmd .Elem }}) 421 changed = true 422 } 423 slh.ElemContainerState(j) 424 {{ if eq .Elem "interface{}" }}d.decode(&v[uint(j)]){{ else }}v[uint(j)] = {{ decmd .Elem false }}{{ end }} 425 } 426 if j < len(v) { 427 v = v[:uint(j)] 428 changed = true 429 } else if j == 0 && v == nil { 430 v = []{{ .Elem }}{} 431 changed = true 432 } 433 slh.End() 434 return v, changed 435 } 436 func (fastpathT) {{ .MethodNamePfx "Dec" false }}N(v []{{ .Elem }}, d *Decoder) { 437 {{ if eq .Elem "uint8" "byte" -}} 438 switch d.d.ContainerType() { 439 case valueTypeNil, valueTypeMap: 440 break 441 default: 442 v2 := d.decodeBytesInto(v[:len(v):len(v)]) 443 if !(len(v2) > 0 && len(v2) == len(v) && &v2[0] == &v[0]) { // not same slice 444 copy(v, v2) 445 } 446 return 447 } 448 {{ end -}} 449 slh, containerLenS := d.decSliceHelperStart() 450 if slh.IsNil { 451 return 452 } 453 if containerLenS == 0 { 454 slh.End() 455 return 456 } 457 hasLen := containerLenS > 0 458 for j := 0; d.containerNext(j, containerLenS, hasLen); j++ { 459 {{/* // if indefinite, etc, then expand the slice if necessary */ -}} 460 if j >= len(v) { 461 slh.arrayCannotExpand(hasLen, len(v), j, containerLenS) 462 return 463 } 464 slh.ElemContainerState(j) 465 {{ if eq .Elem "interface{}" -}} 466 d.decode(&v[uint(j)]) 467 {{- else -}} 468 v[uint(j)] = {{ decmd .Elem false }} 469 {{- end }} 470 } 471 slh.End() 472 } 473 {{end}}{{end}}{{end -}} 474 475 {{range .Values}}{{if not .Primitive}}{{if .MapKey -}} 476 {{/* 477 Maps can change if they are 478 - addressable (from a ptr) 479 - settable (e.g. contained in an interface{}) 480 481 Also, these methods are called by decodeValue directly, after handling a TryNil. 482 Consequently, there's no need to check for containerLenNil here. 483 */ -}} 484 func (d *Decoder) {{ .MethodNamePfx "fastpathDec" false }}R(f *codecFnInfo, rv reflect.Value) { 485 containerLen := d.mapStart(d.d.ReadMapStart()) 486 {{/* 487 if containerLen == containerLenNil { 488 if rv.Kind() == reflect.Ptr { 489 *(rv2i(rv).(*map[{{ .MapKey }}]{{ .Elem }})) = nil 490 } 491 return 492 } 493 */ -}} 494 if rv.Kind() == reflect.Ptr { 495 vp, _ := rv2i(rv).(*map[{{ .MapKey }}]{{ .Elem }}) 496 if *vp == nil { 497 *vp = make(map[{{ .MapKey }}]{{ .Elem }}, decInferLen(containerLen, d.h.MaxInitLen, {{ .Size }})) 498 } 499 if containerLen != 0 { 500 fastpathTV.{{ .MethodNamePfx "Dec" false }}L(*vp, containerLen, d) 501 } 502 } else if containerLen != 0 { 503 fastpathTV.{{ .MethodNamePfx "Dec" false }}L(rv2i(rv).(map[{{ .MapKey }}]{{ .Elem }}), containerLen, d) 504 } 505 d.mapEnd() 506 } 507 func (f fastpathT) {{ .MethodNamePfx "Dec" false }}X(vp *map[{{ .MapKey }}]{{ .Elem }}, d *Decoder) { 508 containerLen := d.mapStart(d.d.ReadMapStart()) 509 if containerLen == containerLenNil { 510 *vp = nil 511 } else { 512 if *vp == nil { 513 *vp = make(map[{{ .MapKey }}]{{ .Elem }}, decInferLen(containerLen, d.h.MaxInitLen, {{ .Size }})) 514 } 515 if containerLen != 0 { 516 f.{{ .MethodNamePfx "Dec" false }}L(*vp, containerLen, d) 517 } 518 d.mapEnd() 519 } 520 } 521 func (fastpathT) {{ .MethodNamePfx "Dec" false }}L(v map[{{ .MapKey }}]{{ .Elem }}, containerLen int, d *Decoder) { 522 {{/* No need to check if containerLen == containerLenNil, as that is checked by R and L above */ -}} 523 if v == nil { 524 d.errorf("cannot decode into nil map[{{ .MapKey }}]{{ .Elem }} given stream length: %v", containerLen) 525 {{/* d.swallowMapContents(containerLen) */ -}} 526 return 527 } 528 {{if eq .Elem "interface{}" }}mapGet := v != nil && !d.h.MapValueReset && !d.h.InterfaceReset 529 {{else if eq .Elem "bytes" "[]byte" }}mapGet := v != nil && !d.h.MapValueReset 530 {{end -}} 531 var mk {{ .MapKey }} 532 var mv {{ .Elem }} 533 hasLen := containerLen > 0 534 for j := 0; d.containerNext(j, containerLen, hasLen); j++ { 535 d.mapElemKey() 536 {{ if eq .MapKey "interface{}" }}mk = nil 537 d.decode(&mk) 538 if bv, bok := mk.([]byte); bok { 539 mk = d.stringZC(bv) {{/* // maps cannot have []byte as key. switch to string. */}} 540 }{{ else }}mk = {{ decmd .MapKey true }}{{ end }} 541 d.mapElemValue() 542 {{ if eq .Elem "interface{}" "[]byte" "bytes" -}} 543 if mapGet { mv = v[mk] } else { mv = nil } 544 {{ end -}} 545 {{ if eq .Elem "interface{}" -}} 546 d.decode(&mv) 547 {{ else if eq .Elem "[]byte" "bytes" -}} 548 mv = d.decodeBytesInto(mv) 549 {{ else -}} 550 mv = {{ decmd .Elem false }} 551 {{ end -}} 552 v[mk] = mv 553 } 554 } 555 {{end}}{{end}}{{end}}