funcdata_go116.go (18111B)
1 //go:build go1.16 && !go1.18 2 // +build go1.16,!go1.18 3 4 /* 5 * Copyright 2021 ByteDance Inc. 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 20 package loader 21 22 import ( 23 `encoding` 24 `os` 25 `unsafe` 26 27 `github.com/bytedance/sonic/internal/rt` 28 ) 29 30 const ( 31 _Magic uint32 = 0xfffffffa 32 ) 33 34 type pcHeader struct { 35 magic uint32 // 0xFFFFFFF0 36 pad1, pad2 uint8 // 0,0 37 minLC uint8 // min instruction size 38 ptrSize uint8 // size of a ptr in bytes 39 nfunc int // number of functions in the module 40 nfiles uint // number of entries in the file tab 41 funcnameOffset uintptr // offset to the funcnametab variable from pcHeader 42 cuOffset uintptr // offset to the cutab variable from pcHeader 43 filetabOffset uintptr // offset to the filetab variable from pcHeader 44 pctabOffset uintptr // offset to the pctab variable from pcHeader 45 pclnOffset uintptr // offset to the pclntab variable from pcHeader 46 } 47 48 type moduledata struct { 49 pcHeader *pcHeader 50 funcnametab []byte 51 cutab []uint32 52 filetab []byte 53 pctab []byte 54 pclntable []byte 55 ftab []funcTab 56 findfunctab uintptr 57 minpc, maxpc uintptr // first func address, last func address + last func size 58 59 text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC 60 noptrdata, enoptrdata uintptr 61 data, edata uintptr 62 bss, ebss uintptr 63 noptrbss, enoptrbss uintptr 64 end, gcdata, gcbss uintptr 65 types, etypes uintptr 66 67 textsectmap []textSection // see runtime/symtab.go: textAddr() 68 typelinks []int32 // offsets from types 69 itablinks []*rt.GoItab 70 71 ptab []ptabEntry 72 73 pluginpath string 74 pkghashes []modulehash 75 76 modulename string 77 modulehashes []modulehash 78 79 hasmain uint8 // 1 if module contains the main function, 0 otherwise 80 81 gcdatamask, gcbssmask bitVector 82 83 typemap map[int32]*rt.GoType // offset to *_rtype in previous module 84 85 bad bool // module failed to load and should be ignored 86 87 next *moduledata 88 } 89 90 type _func struct { 91 entry uintptr // start pc, as offset from moduledata.text/pcHeader.textStart 92 nameOff int32 // function name, as index into moduledata.funcnametab. 93 94 args int32 // in/out args size 95 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. 96 97 pcsp uint32 98 pcfile uint32 99 pcln uint32 100 npcdata uint32 101 cuOffset uint32 // runtime.cutab offset of this function's CU 102 funcID uint8 // set for certain special runtime functions 103 _ [2]byte // pad 104 nfuncdata uint8 // 105 106 // The end of the struct is followed immediately by two variable-length 107 // arrays that reference the pcdata and funcdata locations for this 108 // function. 109 110 // pcdata contains the offset into moduledata.pctab for the start of 111 // that index's table. e.g., 112 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of 113 // the unsafe point table. 114 // 115 // An offset of 0 indicates that there is no table. 116 // 117 // pcdata [npcdata]uint32 118 119 // funcdata contains the offset past moduledata.gofunc which contains a 120 // pointer to that index's funcdata. e.g., 121 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is 122 // the argument pointer map. 123 // 124 // An offset of ^uint32(0) indicates that there is no entry. 125 // 126 // funcdata [nfuncdata]uint32 127 } 128 129 type funcTab struct { 130 entry uintptr 131 funcoff uintptr 132 } 133 134 type bitVector struct { 135 n int32 // # of bits 136 bytedata *uint8 137 } 138 139 type ptabEntry struct { 140 name int32 141 typ int32 142 } 143 144 type textSection struct { 145 vaddr uintptr // prelinked section vaddr 146 end uintptr // vaddr + section length 147 baseaddr uintptr // relocated section address 148 } 149 150 type modulehash struct { 151 modulename string 152 linktimehash string 153 runtimehash *string 154 } 155 156 // findfuncbucket is an array of these structures. 157 // Each bucket represents 4096 bytes of the text segment. 158 // Each subbucket represents 256 bytes of the text segment. 159 // To find a function given a pc, locate the bucket and subbucket for 160 // that pc. Add together the idx and subbucket value to obtain a 161 // function index. Then scan the functab array starting at that 162 // index to find the target function. 163 // This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead. 164 type findfuncbucket struct { 165 idx uint32 166 _SUBBUCKETS [16]byte 167 } 168 169 170 type compilationUnit struct { 171 fileNames []string 172 } 173 174 // func name table format: 175 // nameOff[0] -> namePartA namePartB namePartC \x00 176 // nameOff[1] -> namePartA namePartB namePartC \x00 177 // ... 178 func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) { 179 offs = make([]int32, len(funcs)) 180 offset := 0 181 182 for i, f := range funcs { 183 offs[i] = int32(offset) 184 185 a, b, c := funcNameParts(f.Name) 186 tab = append(tab, a...) 187 tab = append(tab, b...) 188 tab = append(tab, c...) 189 tab = append(tab, 0) 190 offset += len(a) + len(b) + len(c) + 1 191 } 192 193 return 194 } 195 196 // CU table format: 197 // cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1] 198 // cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1] 199 // ... 200 // 201 // file name table format: 202 // filetabOffset[0] -> CUs[0].fileNames[0] \x00 203 // ... 204 // filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00 205 // ... 206 // filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00 207 func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) { 208 cuOffsets = make([]uint32, len(cus)) 209 cuOffset := 0 210 fileOffset := 0 211 212 for i, cu := range cus { 213 cuOffsets[i] = uint32(cuOffset) 214 215 for _, name := range cu.fileNames { 216 cutab = append(cutab, uint32(fileOffset)) 217 218 fileOffset += len(name) + 1 219 filetab = append(filetab, name...) 220 filetab = append(filetab, 0) 221 } 222 223 cuOffset += len(cu.fileNames) 224 } 225 226 return 227 } 228 229 func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) { 230 fstart = len(*out) 231 *out = append(*out, byte(0)) 232 offs := uint32(1) 233 234 funcdataOffs = make([][]uint32, len(funcs)) 235 for i, f := range funcs { 236 237 var writer = func(fd encoding.BinaryMarshaler) { 238 var ab []byte 239 var err error 240 if fd != nil { 241 ab, err = fd.MarshalBinary() 242 if err != nil { 243 panic(err) 244 } 245 funcdataOffs[i] = append(funcdataOffs[i], offs) 246 } else { 247 ab = []byte{0} 248 funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET) 249 } 250 *out = append(*out, ab...) 251 offs += uint32(len(ab)) 252 } 253 254 writer(f.ArgsPointerMaps) 255 writer(f.LocalsPointerMaps) 256 writer(f.StackObjects) 257 writer(f.InlTree) 258 writer(f.OpenCodedDeferInfo) 259 writer(f.ArgInfo) 260 writer(f.ArgLiveInfo) 261 writer(f.WrapInfo) 262 } 263 return 264 } 265 266 func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab, pclntabSize int64, startLocations []uint32) { 267 // Allocate space for the pc->func table. This structure consists of a pc offset 268 // and an offset to the func structure. After that, we have a single pc 269 // value that marks the end of the last function in the binary. 270 pclntabSize = int64(len(funcs)*2*int(_PtrSize) + int(_PtrSize)) 271 startLocations = make([]uint32, len(funcs)) 272 for i, f := range funcs { 273 pclntabSize = rnd(pclntabSize, int64(_PtrSize)) 274 //writePCToFunc 275 startLocations[i] = uint32(pclntabSize) 276 pclntabSize += int64(uint8(_FUNC_SIZE) + f.nfuncdata*_PtrSize + uint8(f.npcdata)*4) 277 } 278 ftab = make([]funcTab, 0, len(funcs)+1) 279 280 // write a map of pc->func info offsets 281 for i, f := range funcs { 282 ftab = append(ftab, funcTab{uintptr(f.entry), uintptr(startLocations[i])}) 283 } 284 285 // Final entry of table is just end pc offset. 286 lastFunc := funcs[len(funcs)-1] 287 ftab = append(ftab, funcTab{lastFunc.entry + uintptr(lastFuncSize), 0}) 288 289 return 290 } 291 292 // Pcln table format: [...]funcTab + [...]_Func 293 func makePclntable(size int64, startLocations []uint32, funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataAddr uintptr, funcdataOffs [][]uint32) (pclntab []byte) { 294 pclntab = make([]byte, size, size) 295 296 // write a map of pc->func info offsets 297 offs := 0 298 for i, f := range funcs { 299 byteOrder.PutUint64(pclntab[offs:offs+8], uint64(f.entry)) 300 byteOrder.PutUint64(pclntab[offs+8:offs+16], uint64(startLocations[i])) 301 offs += 16 302 } 303 // Final entry of table is just end pc offset. 304 lastFunc := funcs[len(funcs)-1] 305 byteOrder.PutUint64(pclntab[offs:offs+8], uint64(lastFunc.entry)+uint64(lastFuncSize)) 306 offs += 8 307 308 // write func info table 309 for i, f := range funcs { 310 off := startLocations[i] 311 312 // write _func structure to pclntab 313 byteOrder.PutUint64(pclntab[off:off+8], uint64(f.entry)) 314 off += 8 315 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.nameOff)) 316 off += 4 317 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.args)) 318 off += 4 319 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.deferreturn)) 320 off += 4 321 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcsp)) 322 off += 4 323 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcfile)) 324 off += 4 325 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.pcln)) 326 off += 4 327 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.npcdata)) 328 off += 4 329 byteOrder.PutUint32(pclntab[off:off+4], uint32(f.cuOffset)) 330 off += 4 331 pclntab[off] = f.funcID 332 // NOTICE: _[2]byte alignment 333 off += 3 334 pclntab[off] = f.nfuncdata 335 off += 1 336 337 // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3 338 for j := 3; j < len(pcdataOffs[i]); j++ { 339 byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j])) 340 off += 4 341 } 342 343 off = uint32(rnd(int64(off), int64(_PtrSize))) 344 345 // funcdata refs as offsets from gofunc 346 for _, funcdata := range funcdataOffs[i] { 347 if funcdata == _INVALID_FUNCDATA_OFFSET { 348 byteOrder.PutUint64(pclntab[off:off+8], 0) 349 } else { 350 byteOrder.PutUint64(pclntab[off:off+8], uint64(funcdataAddr)+uint64(funcdata)) 351 } 352 off += 8 353 } 354 } 355 356 return 357 } 358 359 // findfunc table used to map pc to belonging func, 360 // returns the index in the func table. 361 // 362 // All text section are divided into buckets sized _BUCKETSIZE(4K): 363 // every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64), 364 // and it has a base idx to plus the offset stored in jth subbucket. 365 // see findfunc() in runtime/symtab.go 366 func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) { 367 start = len(*out) 368 369 max := ftab[len(ftab)-1].entry 370 min := ftab[0].entry 371 nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE 372 n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE 373 374 tab := make([]findfuncbucket, 0, nbuckets) 375 var s, e = 0, 0 376 for i := 0; i<int(nbuckets); i++ { 377 var pc = min + uintptr((i+1)*_BUCKETSIZE) 378 // find the end func of the bucket 379 for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {} 380 // store the start func of the bucket 381 var fb = findfuncbucket{idx: uint32(s)} 382 383 for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ { 384 pc = min + uintptr(i*_BUCKETSIZE) + uintptr((j+1)*_SUB_BUCKETSIZE) 385 var ss = s 386 // find the end func of the subbucket 387 for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {} 388 // store the start func of the subbucket 389 fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx) 390 s = ss 391 } 392 s = e 393 tab = append(tab, fb) 394 } 395 396 // write findfuncbucket 397 if len(tab) > 0 { 398 size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab) 399 *out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...) 400 } 401 return 402 } 403 404 func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) { 405 mod = new(moduledata) 406 mod.modulename = name 407 408 // make filename table 409 cu := make([]string, 0, len(filenames)) 410 for _, f := range filenames { 411 cu = append(cu, f) 412 } 413 cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}}) 414 mod.cutab = cutab 415 mod.filetab = filetab 416 417 // make funcname table 418 funcnametab, nameOffs := makeFuncnameTab(funcs) 419 mod.funcnametab = funcnametab 420 421 // mmap() text and funcdata segements 422 p := os.Getpagesize() 423 size := int(rnd(int64(len(text)), int64(p))) 424 addr := mmap(size) 425 // copy the machine code 426 s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size) 427 copy(s, text) 428 // make it executable 429 mprotect(addr, size) 430 431 // make pcdata table 432 // NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata 433 pctab, pcdataOffs, _funcs := makePctab(funcs, addr, cuOffs, nameOffs) 434 mod.pctab = pctab 435 436 // write func data 437 // NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata 438 // TODO: estimate accurate capacity 439 cache := make([]byte, 0, len(funcs)*int(_PtrSize)) 440 fstart, funcdataOffs := writeFuncdata(&cache, funcs) 441 442 // make pc->func (binary search) func table 443 lastFuncsize := funcs[len(funcs)-1].TextSize 444 ftab, pclntSize, startLocations := makeFtab(_funcs, lastFuncsize) 445 mod.ftab = ftab 446 447 // write pc->func (modmap) findfunc table 448 ffstart := writeFindfunctab(&cache, ftab) 449 450 // cache funcdata and findfuncbucket 451 moduleCache.Lock() 452 moduleCache.m[mod] = cache 453 moduleCache.Unlock() 454 mod.findfunctab = uintptr(rt.IndexByte(cache, ffstart)) 455 funcdataAddr := uintptr(rt.IndexByte(cache, fstart)) 456 457 // make pclnt table 458 pclntab := makePclntable(pclntSize, startLocations, _funcs, lastFuncsize, pcdataOffs, funcdataAddr, funcdataOffs) 459 mod.pclntable = pclntab 460 461 // assign addresses 462 mod.text = addr 463 mod.etext = addr + uintptr(size) 464 mod.minpc = addr 465 mod.maxpc = addr + uintptr(len(text)) 466 467 // make pc header 468 mod.pcHeader = &pcHeader { 469 magic : _Magic, 470 minLC : _MinLC, 471 ptrSize : _PtrSize, 472 nfunc : len(funcs), 473 nfiles: uint(len(cu)), 474 funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"), 475 cuOffset: getOffsetOf(moduledata{}, "cutab"), 476 filetabOffset: getOffsetOf(moduledata{}, "filetab"), 477 pctabOffset: getOffsetOf(moduledata{}, "pctab"), 478 pclnOffset: getOffsetOf(moduledata{}, "pclntable"), 479 } 480 481 // sepecial case: gcdata and gcbss must by non-empty 482 mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) 483 mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) 484 485 return 486 } 487 488 // makePctab generates pcdelta->valuedelta tables for functions, 489 // and returns the table and the entry offset of every kind pcdata in the table. 490 func makePctab(funcs []Func, addr uintptr, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) { 491 _funcs = make([]_func, len(funcs)) 492 493 // Pctab offsets of 0 are considered invalid in the runtime. We respect 494 // that by just padding a single byte at the beginning of runtime.pctab, 495 // that way no real offsets can be zero. 496 pctab = make([]byte, 1, 12*len(funcs)+1) 497 pcdataOffs = make([][]uint32, len(funcs)) 498 499 for i, f := range funcs { 500 _f := &_funcs[i] 501 502 var writer = func(pc *Pcdata) { 503 var ab []byte 504 var err error 505 if pc != nil { 506 ab, err = pc.MarshalBinary() 507 if err != nil { 508 panic(err) 509 } 510 pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab))) 511 } else { 512 ab = []byte{0} 513 pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET) 514 } 515 pctab = append(pctab, ab...) 516 } 517 518 if f.Pcsp != nil { 519 _f.pcsp = uint32(len(pctab)) 520 } 521 writer(f.Pcsp) 522 if f.Pcfile != nil { 523 _f.pcfile = uint32(len(pctab)) 524 } 525 writer(f.Pcfile) 526 if f.Pcline != nil { 527 _f.pcln = uint32(len(pctab)) 528 } 529 writer(f.Pcline) 530 writer(f.PcUnsafePoint) 531 writer(f.PcStackMapIndex) 532 writer(f.PcInlTreeIndex) 533 writer(f.PcArgLiveIndex) 534 535 _f.entry = addr + uintptr(f.EntryOff) 536 _f.nameOff = nameOffset[i] 537 _f.args = f.ArgsSize 538 _f.deferreturn = f.DeferReturn 539 // NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)] 540 _f.npcdata = uint32(_N_PCDATA) 541 _f.cuOffset = cuOffset[i] 542 _f.funcID = f.ID 543 _f.nfuncdata = uint8(_N_FUNCDATA) 544 } 545 546 return 547 } 548 549 func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}