funcdata_go118.go (17735B)
1 // go:build go1.18 && !go1.20 2 // +build go1.18,!go1.20 3 4 /* 5 * Copyright 2021 ByteDance Inc. 6 * 7 * Licensed under the Apache License, Version 2.0 (the "License"); 8 * you may not use this file except in compliance with the License. 9 * You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 20 package loader 21 22 import ( 23 `encoding` 24 `os` 25 `unsafe` 26 27 `github.com/bytedance/sonic/internal/rt` 28 ) 29 30 const ( 31 _Magic uint32 = 0xfffffff0 32 ) 33 34 type pcHeader struct { 35 magic uint32 // 0xFFFFFFF0 36 pad1, pad2 uint8 // 0,0 37 minLC uint8 // min instruction size 38 ptrSize uint8 // size of a ptr in bytes 39 nfunc int // number of functions in the module 40 nfiles uint // number of entries in the file tab 41 textStart uintptr // base for function entry PC offsets in this module, equal to moduledata.text 42 funcnameOffset uintptr // offset to the funcnametab variable from pcHeader 43 cuOffset uintptr // offset to the cutab variable from pcHeader 44 filetabOffset uintptr // offset to the filetab variable from pcHeader 45 pctabOffset uintptr // offset to the pctab variable from pcHeader 46 pclnOffset uintptr // offset to the pclntab variable from pcHeader 47 } 48 49 type moduledata struct { 50 pcHeader *pcHeader 51 funcnametab []byte 52 cutab []uint32 53 filetab []byte 54 pctab []byte 55 pclntable []byte 56 ftab []funcTab 57 findfunctab uintptr 58 minpc, maxpc uintptr // first func address, last func address + last func size 59 60 text, etext uintptr // start/end of text, (etext-text) must be greater than MIN_FUNC 61 noptrdata, enoptrdata uintptr 62 data, edata uintptr 63 bss, ebss uintptr 64 noptrbss, enoptrbss uintptr 65 end, gcdata, gcbss uintptr 66 types, etypes uintptr 67 rodata uintptr 68 gofunc uintptr // go.func.* is actual funcinfo object in image 69 70 textsectmap []textSection // see runtime/symtab.go: textAddr() 71 typelinks []int32 // offsets from types 72 itablinks []*rt.GoItab 73 74 ptab []ptabEntry 75 76 pluginpath string 77 pkghashes []modulehash 78 79 modulename string 80 modulehashes []modulehash 81 82 hasmain uint8 // 1 if module contains the main function, 0 otherwise 83 84 gcdatamask, gcbssmask bitVector 85 86 typemap map[int32]*rt.GoType // offset to *_rtype in previous module 87 88 bad bool // module failed to load and should be ignored 89 90 next *moduledata 91 } 92 93 type _func struct { 94 entryOff uint32 // start pc, as offset from moduledata.text/pcHeader.textStart 95 nameOff int32 // function name, as index into moduledata.funcnametab. 96 97 args int32 // in/out args size 98 deferreturn uint32 // offset of start of a deferreturn call instruction from entry, if any. 99 100 pcsp uint32 101 pcfile uint32 102 pcln uint32 103 npcdata uint32 104 cuOffset uint32 // runtime.cutab offset of this function's CU 105 funcID uint8 // set for certain special runtime functions 106 flag uint8 107 _ [1]byte // pad 108 nfuncdata uint8 // 109 110 // The end of the struct is followed immediately by two variable-length 111 // arrays that reference the pcdata and funcdata locations for this 112 // function. 113 114 // pcdata contains the offset into moduledata.pctab for the start of 115 // that index's table. e.g., 116 // &moduledata.pctab[_func.pcdata[_PCDATA_UnsafePoint]] is the start of 117 // the unsafe point table. 118 // 119 // An offset of 0 indicates that there is no table. 120 // 121 // pcdata [npcdata]uint32 122 123 // funcdata contains the offset past moduledata.gofunc which contains a 124 // pointer to that index's funcdata. e.g., 125 // *(moduledata.gofunc + _func.funcdata[_FUNCDATA_ArgsPointerMaps]) is 126 // the argument pointer map. 127 // 128 // An offset of ^uint32(0) indicates that there is no entry. 129 // 130 // funcdata [nfuncdata]uint32 131 } 132 133 type funcTab struct { 134 entry uint32 135 funcoff uint32 136 } 137 138 type bitVector struct { 139 n int32 // # of bits 140 bytedata *uint8 141 } 142 143 type ptabEntry struct { 144 name int32 145 typ int32 146 } 147 148 type textSection struct { 149 vaddr uintptr // prelinked section vaddr 150 end uintptr // vaddr + section length 151 baseaddr uintptr // relocated section address 152 } 153 154 type modulehash struct { 155 modulename string 156 linktimehash string 157 runtimehash *string 158 } 159 160 // findfuncbucket is an array of these structures. 161 // Each bucket represents 4096 bytes of the text segment. 162 // Each subbucket represents 256 bytes of the text segment. 163 // To find a function given a pc, locate the bucket and subbucket for 164 // that pc. Add together the idx and subbucket value to obtain a 165 // function index. Then scan the functab array starting at that 166 // index to find the target function. 167 // This table uses 20 bytes for every 4096 bytes of code, or ~0.5% overhead. 168 type findfuncbucket struct { 169 idx uint32 170 _SUBBUCKETS [16]byte 171 } 172 173 // func name table format: 174 // nameOff[0] -> namePartA namePartB namePartC \x00 175 // nameOff[1] -> namePartA namePartB namePartC \x00 176 // ... 177 func makeFuncnameTab(funcs []Func) (tab []byte, offs []int32) { 178 offs = make([]int32, len(funcs)) 179 offset := 0 180 181 for i, f := range funcs { 182 offs[i] = int32(offset) 183 184 a, b, c := funcNameParts(f.Name) 185 tab = append(tab, a...) 186 tab = append(tab, b...) 187 tab = append(tab, c...) 188 tab = append(tab, 0) 189 offset += len(a) + len(b) + len(c) + 1 190 } 191 192 return 193 } 194 195 type compilationUnit struct { 196 fileNames []string 197 } 198 199 // CU table format: 200 // cuOffsets[0] -> filetabOffset[0] filetabOffset[1] ... filetabOffset[len(CUs[0].fileNames)-1] 201 // cuOffsets[1] -> filetabOffset[len(CUs[0].fileNames)] ... filetabOffset[len(CUs[0].fileNames) + len(CUs[1].fileNames)-1] 202 // ... 203 // 204 // file name table format: 205 // filetabOffset[0] -> CUs[0].fileNames[0] \x00 206 // ... 207 // filetabOffset[len(CUs[0]-1)] -> CUs[0].fileNames[len(CUs[0].fileNames)-1] \x00 208 // ... 209 // filetabOffset[SUM(CUs,fileNames)-1] -> CUs[len(CU)-1].fileNames[len(CUs[len(CU)-1].fileNames)-1] \x00 210 func makeFilenametab(cus []compilationUnit) (cutab []uint32, filetab []byte, cuOffsets []uint32) { 211 cuOffsets = make([]uint32, len(cus)) 212 cuOffset := 0 213 fileOffset := 0 214 215 for i, cu := range cus { 216 cuOffsets[i] = uint32(cuOffset) 217 218 for _, name := range cu.fileNames { 219 cutab = append(cutab, uint32(fileOffset)) 220 221 fileOffset += len(name) + 1 222 filetab = append(filetab, name...) 223 filetab = append(filetab, 0) 224 } 225 226 cuOffset += len(cu.fileNames) 227 } 228 229 return 230 } 231 232 func writeFuncdata(out *[]byte, funcs []Func) (fstart int, funcdataOffs [][]uint32) { 233 fstart = len(*out) 234 *out = append(*out, byte(0)) 235 offs := uint32(1) 236 237 funcdataOffs = make([][]uint32, len(funcs)) 238 for i, f := range funcs { 239 240 var writer = func(fd encoding.BinaryMarshaler) { 241 var ab []byte 242 var err error 243 if fd != nil { 244 ab, err = fd.MarshalBinary() 245 if err != nil { 246 panic(err) 247 } 248 funcdataOffs[i] = append(funcdataOffs[i], offs) 249 } else { 250 ab = []byte{0} 251 funcdataOffs[i] = append(funcdataOffs[i], _INVALID_FUNCDATA_OFFSET) 252 } 253 *out = append(*out, ab...) 254 offs += uint32(len(ab)) 255 } 256 257 writer(f.ArgsPointerMaps) 258 writer(f.LocalsPointerMaps) 259 writer(f.StackObjects) 260 writer(f.InlTree) 261 writer(f.OpenCodedDeferInfo) 262 writer(f.ArgInfo) 263 writer(f.ArgLiveInfo) 264 writer(f.WrapInfo) 265 } 266 return 267 } 268 269 func makeFtab(funcs []_func, lastFuncSize uint32) (ftab []funcTab) { 270 // Allocate space for the pc->func table. This structure consists of a pc offset 271 // and an offset to the func structure. After that, we have a single pc 272 // value that marks the end of the last function in the binary. 273 var size int64 = int64(len(funcs)*2*4 + 4) 274 var startLocations = make([]uint32, len(funcs)) 275 for i, f := range funcs { 276 size = rnd(size, int64(_PtrSize)) 277 //writePCToFunc 278 startLocations[i] = uint32(size) 279 size += int64(uint8(_FUNC_SIZE)+f.nfuncdata*4+uint8(f.npcdata)*4) 280 } 281 282 ftab = make([]funcTab, 0, len(funcs)+1) 283 284 // write a map of pc->func info offsets 285 for i, f := range funcs { 286 ftab = append(ftab, funcTab{uint32(f.entryOff), uint32(startLocations[i])}) 287 } 288 289 // Final entry of table is just end pc offset. 290 lastFunc := funcs[len(funcs)-1] 291 ftab = append(ftab, funcTab{uint32(lastFunc.entryOff + lastFuncSize), 0}) 292 293 return 294 } 295 296 // Pcln table format: [...]funcTab + [...]_Func 297 func makePclntable(funcs []_func, lastFuncSize uint32, pcdataOffs [][]uint32, funcdataOffs [][]uint32) (pclntab []byte) { 298 // Allocate space for the pc->func table. This structure consists of a pc offset 299 // and an offset to the func structure. After that, we have a single pc 300 // value that marks the end of the last function in the binary. 301 var size int64 = int64(len(funcs)*2*4 + 4) 302 var startLocations = make([]uint32, len(funcs)) 303 for i := range funcs { 304 size = rnd(size, int64(_PtrSize)) 305 //writePCToFunc 306 startLocations[i] = uint32(size) 307 size += int64(int(_FUNC_SIZE)+len(funcdataOffs[i])*4+len(pcdataOffs[i])*4) 308 } 309 310 pclntab = make([]byte, size, size) 311 312 // write a map of pc->func info offsets 313 offs := 0 314 for i, f := range funcs { 315 byteOrder.PutUint32(pclntab[offs:offs+4], uint32(f.entryOff)) 316 byteOrder.PutUint32(pclntab[offs+4:offs+8], uint32(startLocations[i])) 317 offs += 8 318 } 319 // Final entry of table is just end pc offset. 320 lastFunc := funcs[len(funcs)-1] 321 byteOrder.PutUint32(pclntab[offs:offs+4], uint32(lastFunc.entryOff+lastFuncSize)) 322 323 // write func info table 324 for i, f := range funcs { 325 off := startLocations[i] 326 327 // write _func structure to pclntab 328 fb := rt.BytesFrom(unsafe.Pointer(&f), int(_FUNC_SIZE), int(_FUNC_SIZE)) 329 copy(pclntab[off:off+uint32(_FUNC_SIZE)], fb) 330 off += uint32(_FUNC_SIZE) 331 332 // NOTICE: _func.pcdata always starts from PcUnsafePoint, which is index 3 333 for j := 3; j < len(pcdataOffs[i]); j++ { 334 byteOrder.PutUint32(pclntab[off:off+4], uint32(pcdataOffs[i][j])) 335 off += 4 336 } 337 338 // funcdata refs as offsets from gofunc 339 for _, funcdata := range funcdataOffs[i] { 340 byteOrder.PutUint32(pclntab[off:off+4], uint32(funcdata)) 341 off += 4 342 } 343 344 } 345 346 return 347 } 348 349 // findfunc table used to map pc to belonging func, 350 // returns the index in the func table. 351 // 352 // All text section are divided into buckets sized _BUCKETSIZE(4K): 353 // every bucket is divided into _SUBBUCKETS sized _SUB_BUCKETSIZE(64), 354 // and it has a base idx to plus the offset stored in jth subbucket. 355 // see findfunc() in runtime/symtab.go 356 func writeFindfunctab(out *[]byte, ftab []funcTab) (start int) { 357 start = len(*out) 358 359 max := ftab[len(ftab)-1].entry 360 min := ftab[0].entry 361 nbuckets := (max - min + _BUCKETSIZE - 1) / _BUCKETSIZE 362 n := (max - min + _SUB_BUCKETSIZE - 1) / _SUB_BUCKETSIZE 363 364 tab := make([]findfuncbucket, 0, nbuckets) 365 var s, e = 0, 0 366 for i := 0; i<int(nbuckets); i++ { 367 var pc = min + uint32((i+1)*_BUCKETSIZE) 368 // find the end func of the bucket 369 for ; e < len(ftab)-1 && ftab[e+1].entry <= pc; e++ {} 370 // store the start func of the bucket 371 var fb = findfuncbucket{idx: uint32(s)} 372 373 for j := 0; j<_SUBBUCKETS && (i*_SUBBUCKETS+j)<int(n); j++ { 374 pc = min + uint32(i*_BUCKETSIZE) + uint32((j+1)*_SUB_BUCKETSIZE) 375 var ss = s 376 // find the end func of the subbucket 377 for ; ss < len(ftab)-1 && ftab[ss+1].entry <= pc; ss++ {} 378 // store the start func of the subbucket 379 fb._SUBBUCKETS[j] = byte(uint32(s) - fb.idx) 380 s = ss 381 } 382 s = e 383 tab = append(tab, fb) 384 } 385 386 // write findfuncbucket 387 if len(tab) > 0 { 388 size := int(unsafe.Sizeof(findfuncbucket{}))*len(tab) 389 *out = append(*out, rt.BytesFrom(unsafe.Pointer(&tab[0]), size, size)...) 390 } 391 return 392 } 393 394 func makeModuledata(name string, filenames []string, funcs []Func, text []byte) (mod *moduledata) { 395 mod = new(moduledata) 396 mod.modulename = name 397 398 // make filename table 399 cu := make([]string, 0, len(filenames)) 400 for _, f := range filenames { 401 cu = append(cu, f) 402 } 403 cutab, filetab, cuOffs := makeFilenametab([]compilationUnit{{cu}}) 404 mod.cutab = cutab 405 mod.filetab = filetab 406 407 // make funcname table 408 funcnametab, nameOffs := makeFuncnameTab(funcs) 409 mod.funcnametab = funcnametab 410 411 // make pcdata table 412 // NOTICE: _func only use offset to index pcdata, thus no need mmap() pcdata 413 pctab, pcdataOffs, _funcs := makePctab(funcs, cuOffs, nameOffs) 414 mod.pctab = pctab 415 416 // write func data 417 // NOTICE: _func use mod.gofunc+offset to directly point funcdata, thus need cache funcdata 418 // TODO: estimate accurate capacity 419 cache := make([]byte, 0, len(funcs)*int(_PtrSize)) 420 fstart, funcdataOffs := writeFuncdata(&cache, funcs) 421 422 // make pc->func (binary search) func table 423 lastFuncsize := funcs[len(funcs)-1].TextSize 424 ftab := makeFtab(_funcs, lastFuncsize) 425 mod.ftab = ftab 426 427 // write pc->func (modmap) findfunc table 428 ffstart := writeFindfunctab(&cache, ftab) 429 430 // make pclnt table 431 pclntab := makePclntable(_funcs, lastFuncsize, pcdataOffs, funcdataOffs) 432 mod.pclntable = pclntab 433 434 // mmap() text and funcdata segements 435 p := os.Getpagesize() 436 size := int(rnd(int64(len(text)), int64(p))) 437 addr := mmap(size) 438 // copy the machine code 439 s := rt.BytesFrom(unsafe.Pointer(addr), len(text), size) 440 copy(s, text) 441 // make it executable 442 mprotect(addr, size) 443 444 // assign addresses 445 mod.text = addr 446 mod.etext = addr + uintptr(size) 447 mod.minpc = addr 448 mod.maxpc = addr + uintptr(len(text)) 449 450 // cache funcdata and findfuncbucket 451 moduleCache.Lock() 452 moduleCache.m[mod] = cache 453 moduleCache.Unlock() 454 mod.gofunc = uintptr(unsafe.Pointer(&cache[fstart])) 455 mod.findfunctab = uintptr(unsafe.Pointer(&cache[ffstart])) 456 457 // make pc header 458 mod.pcHeader = &pcHeader { 459 magic : _Magic, 460 minLC : _MinLC, 461 ptrSize : _PtrSize, 462 nfunc : len(funcs), 463 nfiles: uint(len(cu)), 464 textStart: mod.text, 465 funcnameOffset: getOffsetOf(moduledata{}, "funcnametab"), 466 cuOffset: getOffsetOf(moduledata{}, "cutab"), 467 filetabOffset: getOffsetOf(moduledata{}, "filetab"), 468 pctabOffset: getOffsetOf(moduledata{}, "pctab"), 469 pclnOffset: getOffsetOf(moduledata{}, "pclntable"), 470 } 471 472 // sepecial case: gcdata and gcbss must by non-empty 473 mod.gcdata = uintptr(unsafe.Pointer(&emptyByte)) 474 mod.gcbss = uintptr(unsafe.Pointer(&emptyByte)) 475 476 return 477 } 478 479 // makePctab generates pcdelta->valuedelta tables for functions, 480 // and returns the table and the entry offset of every kind pcdata in the table. 481 func makePctab(funcs []Func, cuOffset []uint32, nameOffset []int32) (pctab []byte, pcdataOffs [][]uint32, _funcs []_func) { 482 _funcs = make([]_func, len(funcs)) 483 484 // Pctab offsets of 0 are considered invalid in the runtime. We respect 485 // that by just padding a single byte at the beginning of runtime.pctab, 486 // that way no real offsets can be zero. 487 pctab = make([]byte, 1, 12*len(funcs)+1) 488 pcdataOffs = make([][]uint32, len(funcs)) 489 490 for i, f := range funcs { 491 _f := &_funcs[i] 492 493 var writer = func(pc *Pcdata) { 494 var ab []byte 495 var err error 496 if pc != nil { 497 ab, err = pc.MarshalBinary() 498 if err != nil { 499 panic(err) 500 } 501 pcdataOffs[i] = append(pcdataOffs[i], uint32(len(pctab))) 502 } else { 503 ab = []byte{0} 504 pcdataOffs[i] = append(pcdataOffs[i], _PCDATA_INVALID_OFFSET) 505 } 506 pctab = append(pctab, ab...) 507 } 508 509 if f.Pcsp != nil { 510 _f.pcsp = uint32(len(pctab)) 511 } 512 writer(f.Pcsp) 513 if f.Pcfile != nil { 514 _f.pcfile = uint32(len(pctab)) 515 } 516 writer(f.Pcfile) 517 if f.Pcline != nil { 518 _f.pcln = uint32(len(pctab)) 519 } 520 writer(f.Pcline) 521 writer(f.PcUnsafePoint) 522 writer(f.PcStackMapIndex) 523 writer(f.PcInlTreeIndex) 524 writer(f.PcArgLiveIndex) 525 526 _f.entryOff = f.EntryOff 527 _f.nameOff = nameOffset[i] 528 _f.args = f.ArgsSize 529 _f.deferreturn = f.DeferReturn 530 // NOTICE: _func.pcdata is always as [PCDATA_UnsafePoint(0) : PCDATA_ArgLiveIndex(3)] 531 _f.npcdata = uint32(_N_PCDATA) 532 _f.cuOffset = cuOffset[i] 533 _f.funcID = f.ID 534 _f.flag = f.Flag 535 _f.nfuncdata = uint8(_N_FUNCDATA) 536 } 537 538 return 539 } 540 541 func registerFunction(name string, pc uintptr, textSize uintptr, fp int, args int, size uintptr, argptrs uintptr, localptrs uintptr) {}