gtsocial-umbx

Unnamed repository; edit this file 'description' to name the repository.
Log | Files | Refs | README | LICENSE

prog.go (24925B)


      1 package ebpf
      2 
      3 import (
      4 	"bytes"
      5 	"encoding/binary"
      6 	"errors"
      7 	"fmt"
      8 	"math"
      9 	"path/filepath"
     10 	"runtime"
     11 	"strings"
     12 	"time"
     13 
     14 	"github.com/cilium/ebpf/asm"
     15 	"github.com/cilium/ebpf/btf"
     16 	"github.com/cilium/ebpf/internal"
     17 	"github.com/cilium/ebpf/internal/sys"
     18 	"github.com/cilium/ebpf/internal/unix"
     19 )
     20 
     21 // ErrNotSupported is returned whenever the kernel doesn't support a feature.
     22 var ErrNotSupported = internal.ErrNotSupported
     23 
     24 // ProgramID represents the unique ID of an eBPF program.
     25 type ProgramID uint32
     26 
     27 const (
     28 	// Number of bytes to pad the output buffer for BPF_PROG_TEST_RUN.
     29 	// This is currently the maximum of spare space allocated for SKB
     30 	// and XDP programs, and equal to XDP_PACKET_HEADROOM + NET_IP_ALIGN.
     31 	outputPad = 256 + 2
     32 )
     33 
     34 // DefaultVerifierLogSize is the default number of bytes allocated for the
     35 // verifier log.
     36 const DefaultVerifierLogSize = 64 * 1024
     37 
     38 // ProgramOptions control loading a program into the kernel.
     39 type ProgramOptions struct {
     40 	// Controls the detail emitted by the kernel verifier. Set to non-zero
     41 	// to enable logging.
     42 	LogLevel uint32
     43 	// Controls the output buffer size for the verifier. Defaults to
     44 	// DefaultVerifierLogSize.
     45 	LogSize int
     46 	// Type information used for CO-RE relocations and when attaching to
     47 	// kernel functions.
     48 	//
     49 	// This is useful in environments where the kernel BTF is not available
     50 	// (containers) or where it is in a non-standard location. Defaults to
     51 	// use the kernel BTF from a well-known location if nil.
     52 	KernelTypes *btf.Spec
     53 }
     54 
     55 // ProgramSpec defines a Program.
     56 type ProgramSpec struct {
     57 	// Name is passed to the kernel as a debug aid. Must only contain
     58 	// alpha numeric and '_' characters.
     59 	Name string
     60 
     61 	// Type determines at which hook in the kernel a program will run.
     62 	Type ProgramType
     63 
     64 	// AttachType of the program, needed to differentiate allowed context
     65 	// accesses in some newer program types like CGroupSockAddr.
     66 	//
     67 	// Available on kernels 4.17 and later.
     68 	AttachType AttachType
     69 
     70 	// Name of a kernel data structure or function to attach to. Its
     71 	// interpretation depends on Type and AttachType.
     72 	AttachTo string
     73 
     74 	// The program to attach to. Must be provided manually.
     75 	AttachTarget *Program
     76 
     77 	// The name of the ELF section this program orininated from.
     78 	SectionName string
     79 
     80 	Instructions asm.Instructions
     81 
     82 	// Flags is passed to the kernel and specifies additional program
     83 	// load attributes.
     84 	Flags uint32
     85 
     86 	// License of the program. Some helpers are only available if
     87 	// the license is deemed compatible with the GPL.
     88 	//
     89 	// See https://www.kernel.org/doc/html/latest/process/license-rules.html#id1
     90 	License string
     91 
     92 	// Version used by Kprobe programs.
     93 	//
     94 	// Deprecated on kernels 5.0 and later. Leave empty to let the library
     95 	// detect this value automatically.
     96 	KernelVersion uint32
     97 
     98 	// The BTF associated with this program. Changing Instructions
     99 	// will most likely invalidate the contained data, and may
    100 	// result in errors when attempting to load it into the kernel.
    101 	BTF *btf.Spec
    102 
    103 	// The byte order this program was compiled for, may be nil.
    104 	ByteOrder binary.ByteOrder
    105 }
    106 
    107 // Copy returns a copy of the spec.
    108 func (ps *ProgramSpec) Copy() *ProgramSpec {
    109 	if ps == nil {
    110 		return nil
    111 	}
    112 
    113 	cpy := *ps
    114 	cpy.Instructions = make(asm.Instructions, len(ps.Instructions))
    115 	copy(cpy.Instructions, ps.Instructions)
    116 	return &cpy
    117 }
    118 
    119 // Tag calculates the kernel tag for a series of instructions.
    120 //
    121 // Use asm.Instructions.Tag if you need to calculate for non-native endianness.
    122 func (ps *ProgramSpec) Tag() (string, error) {
    123 	return ps.Instructions.Tag(internal.NativeEndian)
    124 }
    125 
    126 type VerifierError = internal.VerifierError
    127 
    128 // Program represents BPF program loaded into the kernel.
    129 //
    130 // It is not safe to close a Program which is used by other goroutines.
    131 type Program struct {
    132 	// Contains the output of the kernel verifier if enabled,
    133 	// otherwise it is empty.
    134 	VerifierLog string
    135 
    136 	fd         *sys.FD
    137 	name       string
    138 	pinnedPath string
    139 	typ        ProgramType
    140 }
    141 
    142 // NewProgram creates a new Program.
    143 //
    144 // See NewProgramWithOptions for details.
    145 func NewProgram(spec *ProgramSpec) (*Program, error) {
    146 	return NewProgramWithOptions(spec, ProgramOptions{})
    147 }
    148 
    149 // NewProgramWithOptions creates a new Program.
    150 //
    151 // Loading a program for the first time will perform
    152 // feature detection by loading small, temporary programs.
    153 //
    154 // Returns an error wrapping VerifierError if the program or its BTF is rejected
    155 // by the kernel.
    156 func NewProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, error) {
    157 	if spec == nil {
    158 		return nil, errors.New("can't load a program from a nil spec")
    159 	}
    160 
    161 	handles := newHandleCache()
    162 	defer handles.close()
    163 
    164 	prog, err := newProgramWithOptions(spec, opts, handles)
    165 	if errors.Is(err, asm.ErrUnsatisfiedMapReference) {
    166 		return nil, fmt.Errorf("cannot load program without loading its whole collection: %w", err)
    167 	}
    168 	return prog, err
    169 }
    170 
    171 func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions, handles *handleCache) (*Program, error) {
    172 	if len(spec.Instructions) == 0 {
    173 		return nil, errors.New("instructions cannot be empty")
    174 	}
    175 
    176 	if spec.Type == UnspecifiedProgram {
    177 		return nil, errors.New("can't load program of unspecified type")
    178 	}
    179 
    180 	if spec.ByteOrder != nil && spec.ByteOrder != internal.NativeEndian {
    181 		return nil, fmt.Errorf("can't load %s program on %s", spec.ByteOrder, internal.NativeEndian)
    182 	}
    183 
    184 	// Kernels before 5.0 (6c4fc209fcf9 "bpf: remove useless version check for prog load")
    185 	// require the version field to be set to the value of the KERNEL_VERSION
    186 	// macro for kprobe-type programs.
    187 	// Overwrite Kprobe program version if set to zero or the magic version constant.
    188 	kv := spec.KernelVersion
    189 	if spec.Type == Kprobe && (kv == 0 || kv == internal.MagicKernelVersion) {
    190 		v, err := internal.KernelVersion()
    191 		if err != nil {
    192 			return nil, fmt.Errorf("detecting kernel version: %w", err)
    193 		}
    194 		kv = v.Kernel()
    195 	}
    196 
    197 	attr := &sys.ProgLoadAttr{
    198 		ProgType:           sys.ProgType(spec.Type),
    199 		ProgFlags:          spec.Flags,
    200 		ExpectedAttachType: sys.AttachType(spec.AttachType),
    201 		License:            sys.NewStringPointer(spec.License),
    202 		KernVersion:        kv,
    203 	}
    204 
    205 	if haveObjName() == nil {
    206 		attr.ProgName = sys.NewObjName(spec.Name)
    207 	}
    208 
    209 	kernelTypes := opts.KernelTypes
    210 
    211 	insns := make(asm.Instructions, len(spec.Instructions))
    212 	copy(insns, spec.Instructions)
    213 
    214 	var btfDisabled bool
    215 	if spec.BTF != nil {
    216 		if err := applyRelocations(insns, spec.BTF, kernelTypes); err != nil {
    217 			return nil, fmt.Errorf("apply CO-RE relocations: %w", err)
    218 		}
    219 
    220 		handle, err := handles.btfHandle(spec.BTF)
    221 		btfDisabled = errors.Is(err, btf.ErrNotSupported)
    222 		if err != nil && !btfDisabled {
    223 			return nil, fmt.Errorf("load BTF: %w", err)
    224 		}
    225 
    226 		if handle != nil {
    227 			attr.ProgBtfFd = uint32(handle.FD())
    228 
    229 			fib, lib, err := btf.MarshalExtInfos(insns, spec.BTF.TypeID)
    230 			if err != nil {
    231 				return nil, err
    232 			}
    233 
    234 			attr.FuncInfoRecSize = btf.FuncInfoSize
    235 			attr.FuncInfoCnt = uint32(len(fib)) / btf.FuncInfoSize
    236 			attr.FuncInfo = sys.NewSlicePointer(fib)
    237 
    238 			attr.LineInfoRecSize = btf.LineInfoSize
    239 			attr.LineInfoCnt = uint32(len(lib)) / btf.LineInfoSize
    240 			attr.LineInfo = sys.NewSlicePointer(lib)
    241 		}
    242 	}
    243 
    244 	if err := fixupAndValidate(insns); err != nil {
    245 		return nil, err
    246 	}
    247 
    248 	buf := bytes.NewBuffer(make([]byte, 0, insns.Size()))
    249 	err := insns.Marshal(buf, internal.NativeEndian)
    250 	if err != nil {
    251 		return nil, err
    252 	}
    253 
    254 	bytecode := buf.Bytes()
    255 	attr.Insns = sys.NewSlicePointer(bytecode)
    256 	attr.InsnCnt = uint32(len(bytecode) / asm.InstructionSize)
    257 
    258 	if spec.AttachTarget != nil {
    259 		targetID, err := findTargetInProgram(spec.AttachTarget, spec.AttachTo, spec.Type, spec.AttachType)
    260 		if err != nil {
    261 			return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err)
    262 		}
    263 
    264 		attr.AttachBtfId = uint32(targetID)
    265 		attr.AttachProgFd = uint32(spec.AttachTarget.FD())
    266 		defer runtime.KeepAlive(spec.AttachTarget)
    267 	} else if spec.AttachTo != "" {
    268 		targetID, err := findTargetInKernel(kernelTypes, spec.AttachTo, spec.Type, spec.AttachType)
    269 		if err != nil && !errors.Is(err, errUnrecognizedAttachType) {
    270 			// We ignore errUnrecognizedAttachType since AttachTo may be non-empty
    271 			// for programs that don't attach anywhere.
    272 			return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err)
    273 		}
    274 
    275 		attr.AttachBtfId = uint32(targetID)
    276 	}
    277 
    278 	logSize := DefaultVerifierLogSize
    279 	if opts.LogSize > 0 {
    280 		logSize = opts.LogSize
    281 	}
    282 
    283 	var logBuf []byte
    284 	if opts.LogLevel > 0 {
    285 		logBuf = make([]byte, logSize)
    286 		attr.LogLevel = opts.LogLevel
    287 		attr.LogSize = uint32(len(logBuf))
    288 		attr.LogBuf = sys.NewSlicePointer(logBuf)
    289 	}
    290 
    291 	fd, err := sys.ProgLoad(attr)
    292 	if err == nil {
    293 		return &Program{unix.ByteSliceToString(logBuf), fd, spec.Name, "", spec.Type}, nil
    294 	}
    295 
    296 	if opts.LogLevel == 0 && opts.LogSize >= 0 {
    297 		// Re-run with the verifier enabled to get better error messages.
    298 		logBuf = make([]byte, logSize)
    299 		attr.LogLevel = 1
    300 		attr.LogSize = uint32(len(logBuf))
    301 		attr.LogBuf = sys.NewSlicePointer(logBuf)
    302 		_, _ = sys.ProgLoad(attr)
    303 	}
    304 
    305 	switch {
    306 	case errors.Is(err, unix.EPERM):
    307 		if len(logBuf) > 0 && logBuf[0] == 0 {
    308 			// EPERM due to RLIMIT_MEMLOCK happens before the verifier, so we can
    309 			// check that the log is empty to reduce false positives.
    310 			return nil, fmt.Errorf("load program: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err)
    311 		}
    312 
    313 		fallthrough
    314 
    315 	case errors.Is(err, unix.EINVAL):
    316 		if hasFunctionReferences(spec.Instructions) {
    317 			if err := haveBPFToBPFCalls(); err != nil {
    318 				return nil, fmt.Errorf("load program: %w", err)
    319 			}
    320 		}
    321 	}
    322 
    323 	err = internal.ErrorWithLog(err, logBuf)
    324 	if btfDisabled {
    325 		return nil, fmt.Errorf("load program: %w (BTF disabled)", err)
    326 	}
    327 	return nil, fmt.Errorf("load program: %w", err)
    328 }
    329 
    330 // NewProgramFromFD creates a program from a raw fd.
    331 //
    332 // You should not use fd after calling this function.
    333 //
    334 // Requires at least Linux 4.10.
    335 func NewProgramFromFD(fd int) (*Program, error) {
    336 	f, err := sys.NewFD(fd)
    337 	if err != nil {
    338 		return nil, err
    339 	}
    340 
    341 	return newProgramFromFD(f)
    342 }
    343 
    344 // NewProgramFromID returns the program for a given id.
    345 //
    346 // Returns ErrNotExist, if there is no eBPF program with the given id.
    347 func NewProgramFromID(id ProgramID) (*Program, error) {
    348 	fd, err := sys.ProgGetFdById(&sys.ProgGetFdByIdAttr{
    349 		Id: uint32(id),
    350 	})
    351 	if err != nil {
    352 		return nil, fmt.Errorf("get program by id: %w", err)
    353 	}
    354 
    355 	return newProgramFromFD(fd)
    356 }
    357 
    358 func newProgramFromFD(fd *sys.FD) (*Program, error) {
    359 	info, err := newProgramInfoFromFd(fd)
    360 	if err != nil {
    361 		fd.Close()
    362 		return nil, fmt.Errorf("discover program type: %w", err)
    363 	}
    364 
    365 	return &Program{"", fd, "", "", info.Type}, nil
    366 }
    367 
    368 func (p *Program) String() string {
    369 	if p.name != "" {
    370 		return fmt.Sprintf("%s(%s)#%v", p.typ, p.name, p.fd)
    371 	}
    372 	return fmt.Sprintf("%s(%v)", p.typ, p.fd)
    373 }
    374 
    375 // Type returns the underlying type of the program.
    376 func (p *Program) Type() ProgramType {
    377 	return p.typ
    378 }
    379 
    380 // Info returns metadata about the program.
    381 //
    382 // Requires at least 4.10.
    383 func (p *Program) Info() (*ProgramInfo, error) {
    384 	return newProgramInfoFromFd(p.fd)
    385 }
    386 
    387 // Handle returns a reference to the program's type information in the kernel.
    388 //
    389 // Returns ErrNotSupported if the kernel has no BTF support, or if there is no
    390 // BTF associated with the program.
    391 func (p *Program) Handle() (*btf.Handle, error) {
    392 	info, err := p.Info()
    393 	if err != nil {
    394 		return nil, err
    395 	}
    396 
    397 	id, ok := info.BTFID()
    398 	if !ok {
    399 		return nil, fmt.Errorf("program %s: retrieve BTF ID: %w", p, ErrNotSupported)
    400 	}
    401 
    402 	return btf.NewHandleFromID(id)
    403 }
    404 
    405 // FD gets the file descriptor of the Program.
    406 //
    407 // It is invalid to call this function after Close has been called.
    408 func (p *Program) FD() int {
    409 	return p.fd.Int()
    410 }
    411 
    412 // Clone creates a duplicate of the Program.
    413 //
    414 // Closing the duplicate does not affect the original, and vice versa.
    415 //
    416 // Cloning a nil Program returns nil.
    417 func (p *Program) Clone() (*Program, error) {
    418 	if p == nil {
    419 		return nil, nil
    420 	}
    421 
    422 	dup, err := p.fd.Dup()
    423 	if err != nil {
    424 		return nil, fmt.Errorf("can't clone program: %w", err)
    425 	}
    426 
    427 	return &Program{p.VerifierLog, dup, p.name, "", p.typ}, nil
    428 }
    429 
    430 // Pin persists the Program on the BPF virtual file system past the lifetime of
    431 // the process that created it
    432 //
    433 // Calling Pin on a previously pinned program will overwrite the path, except when
    434 // the new path already exists. Re-pinning across filesystems is not supported.
    435 //
    436 // This requires bpffs to be mounted above fileName. See https://docs.cilium.io/en/k8s-doc/admin/#admin-mount-bpffs
    437 func (p *Program) Pin(fileName string) error {
    438 	if err := internal.Pin(p.pinnedPath, fileName, p.fd); err != nil {
    439 		return err
    440 	}
    441 	p.pinnedPath = fileName
    442 	return nil
    443 }
    444 
    445 // Unpin removes the persisted state for the Program from the BPF virtual filesystem.
    446 //
    447 // Failed calls to Unpin will not alter the state returned by IsPinned.
    448 //
    449 // Unpinning an unpinned Program returns nil.
    450 func (p *Program) Unpin() error {
    451 	if err := internal.Unpin(p.pinnedPath); err != nil {
    452 		return err
    453 	}
    454 	p.pinnedPath = ""
    455 	return nil
    456 }
    457 
    458 // IsPinned returns true if the Program has a non-empty pinned path.
    459 func (p *Program) IsPinned() bool {
    460 	return p.pinnedPath != ""
    461 }
    462 
    463 // Close the Program's underlying file descriptor, which could unload
    464 // the program from the kernel if it is not pinned or attached to a
    465 // kernel hook.
    466 func (p *Program) Close() error {
    467 	if p == nil {
    468 		return nil
    469 	}
    470 
    471 	return p.fd.Close()
    472 }
    473 
    474 // Various options for Run'ing a Program
    475 type RunOptions struct {
    476 	// Program's data input. Required field.
    477 	Data []byte
    478 	// Program's data after Program has run. Caller must allocate. Optional field.
    479 	DataOut []byte
    480 	// Program's context input. Optional field.
    481 	Context interface{}
    482 	// Program's context after Program has run. Must be a pointer or slice. Optional field.
    483 	ContextOut interface{}
    484 	// Number of times to run Program. Optional field. Defaults to 1.
    485 	Repeat uint32
    486 	// Optional flags.
    487 	Flags uint32
    488 	// CPU to run Program on. Optional field.
    489 	// Note not all program types support this field.
    490 	CPU uint32
    491 	// Called whenever the syscall is interrupted, and should be set to testing.B.ResetTimer
    492 	// or similar. Typically used during benchmarking. Optional field.
    493 	Reset func()
    494 }
    495 
    496 // Test runs the Program in the kernel with the given input and returns the
    497 // value returned by the eBPF program. outLen may be zero.
    498 //
    499 // Note: the kernel expects at least 14 bytes input for an ethernet header for
    500 // XDP and SKB programs.
    501 //
    502 // This function requires at least Linux 4.12.
    503 func (p *Program) Test(in []byte) (uint32, []byte, error) {
    504 	// Older kernels ignore the dataSizeOut argument when copying to user space.
    505 	// Combined with things like bpf_xdp_adjust_head() we don't really know what the final
    506 	// size will be. Hence we allocate an output buffer which we hope will always be large
    507 	// enough, and panic if the kernel wrote past the end of the allocation.
    508 	// See https://patchwork.ozlabs.org/cover/1006822/
    509 	var out []byte
    510 	if len(in) > 0 {
    511 		out = make([]byte, len(in)+outputPad)
    512 	}
    513 
    514 	opts := RunOptions{
    515 		Data:    in,
    516 		DataOut: out,
    517 		Repeat:  1,
    518 	}
    519 
    520 	ret, _, err := p.testRun(&opts)
    521 	if err != nil {
    522 		return ret, nil, fmt.Errorf("can't test program: %w", err)
    523 	}
    524 	return ret, opts.DataOut, nil
    525 }
    526 
    527 // Run runs the Program in kernel with given RunOptions.
    528 //
    529 // Note: the same restrictions from Test apply.
    530 func (p *Program) Run(opts *RunOptions) (uint32, error) {
    531 	ret, _, err := p.testRun(opts)
    532 	if err != nil {
    533 		return ret, fmt.Errorf("can't test program: %w", err)
    534 	}
    535 	return ret, nil
    536 }
    537 
    538 // Benchmark runs the Program with the given input for a number of times
    539 // and returns the time taken per iteration.
    540 //
    541 // Returns the result of the last execution of the program and the time per
    542 // run or an error. reset is called whenever the benchmark syscall is
    543 // interrupted, and should be set to testing.B.ResetTimer or similar.
    544 //
    545 // Note: profiling a call to this function will skew it's results, see
    546 // https://github.com/cilium/ebpf/issues/24
    547 //
    548 // This function requires at least Linux 4.12.
    549 func (p *Program) Benchmark(in []byte, repeat int, reset func()) (uint32, time.Duration, error) {
    550 	if uint(repeat) > math.MaxUint32 {
    551 		return 0, 0, fmt.Errorf("repeat is too high")
    552 	}
    553 
    554 	opts := RunOptions{
    555 		Data:   in,
    556 		Repeat: uint32(repeat),
    557 		Reset:  reset,
    558 	}
    559 
    560 	ret, total, err := p.testRun(&opts)
    561 	if err != nil {
    562 		return ret, total, fmt.Errorf("can't benchmark program: %w", err)
    563 	}
    564 	return ret, total, nil
    565 }
    566 
    567 var haveProgTestRun = internal.FeatureTest("BPF_PROG_TEST_RUN", "4.12", func() error {
    568 	prog, err := NewProgram(&ProgramSpec{
    569 		// SocketFilter does not require privileges on newer kernels.
    570 		Type: SocketFilter,
    571 		Instructions: asm.Instructions{
    572 			asm.LoadImm(asm.R0, 0, asm.DWord),
    573 			asm.Return(),
    574 		},
    575 		License: "MIT",
    576 	})
    577 	if err != nil {
    578 		// This may be because we lack sufficient permissions, etc.
    579 		return err
    580 	}
    581 	defer prog.Close()
    582 
    583 	// Programs require at least 14 bytes input
    584 	in := make([]byte, 14)
    585 	attr := sys.ProgRunAttr{
    586 		ProgFd:     uint32(prog.FD()),
    587 		DataSizeIn: uint32(len(in)),
    588 		DataIn:     sys.NewSlicePointer(in),
    589 	}
    590 
    591 	err = sys.ProgRun(&attr)
    592 	switch {
    593 	case errors.Is(err, unix.EINVAL):
    594 		// Check for EINVAL specifically, rather than err != nil since we
    595 		// otherwise misdetect due to insufficient permissions.
    596 		return internal.ErrNotSupported
    597 
    598 	case errors.Is(err, unix.EINTR):
    599 		// We know that PROG_TEST_RUN is supported if we get EINTR.
    600 		return nil
    601 
    602 	case errors.Is(err, unix.ENOTSUPP):
    603 		// The first PROG_TEST_RUN patches shipped in 4.12 didn't include
    604 		// a test runner for SocketFilter. ENOTSUPP means PROG_TEST_RUN is
    605 		// supported, but not for the program type used in the probe.
    606 		return nil
    607 	}
    608 
    609 	return err
    610 })
    611 
    612 func (p *Program) testRun(opts *RunOptions) (uint32, time.Duration, error) {
    613 	if uint(len(opts.Data)) > math.MaxUint32 {
    614 		return 0, 0, fmt.Errorf("input is too long")
    615 	}
    616 
    617 	if err := haveProgTestRun(); err != nil {
    618 		return 0, 0, err
    619 	}
    620 
    621 	var ctxBytes []byte
    622 	if opts.Context != nil {
    623 		ctx := new(bytes.Buffer)
    624 		if err := binary.Write(ctx, internal.NativeEndian, opts.Context); err != nil {
    625 			return 0, 0, fmt.Errorf("cannot serialize context: %v", err)
    626 		}
    627 		ctxBytes = ctx.Bytes()
    628 	}
    629 
    630 	var ctxOut []byte
    631 	if opts.ContextOut != nil {
    632 		ctxOut = make([]byte, binary.Size(opts.ContextOut))
    633 	}
    634 
    635 	attr := sys.ProgRunAttr{
    636 		ProgFd:      p.fd.Uint(),
    637 		DataSizeIn:  uint32(len(opts.Data)),
    638 		DataSizeOut: uint32(len(opts.DataOut)),
    639 		DataIn:      sys.NewSlicePointer(opts.Data),
    640 		DataOut:     sys.NewSlicePointer(opts.DataOut),
    641 		Repeat:      uint32(opts.Repeat),
    642 		CtxSizeIn:   uint32(len(ctxBytes)),
    643 		CtxSizeOut:  uint32(len(ctxOut)),
    644 		CtxIn:       sys.NewSlicePointer(ctxBytes),
    645 		CtxOut:      sys.NewSlicePointer(ctxOut),
    646 		Flags:       opts.Flags,
    647 		Cpu:         opts.CPU,
    648 	}
    649 
    650 	for {
    651 		err := sys.ProgRun(&attr)
    652 		if err == nil {
    653 			break
    654 		}
    655 
    656 		if errors.Is(err, unix.EINTR) {
    657 			if opts.Reset != nil {
    658 				opts.Reset()
    659 			}
    660 			continue
    661 		}
    662 
    663 		if errors.Is(err, unix.ENOTSUPP) {
    664 			return 0, 0, fmt.Errorf("kernel doesn't support testing program type %s: %w", p.Type(), ErrNotSupported)
    665 		}
    666 
    667 		return 0, 0, fmt.Errorf("can't run test: %w", err)
    668 	}
    669 
    670 	if opts.DataOut != nil {
    671 		if int(attr.DataSizeOut) > cap(opts.DataOut) {
    672 			// Houston, we have a problem. The program created more data than we allocated,
    673 			// and the kernel wrote past the end of our buffer.
    674 			panic("kernel wrote past end of output buffer")
    675 		}
    676 		opts.DataOut = opts.DataOut[:int(attr.DataSizeOut)]
    677 	}
    678 
    679 	if len(ctxOut) != 0 {
    680 		b := bytes.NewReader(ctxOut)
    681 		if err := binary.Read(b, internal.NativeEndian, opts.ContextOut); err != nil {
    682 			return 0, 0, fmt.Errorf("failed to decode ContextOut: %v", err)
    683 		}
    684 	}
    685 
    686 	total := time.Duration(attr.Duration) * time.Nanosecond
    687 	return attr.Retval, total, nil
    688 }
    689 
    690 func unmarshalProgram(buf []byte) (*Program, error) {
    691 	if len(buf) != 4 {
    692 		return nil, errors.New("program id requires 4 byte value")
    693 	}
    694 
    695 	// Looking up an entry in a nested map or prog array returns an id,
    696 	// not an fd.
    697 	id := internal.NativeEndian.Uint32(buf)
    698 	return NewProgramFromID(ProgramID(id))
    699 }
    700 
    701 func marshalProgram(p *Program, length int) ([]byte, error) {
    702 	if length != 4 {
    703 		return nil, fmt.Errorf("can't marshal program to %d bytes", length)
    704 	}
    705 
    706 	buf := make([]byte, 4)
    707 	internal.NativeEndian.PutUint32(buf, p.fd.Uint())
    708 	return buf, nil
    709 }
    710 
    711 // LoadPinnedProgram loads a Program from a BPF file.
    712 //
    713 // Requires at least Linux 4.11.
    714 func LoadPinnedProgram(fileName string, opts *LoadPinOptions) (*Program, error) {
    715 	fd, err := sys.ObjGet(&sys.ObjGetAttr{
    716 		Pathname:  sys.NewStringPointer(fileName),
    717 		FileFlags: opts.Marshal(),
    718 	})
    719 	if err != nil {
    720 		return nil, err
    721 	}
    722 
    723 	info, err := newProgramInfoFromFd(fd)
    724 	if err != nil {
    725 		_ = fd.Close()
    726 		return nil, fmt.Errorf("info for %s: %w", fileName, err)
    727 	}
    728 
    729 	return &Program{"", fd, filepath.Base(fileName), fileName, info.Type}, nil
    730 }
    731 
    732 // SanitizeName replaces all invalid characters in name with replacement.
    733 // Passing a negative value for replacement will delete characters instead
    734 // of replacing them. Use this to automatically generate valid names for maps
    735 // and programs at runtime.
    736 //
    737 // The set of allowed characters depends on the running kernel version.
    738 // Dots are only allowed as of kernel 5.2.
    739 func SanitizeName(name string, replacement rune) string {
    740 	return strings.Map(func(char rune) rune {
    741 		if invalidBPFObjNameChar(char) {
    742 			return replacement
    743 		}
    744 		return char
    745 	}, name)
    746 }
    747 
    748 // ProgramGetNextID returns the ID of the next eBPF program.
    749 //
    750 // Returns ErrNotExist, if there is no next eBPF program.
    751 func ProgramGetNextID(startID ProgramID) (ProgramID, error) {
    752 	attr := &sys.ProgGetNextIdAttr{Id: uint32(startID)}
    753 	return ProgramID(attr.NextId), sys.ProgGetNextId(attr)
    754 }
    755 
    756 // BindMap binds map to the program and is only released once program is released.
    757 //
    758 // This may be used in cases where metadata should be associated with the program
    759 // which otherwise does not contain any references to the map.
    760 func (p *Program) BindMap(m *Map) error {
    761 	attr := &sys.ProgBindMapAttr{
    762 		ProgFd: uint32(p.FD()),
    763 		MapFd:  uint32(m.FD()),
    764 	}
    765 
    766 	return sys.ProgBindMap(attr)
    767 }
    768 
    769 var errUnrecognizedAttachType = errors.New("unrecognized attach type")
    770 
    771 // find an attach target type in the kernel.
    772 //
    773 // spec may be nil and defaults to the canonical kernel BTF. name together with
    774 // progType and attachType determine which type we need to attach to.
    775 //
    776 // Returns errUnrecognizedAttachType.
    777 func findTargetInKernel(spec *btf.Spec, name string, progType ProgramType, attachType AttachType) (btf.TypeID, error) {
    778 	type match struct {
    779 		p ProgramType
    780 		a AttachType
    781 	}
    782 
    783 	var (
    784 		typeName, featureName string
    785 		isBTFTypeFunc         = true
    786 	)
    787 
    788 	switch (match{progType, attachType}) {
    789 	case match{LSM, AttachLSMMac}:
    790 		typeName = "bpf_lsm_" + name
    791 		featureName = name + " LSM hook"
    792 	case match{Tracing, AttachTraceIter}:
    793 		typeName = "bpf_iter_" + name
    794 		featureName = name + " iterator"
    795 	case match{Tracing, AttachTraceFEntry}:
    796 		typeName = name
    797 		featureName = fmt.Sprintf("fentry %s", name)
    798 	case match{Tracing, AttachTraceFExit}:
    799 		typeName = name
    800 		featureName = fmt.Sprintf("fexit %s", name)
    801 	case match{Tracing, AttachModifyReturn}:
    802 		typeName = name
    803 		featureName = fmt.Sprintf("fmod_ret %s", name)
    804 	case match{Tracing, AttachTraceRawTp}:
    805 		typeName = fmt.Sprintf("btf_trace_%s", name)
    806 		featureName = fmt.Sprintf("raw_tp %s", name)
    807 		isBTFTypeFunc = false
    808 	default:
    809 		return 0, errUnrecognizedAttachType
    810 	}
    811 
    812 	spec, err := maybeLoadKernelBTF(spec)
    813 	if err != nil {
    814 		return 0, fmt.Errorf("load kernel spec: %w", err)
    815 	}
    816 
    817 	var target btf.Type
    818 	if isBTFTypeFunc {
    819 		var targetFunc *btf.Func
    820 		err = spec.TypeByName(typeName, &targetFunc)
    821 		target = targetFunc
    822 	} else {
    823 		var targetTypedef *btf.Typedef
    824 		err = spec.TypeByName(typeName, &targetTypedef)
    825 		target = targetTypedef
    826 	}
    827 
    828 	if err != nil {
    829 		if errors.Is(err, btf.ErrNotFound) {
    830 			return 0, &internal.UnsupportedFeatureError{
    831 				Name: featureName,
    832 			}
    833 		}
    834 		return 0, fmt.Errorf("find target for %s: %w", featureName, err)
    835 	}
    836 
    837 	return spec.TypeID(target)
    838 }
    839 
    840 // find an attach target type in a program.
    841 //
    842 // Returns errUnrecognizedAttachType.
    843 func findTargetInProgram(prog *Program, name string, progType ProgramType, attachType AttachType) (btf.TypeID, error) {
    844 	type match struct {
    845 		p ProgramType
    846 		a AttachType
    847 	}
    848 
    849 	var typeName string
    850 	switch (match{progType, attachType}) {
    851 	case match{Extension, AttachNone}:
    852 		typeName = name
    853 	default:
    854 		return 0, errUnrecognizedAttachType
    855 	}
    856 
    857 	btfHandle, err := prog.Handle()
    858 	if err != nil {
    859 		return 0, fmt.Errorf("load target BTF: %w", err)
    860 	}
    861 	defer btfHandle.Close()
    862 
    863 	spec, err := btfHandle.Spec(nil)
    864 	if err != nil {
    865 		return 0, err
    866 	}
    867 
    868 	var targetFunc *btf.Func
    869 	err = spec.TypeByName(typeName, &targetFunc)
    870 	if err != nil {
    871 		return 0, fmt.Errorf("find target %s: %w", typeName, err)
    872 	}
    873 
    874 	return spec.TypeID(targetFunc)
    875 }