commit 68b91d2128c7be3c4f060091b172c39057318ad4
parent 8b1e2288d80a723a0c51770f0427fb6c7e07a366
Author: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com>
Date: Sat, 29 Apr 2023 17:44:20 +0100
[performance] tweak http client error handling (#1718)
* update errors library, check for more TLS type error in http client
Signed-off-by: kim <grufwub@gmail.com>
* bump cache library version to match errors library
Signed-off-by: kim <grufwub@gmail.com>
---------
Signed-off-by: kim <grufwub@gmail.com>
Diffstat:
11 files changed, 115 insertions(+), 49 deletions(-)
diff --git a/go.mod b/go.mod
@@ -5,9 +5,9 @@ go 1.20
require (
codeberg.org/gruf/go-bytesize v1.0.2
codeberg.org/gruf/go-byteutil v1.1.2
- codeberg.org/gruf/go-cache/v3 v3.2.5
+ codeberg.org/gruf/go-cache/v3 v3.2.6
codeberg.org/gruf/go-debug v1.3.0
- codeberg.org/gruf/go-errors/v2 v2.1.1
+ codeberg.org/gruf/go-errors/v2 v2.2.0
codeberg.org/gruf/go-fastcopy v1.1.2
codeberg.org/gruf/go-kv v1.6.1
codeberg.org/gruf/go-logger/v2 v2.2.1
diff --git a/go.sum b/go.sum
@@ -49,13 +49,13 @@ codeberg.org/gruf/go-bytesize v1.0.2/go.mod h1:n/GU8HzL9f3UNp/mUKyr1qVmTlj7+xacp
codeberg.org/gruf/go-byteutil v1.0.0/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
codeberg.org/gruf/go-byteutil v1.1.2 h1:TQLZtTxTNca9xEfDIndmo7nBYxeS94nrv/9DS3Nk5Tw=
codeberg.org/gruf/go-byteutil v1.1.2/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU=
-codeberg.org/gruf/go-cache/v3 v3.2.5 h1:C+JwTR4uxjuE6qwqB48d3NCRJejsbzxRpfFEBReaViA=
-codeberg.org/gruf/go-cache/v3 v3.2.5/go.mod h1:up7za8FtNdtttcx6AJ8ffqkrSkXDGTilsd9yJ0IyhfM=
+codeberg.org/gruf/go-cache/v3 v3.2.6 h1:PtAGOvCTGwhqOqIEFBP4M0F6xbaAWYe3t/7QYGNzulI=
+codeberg.org/gruf/go-cache/v3 v3.2.6/go.mod h1:pTeVPEb9DshXUkd8Dg76UcsLpU6EC/tXQ2qb+JrmxEc=
codeberg.org/gruf/go-debug v1.3.0 h1:PIRxQiWUFKtGOGZFdZ3Y0pqyfI0Xr87j224IYe2snZs=
codeberg.org/gruf/go-debug v1.3.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIjLILzpLg=
codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4=
-codeberg.org/gruf/go-errors/v2 v2.1.1 h1:oj7JUIvUBafF60HrwN74JrCMol1Ouh3gq1ggrH5hGTw=
-codeberg.org/gruf/go-errors/v2 v2.1.1/go.mod h1:LfzD9nkAAJpEDbkUqOZQ2jdaQ8VrK0pnR36zLOMFq6Y=
+codeberg.org/gruf/go-errors/v2 v2.2.0 h1:CxnTtR4+BqRGeBHuG/FdCKM4m3otMdfPVez6ReBebkM=
+codeberg.org/gruf/go-errors/v2 v2.2.0/go.mod h1:LfzD9nkAAJpEDbkUqOZQ2jdaQ8VrK0pnR36zLOMFq6Y=
codeberg.org/gruf/go-fastcopy v1.1.2 h1:YwmYXPsyOcRBxKEE2+w1bGAZfclHVaPijFsOVOcnNcw=
codeberg.org/gruf/go-fastcopy v1.1.2/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s=
codeberg.org/gruf/go-fastpath v1.0.1/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI=
diff --git a/internal/cache/util.go b/internal/cache/util.go
@@ -35,7 +35,7 @@ var SentinelError = errors.New("BUG: error should not be returned") //nolint:rev
// ignoreErrors is an error ignoring function capable of being passed to
// caches, which specifically catches and ignores our sentinel error type.
func ignoreErrors(err error) bool {
- return errorsv2.Is(
+ return errorsv2.Comparable(
SentinelError,
context.DeadlineExceeded,
context.Canceled,
diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go
@@ -149,7 +149,7 @@ func New(cfg Config) *Client {
// Initiate outgoing bad hosts lookup cache.
c.badHosts = cache.New[string, struct{}](0, 1000, 0)
- c.badHosts.SetTTL(15*time.Minute, false)
+ c.badHosts.SetTTL(time.Hour, false)
if !c.badHosts.Start(time.Minute) {
log.Panic(nil, "failed to start transport controller cache")
}
@@ -165,7 +165,7 @@ func (c *Client) Do(r *http.Request) (*http.Response, error) {
}
// DoSigned ...
-func (c *Client) DoSigned(r *http.Request, sign SignFunc) (*http.Response, error) {
+func (c *Client) DoSigned(r *http.Request, sign SignFunc) (rsp *http.Response, err error) {
const (
// max no. attempts.
maxRetries = 5
@@ -182,10 +182,16 @@ func (c *Client) DoSigned(r *http.Request, sign SignFunc) (*http.Response, error
if !fastFail {
// Check if recently reached max retries for this host
// so we don't bother with a retry-backoff loop. The only
- // errors that are retried upon are server failure and
- // domain resolution type errors, so this cached result
+ // errors that are retried upon are server failure, TLS
+ // and domain resolution type errors, so this cached result
// indicates this server is likely having issues.
fastFail = c.badHosts.Has(host)
+ defer func() {
+ if err != nil {
+ // On error return mark as bad-host.
+ c.badHosts.Set(host, struct{}{})
+ }
+ }()
}
// Start a log entry for this request
@@ -218,7 +224,7 @@ func (c *Client) DoSigned(r *http.Request, sign SignFunc) (*http.Response, error
l.Infof("performing request")
// Perform the request.
- rsp, err := c.do(r)
+ rsp, err = c.do(r)
if err == nil { //nolint:gocritic
// TooManyRequest means we need to slow
@@ -249,20 +255,27 @@ func (c *Client) DoSigned(r *http.Request, sign SignFunc) (*http.Response, error
}
}
- } else if errorsv2.Is(err,
+ // Ensure unset.
+ rsp = nil
+
+ } else if errorsv2.Comparable(err,
context.DeadlineExceeded,
context.Canceled,
ErrBodyTooLarge,
ErrReservedAddr,
) {
- // Return on non-retryable errors
+ // Non-retryable errors.
+ return nil, err
+ } else if errorsv2.Assignable(err,
+ (*x509.CertificateInvalidError)(nil),
+ (*x509.HostnameError)(nil),
+ (*x509.UnknownAuthorityError)(nil),
+ ) {
+ // Non-retryable TLS errors.
return nil, err
} else if strings.Contains(err.Error(), "stopped after 10 redirects") {
// Don't bother if net/http returned after too many redirects
return nil, err
- } else if errors.As(err, &x509.UnknownAuthorityError{}) {
- // Unknown authority errors we do NOT recover from
- return nil, err
} else if dnserr := (*net.DNSError)(nil); // nocollapse
errors.As(err, &dnserr) && dnserr.IsNotFound {
// DNS lookup failure, this domain does not exist
@@ -292,10 +305,9 @@ func (c *Client) DoSigned(r *http.Request, sign SignFunc) (*http.Response, error
}
}
- // Add "bad" entry for this host.
- c.badHosts.Set(host, struct{}{})
-
- return nil, errors.New("transport reached max retries")
+ // Set error return to trigger setting "bad host".
+ err = errors.New("transport reached max retries")
+ return
}
// do ...
diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go
@@ -95,7 +95,7 @@ func (p *ProcessingEmoji) load(ctx context.Context) (*gtsmodel.Emoji, bool, erro
defer func() {
// This is only done when ctx NOT cancelled.
- done = err == nil || !errors.Is(err,
+ done = err == nil || !errors.Comparable(err,
context.Canceled,
context.DeadlineExceeded,
)
diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go
@@ -95,7 +95,7 @@ func (p *ProcessingMedia) load(ctx context.Context) (*gtsmodel.MediaAttachment,
defer func() {
// This is only done when ctx NOT cancelled.
- done = err == nil || !errors.Is(err,
+ done = err == nil || !errors.Comparable(err,
context.Canceled,
context.DeadlineExceeded,
)
diff --git a/vendor/codeberg.org/gruf/go-cache/v3/result/cache.go b/vendor/codeberg.org/gruf/go-cache/v3/result/cache.go
@@ -138,7 +138,7 @@ func (c *Cache[Value]) SetInvalidateCallback(hook func(Value)) {
func (c *Cache[Value]) IgnoreErrors(ignore func(error) bool) {
if ignore == nil {
ignore = func(err error) bool {
- return errors.Is(
+ return errors.Comparable(
err,
context.Canceled,
context.DeadlineExceeded,
diff --git a/vendor/codeberg.org/gruf/go-errors/v2/errors.go b/vendor/codeberg.org/gruf/go-errors/v2/errors.go
@@ -1,6 +1,7 @@
package errors
import (
+ "errors"
"fmt"
)
@@ -29,7 +30,7 @@ func Stacktrace(err error) Callers {
var e interface {
Stacktrace() Callers
}
- if !As(err, &e) {
+ if !errors.As(err, &e) {
return nil
}
return e.Stacktrace()
diff --git a/vendor/codeberg.org/gruf/go-errors/v2/standard.go b/vendor/codeberg.org/gruf/go-errors/v2/standard.go
@@ -8,15 +8,11 @@ import (
"codeberg.org/gruf/go-bitutil"
)
-// Is reports whether any error in err's chain matches any of targets
-// (up to a max of 64 targets).
-//
-// The chain consists of err itself followed by the sequence of errors obtained by
-// repeatedly calling Unwrap.
-//
-// An error is considered to match a target if it is equal to that target or if
-// it implements a method Is(error) bool such that Is(target) returns true.
-func Is(err error, targets ...error) bool {
+// errtype is a ptr to the error interface type.
+var errtype = reflect.TypeOf((*error)(nil)).Elem()
+
+// Comparable is functionally equivalent to calling errors.Is() on multiple errors (up to a max of 64).
+func Comparable(err error, targets ...error) bool {
var flags bitutil.Flags64
// Flags only has 64 bit-slots
@@ -24,17 +20,15 @@ func Is(err error, targets ...error) bool {
panic("too many targets")
}
- // Check if error is nil so we can catch
- // the fast-case where a target is nil
- isNil := (err == nil)
-
for i := 0; i < len(targets); {
- // Drop nil targets
if targets[i] == nil {
- if isNil /* match! */ {
+ if err == nil {
return true
}
- targets = append(targets[:i], targets[i+1:]...)
+
+ // Drop nil targets from slice.
+ copy(targets[i:], targets[i+1:])
+ targets = targets[:len(targets)-1]
continue
}
@@ -81,11 +75,68 @@ func Is(err error, targets ...error) bool {
return false
}
-// As finds the first error in err's chain that matches target, and if one is found, sets
+// Assignable is functionally equivalent to calling errors.As() on multiple errors,
+// except that it only checks assignability as opposed to setting the target.
+func Assignable(err error, targets ...error) bool {
+ if err == nil {
+ // Fastest case.
+ return false
+ }
+
+ for i := 0; i < len(targets); {
+ if targets[i] == nil {
+ // Drop nil targets from slice.
+ copy(targets[i:], targets[i+1:])
+ targets = targets[:len(targets)-1]
+ continue
+ }
+ i++
+ }
+
+ for err != nil {
+ // Check if this layer supports .As interface
+ as, ok := err.(interface{ As(any) bool })
+
+ // Get reflected err type.
+ te := reflect.TypeOf(err)
+
+ if !ok {
+ // Error does not support interface.
+ //
+ // Check assignability using reflection.
+ for i := 0; i < len(targets); i++ {
+ tt := reflect.TypeOf(targets[i])
+ if te.AssignableTo(tt) {
+ return true
+ }
+ }
+ } else {
+ // Error supports the .As interface.
+ //
+ // Check using .As() and reflection.
+ for i := 0; i < len(targets); i++ {
+ if as.As(targets[i]) {
+ return true
+ } else if tt := reflect.TypeOf(targets[i]); // nocollapse
+ te.AssignableTo(tt) {
+ return true
+ }
+ }
+ }
+
+ // Unwrap to next layer.
+ err = errors.Unwrap(err)
+ }
+
+ return false
+}
+
+// As finds the first error in err's tree that matches target, and if one is found, sets
// target to that error value and returns true. Otherwise, it returns false.
//
-// The chain consists of err itself followed by the sequence of errors obtained by
-// repeatedly calling Unwrap.
+// The tree consists of err itself, followed by the errors obtained by repeatedly
+// calling Unwrap. When err wraps multiple errors, As examines err followed by a
+// depth-first traversal of its children.
//
// An error matches target if the error's concrete value is assignable to the value
// pointed to by target, or if the error has a method As(interface{}) bool such that
@@ -99,7 +150,7 @@ func Is(err error, targets ...error) bool {
// error, or to any interface type.
//
//go:linkname As errors.As
-func As(err error, target interface{}) bool
+func As(err error, target any) bool
// Unwrap returns the result of calling the Unwrap method on err, if err's
// type contains an Unwrap method returning error. Otherwise, Unwrap returns nil.
diff --git a/vendor/codeberg.org/gruf/go-errors/v2/value.go b/vendor/codeberg.org/gruf/go-errors/v2/value.go
@@ -1,5 +1,7 @@
package errors
+import "errors"
+
// WithValue wraps err to store given key-value pair, accessible via Value() function.
func WithValue(err error, key any, value any) error {
if err == nil {
@@ -16,7 +18,7 @@ func WithValue(err error, key any, value any) error {
func Value(err error, key any) any {
var e *errWithValue
- if !As(err, &e) {
+ if !errors.As(err, &e) {
return nil
}
@@ -47,7 +49,7 @@ func (e *errWithValue) Value(key any) any {
return e.val
}
- if !As(e.err, &e) {
+ if !errors.As(e.err, &e) {
return nil
}
}
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -13,7 +13,7 @@ codeberg.org/gruf/go-bytesize
# codeberg.org/gruf/go-byteutil v1.1.2
## explicit; go 1.16
codeberg.org/gruf/go-byteutil
-# codeberg.org/gruf/go-cache/v3 v3.2.5
+# codeberg.org/gruf/go-cache/v3 v3.2.6
## explicit; go 1.19
codeberg.org/gruf/go-cache/v3
codeberg.org/gruf/go-cache/v3/result
@@ -21,7 +21,7 @@ codeberg.org/gruf/go-cache/v3/ttl
# codeberg.org/gruf/go-debug v1.3.0
## explicit; go 1.16
codeberg.org/gruf/go-debug
-# codeberg.org/gruf/go-errors/v2 v2.1.1
+# codeberg.org/gruf/go-errors/v2 v2.2.0
## explicit; go 1.19
codeberg.org/gruf/go-errors/v2
# codeberg.org/gruf/go-fastcopy v1.1.2
