commit e63b6531994adcf976d8e15c2b791682b8531e7d
parent 4b4c935e02cee98d06b8824fc3751dfc88603e52
Author: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Tue, 15 Mar 2022 15:01:19 +0100
[performance] Add dereference shortcuts to avoid making http calls to self (#430)
* update transport (controller) to allow shortcuts
* go fmt
* expose underlying sig transport to allow test sigs
Diffstat:
7 files changed, 114 insertions(+), 22 deletions(-)
diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go
@@ -117,7 +117,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
return fmt.Errorf("error creating media manager: %s", err)
}
oauthServer := oauth.New(ctx, dbService)
- transportController := transport.NewController(dbService, &federation.Clock{}, http.DefaultClient)
+ transportController := transport.NewController(dbService, federatingDB, &federation.Clock{}, http.DefaultClient)
federator := federation.NewFederator(dbService, federatingDB, transportController, typeConverter, mediaManager)
// decide whether to create a noop email sender (won't send emails) or a real one
diff --git a/internal/transport/controller.go b/internal/transport/controller.go
@@ -21,14 +21,18 @@ package transport
import (
"context"
"crypto"
+ "encoding/json"
"fmt"
+ "net/url"
"sync"
"github.com/go-fed/httpsig"
"github.com/spf13/viper"
"github.com/superseriousbusiness/activity/pub"
+ "github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
)
// Controller generates transports for use in making federation requests to other servers.
@@ -42,19 +46,65 @@ type controller struct {
clock pub.Clock
client pub.HttpClient
appAgent string
+
+ // dereferenceFollowersShortcut is a shortcut to dereference followers of an
+ // account on this instance, without making any external api/http calls.
+ //
+ // It is passed to new transports, and should only be invoked when the iri.Host == this host.
+ dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
+
+ // dereferenceUserShortcut is a shortcut to dereference followers an account on
+ // this instance, without making any external api/http calls.
+ //
+ // It is passed to new transports, and should only be invoked when the iri.Host == this host.
+ dereferenceUserShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
+}
+
+func dereferenceFollowersShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) {
+ return func(ctx context.Context, iri *url.URL) ([]byte, error) {
+ followers, err := federatingDB.Followers(ctx, iri)
+ if err != nil {
+ return nil, err
+ }
+
+ i, err := streams.Serialize(followers)
+ if err != nil {
+ return nil, err
+ }
+
+ return json.Marshal(i)
+ }
+}
+
+func dereferenceUserShortcut(federatingDB federatingdb.DB) func(context.Context, *url.URL) ([]byte, error) {
+ return func(ctx context.Context, iri *url.URL) ([]byte, error) {
+ user, err := federatingDB.Get(ctx, iri)
+ if err != nil {
+ return nil, err
+ }
+
+ i, err := streams.Serialize(user)
+ if err != nil {
+ return nil, err
+ }
+
+ return json.Marshal(i)
+ }
}
// NewController returns an implementation of the Controller interface for creating new transports
-func NewController(db db.DB, clock pub.Clock, client pub.HttpClient) Controller {
+func NewController(db db.DB, federatingDB federatingdb.DB, clock pub.Clock, client pub.HttpClient) Controller {
applicationName := viper.GetString(config.Keys.ApplicationName)
host := viper.GetString(config.Keys.Host)
appAgent := fmt.Sprintf("%s %s", applicationName, host)
return &controller{
- db: db,
- clock: clock,
- client: client,
- appAgent: appAgent,
+ db: db,
+ clock: clock,
+ client: client,
+ appAgent: appAgent,
+ dereferenceFollowersShortcut: dereferenceFollowersShortcut(federatingDB),
+ dereferenceUserShortcut: dereferenceUserShortcut(federatingDB),
}
}
@@ -78,15 +128,17 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (T
sigTransport := pub.NewHttpSigTransport(c.client, c.appAgent, c.clock, getSigner, postSigner, pubKeyID, privkey)
return &transport{
- client: c.client,
- appAgent: c.appAgent,
- gofedAgent: "(go-fed/activity v1.0.0)",
- clock: c.clock,
- pubKeyID: pubKeyID,
- privkey: privkey,
- sigTransport: sigTransport,
- getSigner: getSigner,
- getSignerMu: &sync.Mutex{},
+ client: c.client,
+ appAgent: c.appAgent,
+ gofedAgent: "(go-fed/activity v1.0.0)",
+ clock: c.clock,
+ pubKeyID: pubKeyID,
+ privkey: privkey,
+ sigTransport: sigTransport,
+ getSigner: getSigner,
+ getSignerMu: &sync.Mutex{},
+ dereferenceFollowersShortcut: c.dereferenceFollowersShortcut,
+ dereferenceUserShortcut: c.dereferenceUserShortcut,
}, nil
}
diff --git a/internal/transport/deliver.go b/internal/transport/deliver.go
@@ -23,6 +23,8 @@ import (
"net/url"
"github.com/sirupsen/logrus"
+ "github.com/spf13/viper"
+ "github.com/superseriousbusiness/gotosocial/internal/config"
)
func (t *transport) BatchDeliver(ctx context.Context, b []byte, recipients []*url.URL) error {
@@ -30,6 +32,11 @@ func (t *transport) BatchDeliver(ctx context.Context, b []byte, recipients []*ur
}
func (t *transport) Deliver(ctx context.Context, b []byte, to *url.URL) error {
+ // if the 'to' host is our own, just skip this delivery since we by definition already have the message!
+ if to.Host == viper.GetString(config.Keys.Host) {
+ return nil
+ }
+
l := logrus.WithField("func", "Deliver")
l.Debugf("performing POST to %s", to.String())
return t.sigTransport.Deliver(ctx, b, to)
diff --git a/internal/transport/dereference.go b/internal/transport/dereference.go
@@ -23,10 +23,29 @@ import (
"net/url"
"github.com/sirupsen/logrus"
+ "github.com/spf13/viper"
+ "github.com/superseriousbusiness/gotosocial/internal/config"
+ "github.com/superseriousbusiness/gotosocial/internal/uris"
)
func (t *transport) Dereference(ctx context.Context, iri *url.URL) ([]byte, error) {
l := logrus.WithField("func", "Dereference")
+
+ // if the request is to us, we can shortcut for certain URIs rather than going through
+ // the normal request flow, thereby saving time and energy
+ if iri.Host == viper.GetString(config.Keys.Host) {
+ if uris.IsFollowersPath(iri) {
+ // the request is for followers of one of our accounts, which we can shortcut
+ return t.dereferenceFollowersShortcut(ctx, iri)
+ }
+
+ if uris.IsUserPath(iri) {
+ // the request is for one of our accounts, which we can shortcut
+ return t.dereferenceUserShortcut(ctx, iri)
+ }
+ }
+
+ // the request is either for a remote host or for us but we don't have a shortcut, so continue as normal
l.Debugf("performing GET to %s", iri.String())
return t.sigTransport.Dereference(ctx, iri)
}
diff --git a/internal/transport/transport.go b/internal/transport/transport.go
@@ -30,8 +30,11 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
-// Transport wraps the pub.Transport interface with some additional
-// functionality for fetching remote media.
+// Transport wraps the pub.Transport interface with some additional functionality for fetching remote media.
+//
+// Since the transport has the concept of 'shortcuts' for fetching data locally rather than remotely, it is
+// not *always* the case that calling a Transport function does an http call, but it usually will for remote
+// hosts or resources for which a shortcut isn't provided by the transport controller (also in this package).
type Transport interface {
pub.Transport
// DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize.
@@ -40,6 +43,8 @@ type Transport interface {
DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error)
// Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body.
Finger(ctx context.Context, targetUsername string, targetDomains string) ([]byte, error)
+ // SigTransport returns the underlying http signature transport wrapped by the GoToSocial transport.
+ SigTransport() pub.Transport
}
// transport implements the Transport interface
@@ -53,4 +58,13 @@ type transport struct {
sigTransport *pub.HttpSigTransport
getSigner httpsig.Signer
getSignerMu *sync.Mutex
+
+ // shortcuts for dereferencing things that exist on our instance without making an http call to ourself
+
+ dereferenceFollowersShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
+ dereferenceUserShortcut func(ctx context.Context, iri *url.URL) ([]byte, error)
+}
+
+func (t *transport) SigTransport() pub.Transport {
+ return t.sigTransport
}
diff --git a/testrig/testmodels.go b/testrig/testmodels.go
@@ -1749,8 +1749,8 @@ func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry
panic(err)
}
- // trigger the delivery function, which will trigger the 'do' function of the recorder above
- if err := tp.Deliver(context.Background(), bytes, destination); err != nil {
+ // trigger the delivery function for the underlying signature transport, which will trigger the 'do' function of the recorder above
+ if err := tp.SigTransport().Deliver(context.Background(), bytes, destination); err != nil {
panic(err)
}
@@ -1781,8 +1781,8 @@ func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest
panic(err)
}
- // trigger the delivery function, which will trigger the 'do' function of the recorder above
- if _, err := tp.Dereference(context.Background(), destination); err != nil {
+ // trigger the dereference function for the underlying signature transport, which will trigger the 'do' function of the recorder above
+ if _, err := tp.SigTransport().Dereference(context.Background(), destination); err != nil {
panic(err)
}
diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go
@@ -39,7 +39,7 @@ import (
// PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
// basis.
func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller {
- return transport.NewController(db, &federation.Clock{}, client)
+ return transport.NewController(db, NewTestFederatingDB(db), &federation.Clock{}, client)
}
// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,
