commit 95715f9251da56bfb7a8c7c48ca5bf38d2392f03
parent 65b19411a4185877ac30694e0c169c12d2998c07
Author: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Thu, 9 Feb 2023 09:27:07 +0100
[performance] Don't fetch avatar + header if uri hasn't changed (#1463)
Diffstat:
2 files changed, 73 insertions(+), 163 deletions(-)
diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go
@@ -36,6 +36,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
+ "github.com/superseriousbusiness/gotosocial/internal/transport"
)
func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) {
@@ -145,9 +146,14 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
}
}
+ transport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
+ if err != nil {
+ return nil, fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
+ }
+
if account.Username != "" {
// A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info.
- accDomain, accURI, err := d.fingerRemoteAccount(ctx, requestUser, account.Username, account.Domain)
+ accDomain, accURI, err := d.fingerRemoteAccount(ctx, transport, account.Username, account.Domain)
if err != nil && account.URI == "" {
// this is a new account (to us) with username@domain but failed
@@ -185,7 +191,7 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
defer d.stopHandshake(requestUser, uri)
// Dereference this account to get the latest available.
- apubAcc, err := d.dereferenceAccountable(ctx, requestUser, uri)
+ apubAcc, err := d.dereferenceAccountable(ctx, transport, uri)
if err != nil {
return nil, fmt.Errorf("enrichAccount: error dereferencing account %s: %w", uri, err)
}
@@ -202,7 +208,7 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
// No username was provided, so no webfinger was attempted earlier.
//
// Now we have a username we can attempt it now, this ensures up-to-date accountdomain info.
- accDomain, _, err := d.fingerRemoteAccount(ctx, requestUser, latestAcc.Username, uri.Host)
+ accDomain, _, err := d.fingerRemoteAccount(ctx, transport, latestAcc.Username, uri.Host)
if err == nil {
// Update account with latest info.
@@ -214,9 +220,32 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
latestAcc.ID = account.ID
latestAcc.FetchedAt = time.Now()
- // Fetch latest account media (TODO: check for changed URI to previous).
- if err = d.fetchRemoteAccountMedia(ctx, latestAcc, requestUser, block); err != nil {
- log.Errorf("error fetching remote media for account %s: %v", uri, err)
+ // Fetch latest account avatar only if remote URI has changed
+ if latestAcc.AvatarRemoteURL != "" && latestAcc.AvatarRemoteURL != account.AvatarRemoteURL {
+ d.dereferencingAvatarsLock.Lock()
+ newAvatarID, err := d.fetchRemoteAccountMedia(ctx, transport, latestAcc.AvatarRemoteURL, latestAcc.ID, d.dereferencingAvatars, true, false)
+ d.dereferencingAvatarsLock.Unlock()
+ if err != nil {
+ log.Errorf("error fetching remote avatar for account %s: %v", uri, err)
+ } else {
+ latestAcc.AvatarMediaAttachmentID = newAvatarID
+ }
+ } else {
+ latestAcc.AvatarMediaAttachmentID = account.AvatarMediaAttachmentID // no change / empty url
+ }
+
+ // Fetch latest account header only if remote URI has changed
+ if latestAcc.AvatarRemoteURL != "" && latestAcc.AvatarRemoteURL != account.AvatarRemoteURL {
+ d.dereferencingHeadersLock.Lock()
+ newHeaderID, err := d.fetchRemoteAccountMedia(ctx, transport, latestAcc.HeaderRemoteURL, latestAcc.ID, d.dereferencingHeaders, false, true)
+ d.dereferencingHeadersLock.Unlock()
+ if err != nil {
+ log.Errorf("error fetching remote header for account %s: %v", uri, err)
+ } else {
+ latestAcc.HeaderMediaAttachmentID = newHeaderID
+ }
+ } else {
+ latestAcc.HeaderMediaAttachmentID = account.HeaderMediaAttachmentID // no change / empty url
}
// Fetch the latest remote account emoji IDs used in account display name/bio.
@@ -258,12 +287,7 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
// it finds as something that an account model can be constructed out of.
//
// Will work for Person, Application, or Service models.
-func (d *deref) dereferenceAccountable(ctx context.Context, username string, remoteAccountID *url.URL) (ap.Accountable, error) {
- transport, err := d.transportController.NewTransportForUsername(ctx, username)
- if err != nil {
- return nil, fmt.Errorf("DereferenceAccountable: transport err: %w", err)
- }
-
+func (d *deref) dereferenceAccountable(ctx context.Context, transport transport.Transport, remoteAccountID *url.URL) (ap.Accountable, error) {
b, err := transport.Dereference(ctx, remoteAccountID)
if err != nil {
return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %w", remoteAccountID.String(), err)
@@ -279,7 +303,7 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %w", err)
}
- //nolint shutup linter
+ //nolint:forcetypeassert
switch t.GetTypeName() {
case ap.ActorApplication:
return t.(vocab.ActivityStreamsApplication), nil
@@ -296,156 +320,47 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
return nil, newErrWrongType(fmt.Errorf("DereferenceAccountable: type name %s not supported as Accountable", t.GetTypeName()))
}
-// fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account,
-// using a transport on behalf of requestingUsername.
-//
-// The returned boolean indicates whether anything changed -- in other words, whether the
-// account should be updated in the database.
-//
-// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
-//
-// If refresh is true, then the media will be fetched again even if it's already been fetched before.
-//
-// If blocking is true, then the calls to the media manager made by this function will be blocking:
-// in other words, the function won't return until the header and the avatar have been fully processed.
-func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) error {
- // Fetch a transport beforehand for either(or both) avatar / header dereferencing.
- tsport, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
+func (d *deref) fetchRemoteAccountMedia(
+ ctx context.Context,
+ transport transport.Transport,
+ mediaRemoteURL string,
+ targetAccountID string,
+ dereferencingMap map[string]*media.ProcessingMedia,
+ avatar bool,
+ header bool,
+) (string, error) {
+ // first check if we're already processing this media
+ if alreadyProcessing, ok := dereferencingMap[targetAccountID]; ok {
+ // we're already on it, nothing else to do
+ return alreadyProcessing.AttachmentID(), nil
+ }
+
+ avatarIRI, err := url.Parse(mediaRemoteURL)
if err != nil {
- return fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err)
+ return "", err
}
- if targetAccount.AvatarRemoteURL != "" {
- var processingMedia *media.ProcessingMedia
-
- // Parse the target account's avatar URL into URL object.
- avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL)
- if err != nil {
- return err
- }
-
- d.dereferencingAvatarsLock.Lock() // LOCK HERE
- // first check if we're already processing this media
- if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok {
- // we're already on it, no worries
- processingMedia = alreadyProcessing
- } else {
- data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
- return tsport.DereferenceMedia(innerCtx, avatarIRI)
- }
-
- avatar := true
- newProcessing, err := d.mediaManager.ProcessMedia(ctx, data, nil, targetAccount.ID, &media.AdditionalMediaInfo{
- RemoteURL: &targetAccount.AvatarRemoteURL,
- Avatar: &avatar,
- })
- if err != nil {
- d.dereferencingAvatarsLock.Unlock()
- return err
- }
-
- // store it in our map to indicate it's in process
- d.dereferencingAvatars[targetAccount.ID] = newProcessing
- processingMedia = newProcessing
- }
- d.dereferencingAvatarsLock.Unlock() // UNLOCK HERE
-
- load := func(innerCtx context.Context) error {
- _, err := processingMedia.LoadAttachment(innerCtx)
- return err
- }
-
- cleanup := func() {
- d.dereferencingAvatarsLock.Lock()
- delete(d.dereferencingAvatars, targetAccount.ID)
- d.dereferencingAvatarsLock.Unlock()
- }
-
- // block until loaded if required...
- if blocking {
- if err := loadAndCleanup(ctx, load, cleanup); err != nil {
- return err
- }
- } else {
- // ...otherwise do it async
- go func() {
- dlCtx, done := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
- if err := loadAndCleanup(dlCtx, load, cleanup); err != nil {
- log.Errorf("fetchRemoteAccountMedia: error during async lock and load of avatar: %s", err)
- }
- done()
- }()
- }
-
- targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID()
+ data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
+ return transport.DereferenceMedia(innerCtx, avatarIRI)
}
- if targetAccount.HeaderRemoteURL != "" {
- var processingMedia *media.ProcessingMedia
-
- // Parse the target account's header URL into URL object.
- headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL)
- if err != nil {
- return err
- }
-
- d.dereferencingHeadersLock.Lock() // LOCK HERE
- // first check if we're already processing this media
- if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok {
- // we're already on it, no worries
- processingMedia = alreadyProcessing
- } else {
- data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
- return tsport.DereferenceMedia(innerCtx, headerIRI)
- }
-
- header := true
- newProcessing, err := d.mediaManager.ProcessMedia(ctx, data, nil, targetAccount.ID, &media.AdditionalMediaInfo{
- RemoteURL: &targetAccount.HeaderRemoteURL,
- Header: &header,
- })
- if err != nil {
- d.dereferencingAvatarsLock.Unlock()
- return err
- }
-
- // store it in our map to indicate it's in process
- d.dereferencingHeaders[targetAccount.ID] = newProcessing
- processingMedia = newProcessing
- }
- d.dereferencingHeadersLock.Unlock() // UNLOCK HERE
-
- load := func(innerCtx context.Context) error {
- _, err := processingMedia.LoadAttachment(innerCtx)
- return err
- }
-
- cleanup := func() {
- d.dereferencingHeadersLock.Lock()
- delete(d.dereferencingHeaders, targetAccount.ID)
- d.dereferencingHeadersLock.Unlock()
- }
-
- // block until loaded if required...
- if blocking {
- if err := loadAndCleanup(ctx, load, cleanup); err != nil {
- return err
- }
- } else {
- // ...otherwise do it async
- go func() {
- dlCtx, done := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
- if err := loadAndCleanup(dlCtx, load, cleanup); err != nil {
- log.Errorf("fetchRemoteAccountMedia: error during async lock and load of header: %s", err)
- }
- done()
- }()
- }
+ processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, nil, targetAccountID, &media.AdditionalMediaInfo{
+ RemoteURL: &mediaRemoteURL,
+ Avatar: &avatar,
+ Header: &header,
+ })
+ if err != nil {
+ return "", err
+ }
- targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID()
+ // store it in our map to indicate it's in process
+ dereferencingMap[targetAccountID] = processingMedia
+ defer delete(dereferencingMap, targetAccountID)
+ if _, err := processingMedia.LoadAttachment(ctx); err != nil {
+ return "", err
}
- return nil
+ return processingMedia.AttachmentID(), nil
}
func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) {
diff --git a/internal/federation/dereferencing/finger.go b/internal/federation/dereferencing/finger.go
@@ -27,17 +27,12 @@ import (
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
+ "github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
-func (d *deref) fingerRemoteAccount(ctx context.Context, username string, targetUsername string, targetHost string) (accountDomain string, accountURI *url.URL, err error) {
- t, err := d.transportController.NewTransportForUsername(ctx, username)
- if err != nil {
- err = fmt.Errorf("fingerRemoteAccount: error getting transport for %s: %s", username, err)
- return
- }
-
- b, err := t.Finger(ctx, targetUsername, targetHost)
+func (d *deref) fingerRemoteAccount(ctx context.Context, transport transport.Transport, targetUsername string, targetHost string) (accountDomain string, accountURI *url.URL, err error) {
+ b, err := transport.Finger(ctx, targetUsername, targetHost)
if err != nil {
err = fmt.Errorf("fingerRemoteAccount: error fingering @%s@%s: %s", targetUsername, targetHost, err)
return
