commit dfdc473cefa255c5d0fb6c2e90e2b90815305c2f parent 694a49058951de31cca4ea061e2c08d44e712612 Author: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Sat, 11 Jun 2022 11:01:34 +0200 [chore] Webfinger rework (#627) * move finger to dereferencer * totally break GetRemoteAccount * start reworking finger func a bit * start reworking getRemoteAccount a bit * move mention parts to namestring * rework webfingerget * use util function to extract webfinger parts * use accountDomain * rework finger again, final form * just a real nasty commit, the worst * remove refresh from account * use new ASRepToAccount signature * fix incorrect debug call * fix for new getRemoteAccount * rework GetRemoteAccount * start updating tests to remove repetition * break a lot of tests Move shared test logic into the testrig, rather than having it scattered all over the place. This allows us to just mock the transport controller once, and have all tests use it (unless they need not to for some other reason). * fix up tests to use main mock httpclient * webfinger only if necessary * cheeky linting with the lads * update mentionName regex recognize instance accounts * don't finger instance accounts * test webfinger part extraction * increase default worker count to 4 per cpu * don't repeat regex parsing * final search for discovered accountDomain * be more permissive in namestring lookup * add more extraction tests * simplify GetParseMentionFunc * skip long search if local account * fix broken test Diffstat:
62 files changed, 979 insertions(+), 810 deletions(-)
diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go @@ -85,7 +85,7 @@ var Start action.GTSAction = func(ctx context.Context) error { StatusCode: 200, Body: r, }, nil - }), dbService, fedWorker) + }, ""), dbService, fedWorker) mediaManager := testrig.NewTestMediaManager(dbService, storageBackend) federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager, fedWorker) diff --git a/internal/ap/extract.go b/internal/ap/extract.go @@ -574,7 +574,7 @@ func ExtractMention(i Mentionable) (*gtsmodel.Mention, error) { } // just make sure the mention string is valid so we can handle it properly later on... - _, _, err = util.ExtractMentionParts(mentionString) + _, _, err = util.ExtractNamestringParts(mentionString) if err != nil { return nil, err } diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go @@ -67,7 +67,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go @@ -85,7 +85,7 @@ func (suite *AdminStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/client/auth/auth_test.go b/internal/api/client/auth/auth_test.go @@ -90,7 +90,7 @@ func (suite *AuthStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go @@ -82,7 +82,7 @@ func (suite *ServeFileTestSuite) SetupSuite() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage), fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker) diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go @@ -82,7 +82,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.followRequestModule = followrequest.New(suite.processor).(*followrequest.Module) diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go @@ -85,7 +85,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go @@ -91,7 +91,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go @@ -89,7 +89,7 @@ func (suite *MediaUpdateTestSuite) SetupSuite() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go @@ -19,18 +19,8 @@ package status_test import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/activity/pub" - "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/api/client/status" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -94,7 +84,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.statusModule = status.New(suite.processor).(*status.Module) @@ -104,88 +94,3 @@ func (suite *StatusStandardTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) testrig.StandardStorageTeardown(suite.storage) } - -func (suite *StatusStandardTestSuite) testHttpClient() pub.HttpClient { - remoteAccount := suite.testAccounts["remote_account_1"] - remoteAccountNamestring := fmt.Sprintf("acct:%s@%s", remoteAccount.Username, remoteAccount.Domain) - remoteAccountWebfingerURI := fmt.Sprintf("https://%s/.well-known/webfinger?resource=%s", remoteAccount.Domain, remoteAccountNamestring) - - fmt.Println(remoteAccountWebfingerURI) - - httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - // respond correctly to a webfinger lookup - if req.URL.String() == remoteAccountWebfingerURI { - responseJson := fmt.Sprintf(` - { - "subject": "%s", - "aliases": [ - "%s", - "%s" - ], - "links": [ - { - "rel": "http://webfinger.net/rel/profile-page", - "type": "text/html", - "href": "%s" - }, - { - "rel": "self", - "type": "application/activity+json", - "href": "%s" - } - ] - }`, remoteAccountNamestring, remoteAccount.URI, remoteAccount.URL, remoteAccount.URL, remoteAccount.URI) - responseType := "application/json" - - reader := bytes.NewReader([]byte(responseJson)) - readCloser := io.NopCloser(reader) - response := &http.Response{ - StatusCode: 200, - Body: readCloser, - ContentLength: int64(len(responseJson)), - Header: http.Header{ - "content-type": {responseType}, - }, - } - return response, nil - } - - // respond correctly to an account dereference - if req.URL.String() == remoteAccount.URI { - satanAS, err := suite.tc.AccountToAS(context.Background(), remoteAccount) - if err != nil { - panic(err) - } - - satanI, err := streams.Serialize(satanAS) - if err != nil { - panic(err) - } - satanJson, err := json.Marshal(satanI) - if err != nil { - panic(err) - } - responseType := "application/activity+json" - - reader := bytes.NewReader(satanJson) - readCloser := io.NopCloser(reader) - response := &http.Response{ - StatusCode: 200, - Body: readCloser, - ContentLength: int64(len(satanJson)), - Header: http.Header{ - "content-type": {responseType}, - }, - } - return response, nil - } - - r := ioutil.NopCloser(bytes.NewReader([]byte{})) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - }) - - return httpClient -} diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go @@ -128,7 +128,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() { ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil) // the endpoint we're hitting ctx.Request.Header.Set("accept", "application/json") ctx.Request.Form = url.Values{ - "status": {"hello @foss_satan@fossbros-anonymous.io"}, + "status": {"hello @brand_new_person@unknown-instance.com"}, "visibility": {string(model.VisibilityPublic)}, } suite.statusModule.StatusCreatePOSTHandler(ctx) @@ -145,7 +145,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() { suite.NoError(err) // if the status is properly formatted, that means the account has been put in the db - suite.Equal("<p>hello <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span></p>", statusReply.Content) + suite.Equal(`<p>hello <span class="h-card"><a href="https://unknown-instance.com/@brand_new_person" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>brand_new_person</span></a></span></p>`, statusReply.Content) suite.Equal(model.VisibilityPublic, statusReply.Visibility) } diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go @@ -69,7 +69,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.storage = testrig.NewTestStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/s2s/user/inboxpost_test.go b/internal/api/s2s/user/inboxpost_test.go @@ -88,7 +88,7 @@ func (suite *InboxPostTestSuite) TestPostBlock() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -191,7 +191,7 @@ func (suite *InboxPostTestSuite) TestPostUnblock() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -284,7 +284,7 @@ func (suite *InboxPostTestSuite) TestPostUpdate() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -406,7 +406,7 @@ func (suite *InboxPostTestSuite) TestPostDelete() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/s2s/user/outboxget_test.go b/internal/api/s2s/user/outboxget_test.go @@ -49,7 +49,7 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -107,7 +107,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -165,7 +165,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go @@ -52,7 +52,7 @@ func (suite *RepliesGetTestSuite) TestGetReplies() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -116,7 +116,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -183,7 +183,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/s2s/user/statusget_test.go b/internal/api/s2s/user/statusget_test.go @@ -51,7 +51,7 @@ func (suite *StatusGetTestSuite) TestGetStatus() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -119,7 +119,7 @@ func (suite *StatusGetTestSuite) TestGetStatusLowercase() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go @@ -85,7 +85,7 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.userModule = user.New(suite.processor).(*user.Module) diff --git a/internal/api/s2s/user/userget_test.go b/internal/api/s2s/user/userget_test.go @@ -52,7 +52,7 @@ func (suite *UserGetTestSuite) TestGetUser() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -102,7 +102,7 @@ func (suite *UserGetTestSuite) TestGetUser() { // convert person to account // since this account is already known, we should get a pretty full model of it from the conversion - a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, false) + a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false) suite.NoError(err) suite.EqualValues(targetAccount.Username, a.Username) } @@ -133,7 +133,7 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() { clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker) federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) @@ -182,7 +182,7 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() { suite.True(ok) // convert person to account - a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, false) + a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false) suite.NoError(err) suite.EqualValues(targetAccount.Username, a.Username) } diff --git a/internal/api/s2s/webfinger/webfinger_test.go b/internal/api/s2s/webfinger/webfinger_test.go @@ -88,7 +88,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) diff --git a/internal/api/s2s/webfinger/webfingerget.go b/internal/api/s2s/webfinger/webfingerget.go @@ -22,13 +22,13 @@ import ( "context" "fmt" "net/http" - "strings" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/api" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/util" ) // WebfingerGETRequest swagger:operation GET /.well-known/webfinger webfingerGet @@ -67,35 +67,19 @@ func (m *Module) WebfingerGETRequest(c *gin.Context) { return } - // remove the acct: prefix if it's present - trimAcct := strings.TrimPrefix(resourceQuery, "acct:") - // remove the first @ in @whatever@example.org if it's present - namestring := strings.TrimPrefix(trimAcct, "@") - - // at this point we should have a string like some_user@example.org - l.Debugf("got finger request for '%s'", namestring) - - usernameAndAccountDomain := strings.Split(namestring, "@") - if len(usernameAndAccountDomain) != 2 { - l.Debugf("aborting request because username and domain could not be parsed from %s", namestring) - c.JSON(http.StatusBadRequest, gin.H{"error": "bad request"}) - return - } - - username := strings.ToLower(usernameAndAccountDomain[0]) - requestedAccountDomain := strings.ToLower(usernameAndAccountDomain[1]) - if username == "" || requestedAccountDomain == "" { - l.Debug("aborting request because username or domain was empty") - c.JSON(http.StatusBadRequest, gin.H{"error": "bad request"}) + requestedUsername, requestedHost, err := util.ExtractWebfingerParts(resourceQuery) + if err != nil { + l.Debugf("bad webfinger request with resource query %s: %s", resourceQuery, err) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("bad webfinger request with resource query %s", resourceQuery)}) return } accountDomain := config.GetAccountDomain() host := config.GetHost() - if requestedAccountDomain != accountDomain && requestedAccountDomain != host { - l.Debugf("aborting request because accountDomain %s does not belong to this instance", requestedAccountDomain) - c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("accountDomain %s does not belong to this instance", requestedAccountDomain)}) + if requestedHost != host && requestedHost != accountDomain { + l.Debugf("aborting request because requestedHost %s does not belong to this instance", requestedHost) + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("requested host %s does not belong to this instance", requestedHost)}) return } @@ -106,7 +90,7 @@ func (m *Module) WebfingerGETRequest(c *gin.Context) { ctx = context.WithValue(ctx, ap.ContextRequestingPublicKeyVerifier, verifier) } - resp, errWithCode := m.processor.GetWebfingerAccount(ctx, username) + resp, errWithCode := m.processor.GetWebfingerAccount(ctx, requestedUsername) if errWithCode != nil { api.ErrorHandler(c, errWithCode, m.processor.InstanceGet) return diff --git a/internal/concurrency/workers.go b/internal/concurrency/workers.go @@ -45,7 +45,7 @@ func NewWorkerPool[MsgType any](workers int, queueRatio int) *WorkerPool[MsgType if workers < 1 { // ensure sensible workers - workers = runtime.GOMAXPROCS(0) * 2 + workers = runtime.GOMAXPROCS(0) * 4 } if queueRatio < 1 { // ensure sensible ratio diff --git a/internal/federation/dereference.go b/internal/federation/dereference.go @@ -23,11 +23,12 @@ import ( "net/url" "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (f *federator) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) { - return f.dereferencer.GetRemoteAccount(ctx, username, remoteAccountID, blocking, refresh) +func (f *federator) GetRemoteAccount(ctx context.Context, params dereferencing.GetRemoteAccountParams) (*gtsmodel.Account, error) { + return f.dereferencer.GetRemoteAccount(ctx, params) } func (f *federator) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) { diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go @@ -33,12 +33,15 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" ) +var webfingerInterval = -48 * time.Hour // 2 days in the past + func instanceAccount(account *gtsmodel.Account) bool { return strings.EqualFold(account.Username, account.Domain) || account.FollowersURI == "" || @@ -46,97 +49,238 @@ func instanceAccount(account *gtsmodel.Account) bool { (account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor")) } +// GetRemoteAccountParams wraps parameters for a remote account lookup. +type GetRemoteAccountParams struct { + // The username of the user doing the lookup request (optional). + // If not set, then the GtS instance account will be used to do the lookup. + RequestingUsername string + // The ActivityPub URI of the remote account (optional). + // If not set (nil), the ActivityPub URI of the remote account will be discovered + // via webfinger, so you must set RemoteAccountUsername and RemoteAccountHost + // if this parameter is not set. + RemoteAccountID *url.URL + // The username of the remote account (optional). + // If RemoteAccountID is not set, then this value must be set. + RemoteAccountUsername string + // The host of the remote account (optional). + // If RemoteAccountID is not set, then this value must be set. + RemoteAccountHost string + // Whether to do a blocking call to the remote instance. If true, + // then the account's media and other fields will be fully dereferenced before it is returned. + // If false, then the account's media and other fields will be dereferenced in the background, + // so only a minimal account representation will be returned by GetRemoteAccount. + Blocking bool + // Whether to skip making calls to remote instances. This is useful when you want to + // quickly fetch a remote account from the database or fail, and don't want to cause + // http requests to go flying around. + SkipResolve bool +} + // GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account, -// puts it in the database, and returns it to a caller. -// -// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling -// the remote instance again. Blocking indicates whether the function should block until processing of -// the fetched account is complete. -// -// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true). -func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) { - new := true - - // check if we already have the account in our db, and just return it unless we'd doing a refresh - remoteAccount, err := d.db.GetAccountByURI(ctx, remoteAccountID.String()) - if err == nil { - new = false - if !refresh { - // make sure the account fields are populated before returning: - // even if we're not doing a refresh, the caller might want to block - // until everything is loaded - changed, err := d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking) +// puts or updates it in the database (if necessary), and returns it to a caller. +func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountParams) (remoteAccount *gtsmodel.Account, err error) { + + /* + In this function we want to retrieve a gtsmodel representation of a remote account, with its proper + accountDomain set, while making as few calls to remote instances as possible to save time and bandwidth. + + There are a few different paths through this function, and the path taken depends on how much + initial information we are provided with via parameters, how much information we already have stored, + and what we're allowed to do according to the parameters we've been passed. + + Scenario 1: We're not allowed to resolve remotely, but we've got either the account URI or the + account username + host, so we can check in our database and return if possible. + + Scenario 2: We are allowed to resolve remotely, and we have an account URI but no username or host. + In this case, we can use the URI to resolve the remote account and find the username, + and then we can webfinger the account to discover the accountDomain if necessary. + + Scenario 3: We are allowed to resolve remotely, and we have the username and host but no URI. + In this case, we can webfinger the account to discover the URI, and then dereference + from that. + */ + + // first check if we can retrieve the account locally just with what we've been given + switch { + case params.RemoteAccountID != nil: + // try with uri + if a, dbErr := d.db.GetAccountByURI(ctx, params.RemoteAccountID.String()); dbErr == nil { + remoteAccount = a + } else if dbErr != db.ErrNoEntries { + err = fmt.Errorf("GetRemoteAccount: database error looking for account %s: %s", params.RemoteAccountID, err) + } + case params.RemoteAccountUsername != "" && params.RemoteAccountHost != "": + // try with username/host + a := >smodel.Account{} + where := []db.Where{{Key: "username", Value: params.RemoteAccountUsername}, {Key: "domain", Value: params.RemoteAccountHost}} + if dbErr := d.db.GetWhere(ctx, where, a); dbErr == nil { + remoteAccount = a + } else if dbErr != db.ErrNoEntries { + err = fmt.Errorf("GetRemoteAccount: database error looking for account with username %s and host %s: %s", params.RemoteAccountUsername, params.RemoteAccountHost, err) + } + default: + err = errors.New("GetRemoteAccount: no identifying parameters were set so we cannot get account") + } + + if err != nil { + return + } + + if params.SkipResolve { + // if we can't resolve, return already since there's nothing more we can do + if remoteAccount == nil { + err = errors.New("GetRemoteAccount: error retrieving account with skipResolve set true") + } + return + } + + var accountable ap.Accountable + if params.RemoteAccountUsername == "" || params.RemoteAccountHost == "" { + // try to populate the missing params + // the first one is easy ... + params.RemoteAccountHost = params.RemoteAccountID.Host + // ... but we still need the username so we can do a finger for the accountDomain + + // check if we had the account stored already and got it earlier + if remoteAccount != nil { + params.RemoteAccountUsername = remoteAccount.Username + } else { + // if we didn't already have it, we have dereference it from remote and just... + accountable, err = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err) + err = fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err) + return } - if changed { - updatedAccount, err := d.db.UpdateAccount(ctx, remoteAccount) - if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err) - } - return updatedAccount, err + // ... take the username (for now) + params.RemoteAccountUsername, err = ap.ExtractPreferredUsername(accountable) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error extracting accountable username: %s", err) + return } - - return remoteAccount, nil } } - if new { - // we haven't seen this account before: dereference it from remote - accountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID) + // if we reach this point, params.RemoteAccountHost and params.RemoteAccountUsername must be set + // params.RemoteAccountID may or may not be set, but we have enough information to fetch it if we need it + + // we finger to fetch the account domain but just in case we're not fingering, make a best guess + // already about what the account domain might be; this var will be overwritten later if necessary + var accountDomain string + switch { + case remoteAccount != nil: + accountDomain = remoteAccount.Domain + case params.RemoteAccountID != nil: + accountDomain = params.RemoteAccountID.Host + default: + accountDomain = params.RemoteAccountHost + } + + // to save on remote calls: only webfinger if we don't have a remoteAccount yet, or if we haven't + // fingered the remote account for at least 2 days; don't finger instance accounts + var fingered time.Time + if remoteAccount == nil || (remoteAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(remoteAccount)) { + accountDomain, params.RemoteAccountID, err = d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err) + err = fmt.Errorf("GetRemoteAccount: error while fingering: %s", err) + return + } + fingered = time.Now() + } + + if !fingered.IsZero() && remoteAccount == nil { + // if we just fingered and now have a discovered account domain but still no account, + // we should do a final lookup in the database with the discovered username + accountDomain + // to make absolutely sure we don't already have this account + a := >smodel.Account{} + where := []db.Where{{Key: "username", Value: params.RemoteAccountUsername}, {Key: "domain", Value: accountDomain}} + if dbErr := d.db.GetWhere(ctx, where, a); dbErr == nil { + remoteAccount = a + } else if dbErr != db.ErrNoEntries { + err = fmt.Errorf("GetRemoteAccount: database error looking for account with username %s and host %s: %s", params.RemoteAccountUsername, params.RemoteAccountHost, err) + return + } + } + + // we may also have some extra information already, like the account we had in the db, or the + // accountable representation that we dereferenced from remote + if remoteAccount == nil { + // we still don't have the account, so deference it if we didn't earlier + if accountable == nil { + accountable, err = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err) + return + } } - newAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, accountable, refresh) + // then convert + remoteAccount, err = d.typeConverter.ASRepresentationToAccount(ctx, accountable, accountDomain, false) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err) + err = fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err) + return } - ulid, err := id.NewRandomULID() + // this is a new account so we need to generate a new ID for it + var ulid string + ulid, err = id.NewRandomULID() if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err) + err = fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err) + return } - newAccount.ID = ulid + remoteAccount.ID = ulid - if _, err := d.populateAccountFields(ctx, newAccount, username, refresh, blocking); err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err) + _, err = d.populateAccountFields(ctx, remoteAccount, params.RequestingUsername, params.Blocking) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err) + return } - if err := d.db.Put(ctx, newAccount); err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error putting new account: %s", err) + remoteAccount.LastWebfingeredAt = fingered + remoteAccount.UpdatedAt = time.Now() + + err = d.db.Put(ctx, remoteAccount) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error putting new account: %s", err) + return } - return newAccount, nil + return // the new account } - // we have seen this account before, but we have to refresh it - refreshedAccountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID) - if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error dereferencing refreshedAccountable: %s", err) + // we had the account already, but now we know the account domain, so update it if it's different + if !strings.EqualFold(remoteAccount.Domain, accountDomain) { + remoteAccount.Domain = accountDomain + remoteAccount, err = d.db.UpdateAccount(ctx, remoteAccount) + if err != nil { + err = fmt.Errorf("GetRemoteAccount: error updating account: %s", err) + return + } } - refreshedAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, refreshedAccountable, refresh) + // make sure the account fields are populated before returning: + // the caller might want to block until everything is loaded + var fieldsChanged bool + fieldsChanged, err = d.populateAccountFields(ctx, remoteAccount, params.RequestingUsername, params.Blocking) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error converting refreshedAccountable to refreshedAccount: %s", err) + return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err) } - refreshedAccount.ID = remoteAccount.ID - changed, err := d.populateAccountFields(ctx, refreshedAccount, username, refresh, blocking) - if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error populating further refreshedAccount fields: %s", err) + var fingeredChanged bool + if !fingered.IsZero() { + fingeredChanged = true + remoteAccount.LastWebfingeredAt = fingered } - if changed { - updatedAccount, err := d.db.UpdateAccount(ctx, refreshedAccount) + if fieldsChanged || fingeredChanged { + remoteAccount.UpdatedAt = time.Now() + remoteAccount, err = d.db.UpdateAccount(ctx, remoteAccount) if err != nil { - return nil, fmt.Errorf("GetRemoteAccount: error updating refreshedAccount: %s", err) + return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err) } - return updatedAccount, nil } - return refreshedAccount, nil + return // the account we already had + possibly updated } // dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever @@ -209,7 +353,7 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem // populateAccountFields populates any fields on the given account that weren't populated by the initial // dereferencing. This includes things like header and avatar etc. -func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool, refresh bool) (bool, error) { +func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) { // if we're dealing with an instance account, just bail, we don't need to do anything if instanceAccount(account) { return false, nil @@ -230,7 +374,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc } // fetch the header and avatar - changed, err := d.fetchRemoteAccountMedia(ctx, account, t, refresh, blocking) + changed, err := d.fetchRemoteAccountMedia(ctx, account, t, blocking) if err != nil { return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err) } @@ -250,7 +394,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc // // If blocking is true, then the calls to the media manager made by this function will be blocking: // in other words, the function won't return until the header and the avatar have been fully processed. -func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool, refresh bool) (bool, error) { +func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool) (bool, error) { changed := false accountURI, err := url.Parse(targetAccount.URI) @@ -262,7 +406,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm return changed, fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host) } - if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) { + if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") { var processingMedia *media.ProcessingMedia d.dereferencingAvatarsLock.Lock() // LOCK HERE @@ -320,7 +464,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm changed = true } - if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) { + if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "") { var processingMedia *media.ProcessingMedia d.dereferencingHeadersLock.Lock() // LOCK HERE diff --git a/internal/federation/dereferencing/account_test.go b/internal/federation/dereferencing/account_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -35,7 +36,10 @@ func (suite *AccountTestSuite) TestDereferenceGroup() { fetchingAccount := suite.testAccounts["local_account_1"] groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group") - group, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, groupURL, false, false) + group, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{ + RequestingUsername: fetchingAccount.Username, + RemoteAccountID: groupURL, + }) suite.NoError(err) suite.NotNil(group) suite.NotNil(group) @@ -55,7 +59,10 @@ func (suite *AccountTestSuite) TestDereferenceService() { fetchingAccount := suite.testAccounts["local_account_1"] serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh") - service, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, serviceURL, false, false) + service, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{ + RequestingUsername: fetchingAccount.Username, + RemoteAccountID: serviceURL, + }) suite.NoError(err) suite.NotNil(service) suite.NotNil(service) @@ -69,6 +76,7 @@ func (suite *AccountTestSuite) TestDereferenceService() { suite.NoError(err) suite.Equal(service.ID, dbService.ID) suite.Equal(ap.ActorService, dbService.ActorType) + suite.Equal("example.org", dbService.Domain) } func TestAccountTestSuite(t *testing.T) { diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go @@ -33,7 +33,7 @@ import ( // Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances. type Dereferencer interface { - GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) + GetRemoteAccount(ctx context.Context, params GetRemoteAccountParams) (*gtsmodel.Account, error) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go @@ -19,22 +19,14 @@ package dereferencing_test import ( - "bytes" - "encoding/json" - "io" - "net/http" - "codeberg.org/gruf/go-store/kv" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/messages" - "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -66,106 +58,10 @@ func (suite *DereferencerStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() - suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaManager(suite.db, suite.storage)) + suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)), testrig.NewTestMediaManager(suite.db, suite.storage)) testrig.StandardDBSetup(suite.db, nil) } func (suite *DereferencerStandardTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) } - -// mockTransportController returns basically a miniature muxer, which returns a different -// value based on the request URL. It can be used to return remote statuses, profiles, etc, -// as though they were actually being dereferenced. If the URL doesn't correspond to any person -// or note or attachment that we have stored, then just a 200 code will be returned, with an empty body. -func (suite *DereferencerStandardTestSuite) mockTransportController() transport.Controller { - do := func(req *http.Request) (*http.Response, error) { - logrus.Debugf("received request for %s", req.URL) - - responseBytes := []byte{} - responseType := "" - responseLength := 0 - - if note, ok := suite.testRemoteStatuses[req.URL.String()]; ok { - // the request is for a note that we have stored - noteI, err := streams.Serialize(note) - if err != nil { - panic(err) - } - noteJson, err := json.Marshal(noteI) - if err != nil { - panic(err) - } - responseBytes = noteJson - responseType = "application/activity+json" - } - - if person, ok := suite.testRemotePeople[req.URL.String()]; ok { - // the request is for a person that we have stored - personI, err := streams.Serialize(person) - if err != nil { - panic(err) - } - personJson, err := json.Marshal(personI) - if err != nil { - panic(err) - } - responseBytes = personJson - responseType = "application/activity+json" - } - - if group, ok := suite.testRemoteGroups[req.URL.String()]; ok { - // the request is for a person that we have stored - groupI, err := streams.Serialize(group) - if err != nil { - panic(err) - } - groupJson, err := json.Marshal(groupI) - if err != nil { - panic(err) - } - responseBytes = groupJson - responseType = "application/activity+json" - } - - if service, ok := suite.testRemoteServices[req.URL.String()]; ok { - serviceI, err := streams.Serialize(service) - if err != nil { - panic(err) - } - serviceJson, err := json.Marshal(serviceI) - if err != nil { - panic(err) - } - responseBytes = serviceJson - responseType = "application/activity+json" - } - - if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok { - responseBytes = attachment.Data - responseType = attachment.ContentType - } - - if len(responseBytes) != 0 { - // we found something, so print what we're going to return - logrus.Debugf("returning response %s", string(responseBytes)) - } - responseLength = len(responseBytes) - - reader := bytes.NewReader(responseBytes) - readCloser := io.NopCloser(reader) - response := &http.Response{ - StatusCode: 200, - Body: readCloser, - ContentLength: int64(responseLength), - Header: http.Header{ - "content-type": {responseType}, - }, - } - - return response, nil - } - fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - mockClient := testrig.NewMockHTTPClient(do) - return testrig.NewTestTransportController(mockClient, suite.db, fedWorker) -} diff --git a/internal/federation/dereferencing/finger.go b/internal/federation/dereferencing/finger.go @@ -0,0 +1,80 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package dereferencing + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +func (d *deref) fingerRemoteAccount(ctx context.Context, username string, targetUsername string, targetHost string) (accountDomain string, accountURI *url.URL, err error) { + t, err := d.transportController.NewTransportForUsername(ctx, username) + if err != nil { + err = fmt.Errorf("fingerRemoteAccount: error getting transport for %s: %s", username, err) + return + } + + b, err := t.Finger(ctx, targetUsername, targetHost) + if err != nil { + err = fmt.Errorf("fingerRemoteAccount: error fingering @%s@%s: %s", targetUsername, targetHost, err) + return + } + + resp := &apimodel.WellKnownResponse{} + if err = json.Unmarshal(b, resp); err != nil { + err = fmt.Errorf("fingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse while dereferencing @%s@%s: %s", targetUsername, targetHost, err) + return + } + + if len(resp.Links) == 0 { + err = fmt.Errorf("fingerRemoteAccount: no links found in webfinger response %s", string(b)) + return + } + + if resp.Subject == "" { + err = fmt.Errorf("fingerRemoteAccount: no subject found in webfinger response %s", string(b)) + return + } + + _, accountDomain, err = util.ExtractWebfingerParts(resp.Subject) + if err != nil { + err = fmt.Errorf("fingerRemoteAccount: error extracting webfinger subject parts: %s", err) + } + + // look through the links for the first one that matches what we need + for _, l := range resp.Links { + if l.Rel == "self" && (strings.EqualFold(l.Type, "application/activity+json") || strings.EqualFold(l.Type, "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")) { + if uri, thiserr := url.Parse(l.Href); thiserr == nil && (uri.Scheme == "http" || uri.Scheme == "https") { + // found it! + accountURI = uri + return + } + } + } + + err = errors.New("fingerRemoteAccount: no match found in webfinger response") + return +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go @@ -85,7 +85,10 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat return nil, nil, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err) } - _, err = d.GetRemoteAccount(ctx, username, accountURI, true, false) + _, err = d.GetRemoteAccount(ctx, GetRemoteAccountParams{ + RequestingUsername: username, + RemoteAccountID: accountURI, + }) if err != nil { return nil, nil, fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err) } @@ -316,7 +319,10 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta if targetAccount == nil { // we didn't find the account in our database already // check if we can get the account remotely (dereference it) - if a, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false, false); err != nil { + if a, err := d.GetRemoteAccount(ctx, GetRemoteAccountParams{ + RequestingUsername: requestingUsername, + RemoteAccountID: targetAccountURI, + }); err != nil { errs = append(errs, err.Error()) } else { logrus.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID) diff --git a/internal/federation/federatingactor_test.go b/internal/federation/federatingactor_test.go @@ -19,10 +19,7 @@ package federation_test import ( - "bytes" "context" - "io/ioutil" - "net/http" "net/url" "testing" "time" @@ -60,15 +57,9 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) // setup transport controller with a no-op client so we don't make external calls - sentMessages := []*url.URL{} - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - sentMessages = append(sentMessages, req.URL) - r := ioutil.NopCloser(bytes.NewReader([]byte{})) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -77,7 +68,7 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() { suite.NotNil(activity) // because zork has no remote followers, sent messages should be empty (no messages sent to own instance) - suite.Empty(sentMessages) + suite.Empty(httpClient.SentMessages) } func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { @@ -87,8 +78,8 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { err := suite.db.Put(ctx, >smodel.Follow{ ID: "01G1TRWV4AYCDBX5HRWT2EVBCV", - CreatedAt: time.Now(), - UpdatedAt: time.Now(), + CreatedAt: testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), + UpdatedAt: testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), AccountID: testRemoteAccount.ID, TargetAccountID: testAccount.ID, ShowReblogs: true, @@ -100,7 +91,7 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { testNote := testrig.NewAPNote( testrig.URLMustParse("http://localhost:8080/users/the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"), testrig.URLMustParse("http://localhost:8080/@the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"), - time.Now(), + testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), "boobies", "", testrig.URLMustParse(testAccount.URI), @@ -110,20 +101,12 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { nil, nil, ) - testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), time.Now(), testNote) + testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), testNote) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - // setup transport controller with a no-op client so we don't make external calls - sentMessages := []*url.URL{} - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - sentMessages = append(sentMessages, req.URL) - r := ioutil.NopCloser(bytes.NewReader([]byte{})) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -132,8 +115,10 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() { suite.NotNil(activity) // because we added 1 remote follower for zork, there should be a url in sentMessage - suite.Len(sentMessages, 1) - suite.Equal(testRemoteAccount.InboxURI, sentMessages[0].String()) + suite.Len(httpClient.SentMessages, 1) + msg, ok := httpClient.SentMessages[testRemoteAccount.InboxURI] + suite.True(ok) + suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","actor":"http://localhost:8080/users/the_mighty_zork","id":"http://localhost:8080/whatever_some_create","object":{"attributedTo":"http://localhost:8080/users/the_mighty_zork","content":"boobies","id":"http://localhost:8080/users/the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5","published":"2022-06-02T12:22:21+02:00","tag":[],"to":"http://localhost:8080/users/the_mighty_zork/followers","type":"Note","url":"http://localhost:8080/@the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"},"published":"2022-06-02T12:22:21+02:00","to":"http://localhost:8080/users/the_mighty_zork/followers","type":"Create"}`, string(msg)) } func TestFederatingActorTestSuite(t *testing.T) { diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go @@ -119,7 +119,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { accountable = i } - updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, true) + updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "", true) if err != nil { return fmt.Errorf("UPDATE: error converting to account: %s", err) } diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go @@ -31,6 +31,7 @@ import ( "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/uris" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -197,7 +198,10 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr } } - requestingAccount, err := f.GetRemoteAccount(ctx, username, publicKeyOwnerURI, false, false) + requestingAccount, err := f.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: username, + RemoteAccountID: publicKeyOwnerURI, + }) if err != nil { return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err) } diff --git a/internal/federation/federatingprotocol_test.go b/internal/federation/federatingprotocol_test.go @@ -45,10 +45,8 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook1() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - // setup transport controller with a no-op client so we don't make external calls - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - return nil, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -78,10 +76,9 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook2() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - // setup transport controller with a no-op client so we don't make external calls - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - return nil, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -112,10 +109,9 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook3() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - // setup transport controller with a no-op client so we don't make external calls - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - return nil, nil - }), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + // setup module being tested federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -148,7 +144,9 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + // now setup module being tested, with the mock transport controller federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) @@ -186,7 +184,8 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() { func (suite *FederatingProtocolTestSuite) TestBlocked1() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) sendingAccount := suite.testAccounts["remote_account_1"] @@ -208,7 +207,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked1() { func (suite *FederatingProtocolTestSuite) TestBlocked2() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) sendingAccount := suite.testAccounts["remote_account_1"] @@ -241,7 +241,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked2() { func (suite *FederatingProtocolTestSuite) TestBlocked3() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) sendingAccount := suite.testAccounts["remote_account_1"] @@ -277,7 +278,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked3() { func (suite *FederatingProtocolTestSuite) TestBlocked4() { fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media") + tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker) federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage)) sendingAccount := suite.testAccounts["remote_account_1"] diff --git a/internal/federation/federator.go b/internal/federation/federator.go @@ -53,14 +53,10 @@ type Federator interface { // If something goes wrong during authentication, nil, false, and an error will be returned. AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, gtserror.WithCode) - // FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that - // account, or an error if it doesn't exist or can't be retrieved. - FingerRemoteAccount(ctx context.Context, requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) - DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL) error DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error - GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) + GetRemoteAccount(ctx context.Context, params dereferencing.GetRemoteAccountParams) (*gtsmodel.Account, error) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error) diff --git a/internal/federation/finger.go b/internal/federation/finger.go @@ -1,72 +0,0 @@ -/* - GoToSocial - Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ - -package federation - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/url" - "strings" - - apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" -) - -func (f *federator) FingerRemoteAccount(ctx context.Context, requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) { - if blocked, err := f.db.IsDomainBlocked(ctx, targetDomain); blocked || err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain) - } - - t, err := f.transportController.NewTransportForUsername(ctx, requestingUsername) - if err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) - } - - b, err := t.Finger(ctx, targetUsername, targetDomain) - if err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: error doing request on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) - } - - resp := &apimodel.WellKnownResponse{} - if err := json.Unmarshal(b, resp); err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) - } - - if len(resp.Links) == 0 { - return nil, fmt.Errorf("FingerRemoteAccount: no links found in webfinger response %s", string(b)) - } - - // look through the links for the first one that matches "application/activity+json", this is what we need - for _, l := range resp.Links { - if strings.EqualFold(l.Type, "application/activity+json") { - if l.Href == "" || l.Rel != "self" { - continue - } - accountURI, err := url.Parse(l.Href) - if err != nil { - return nil, fmt.Errorf("FingerRemoteAccount: couldn't parse url %s: %s", l.Href, err) - } - // found it! - return accountURI, nil - } - } - - return nil, errors.New("FingerRemoteAccount: no match found in webfinger response") -} diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go @@ -23,7 +23,6 @@ import ( "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -48,7 +47,6 @@ type AccountStandardTestSuite struct { mediaManager media.Manager oauthServer oauth.Server fromClientAPIChan chan messages.FromClientAPI - httpClient pub.HttpClient transportController transport.Controller federator federation.Federator emailSender email.Sender @@ -97,8 +95,7 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) - suite.httpClient = testrig.NewMockHTTPClient(nil) - suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker) + suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker) suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) diff --git a/internal/processing/account/get.go b/internal/processing/account/get.go @@ -26,6 +26,7 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -80,7 +81,13 @@ func (p *processor) getAccountFor(ctx context.Context, requestingAccount *gtsmod return nil, gtserror.NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err)) } - a, err := p.federator.GetRemoteAccount(ctx, requestingAccount.Username, targetAccountURI, true, false) + a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: requestingAccount.Username, + RemoteAccountID: targetAccountURI, + RemoteAccountHost: targetAccount.Domain, + RemoteAccountUsername: targetAccount.Username, + Blocking: true, + }) if err == nil { targetAccount = a } diff --git a/internal/processing/account_test.go b/internal/processing/account_test.go @@ -60,7 +60,7 @@ func (suite *AccountTestSuite) TestAccountDeleteLocal() { time.Sleep(1 * time.Second) // wait a sec for the delete to process // the delete should be federated outwards to the following account's inbox - sent, ok := suite.sentHTTPRequests[followingAccount.InboxURI] + sent, ok := suite.httpClient.SentMessages[followingAccount.InboxURI] suite.True(ok) delete := &struct { Actor string `json:"actor"` diff --git a/internal/processing/federation/getfollowers.go b/internal/processing/federation/getfollowers.go @@ -24,6 +24,7 @@ import ( "net/url" "github.com/superseriousbusiness/activity/streams" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" ) @@ -40,7 +41,10 @@ func (p *processor) GetFollowers(ctx context.Context, requestedUsername string, return nil, errWithCode } - requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: requestedUsername, + RemoteAccountID: requestingAccountURI, + }) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getfollowing.go b/internal/processing/federation/getfollowing.go @@ -24,6 +24,7 @@ import ( "net/url" "github.com/superseriousbusiness/activity/streams" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" ) @@ -40,7 +41,10 @@ func (p *processor) GetFollowing(ctx context.Context, requestedUsername string, return nil, errWithCode } - requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: requestedUsername, + RemoteAccountID: requestingAccountURI, + }) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getoutbox.go b/internal/processing/federation/getoutbox.go @@ -25,6 +25,7 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" ) @@ -41,7 +42,10 @@ func (p *processor) GetOutbox(ctx context.Context, requestedUsername string, pag return nil, errWithCode } - requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: requestedUsername, + RemoteAccountID: requestingAccountURI, + }) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getstatus.go b/internal/processing/federation/getstatus.go @@ -24,6 +24,7 @@ import ( "net/url" "github.com/superseriousbusiness/activity/streams" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" ) @@ -40,7 +41,10 @@ func (p *processor) GetStatus(ctx context.Context, requestedUsername string, req return nil, errWithCode } - requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: requestedUsername, + RemoteAccountID: requestingAccountURI, + }) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getstatusreplies.go b/internal/processing/federation/getstatusreplies.go @@ -25,6 +25,7 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) @@ -42,7 +43,10 @@ func (p *processor) GetStatusReplies(ctx context.Context, requestedUsername stri return nil, errWithCode } - requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: requestedUsername, + RemoteAccountID: requestingAccountURI, + }) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/federation/getuser.go b/internal/processing/federation/getuser.go @@ -25,6 +25,7 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -52,7 +53,10 @@ func (p *processor) GetUser(ctx context.Context, requestedUsername string, reque // if we're not already handshaking/dereferencing a remote account, dereference it now if !p.federator.Handshaking(ctx, requestedUsername, requestingAccountURI) { - requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false) + requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: requestedUsername, + RemoteAccountID: requestingAccountURI, + }) if err != nil { return nil, gtserror.NewErrorUnauthorized(err) } diff --git a/internal/processing/followrequest_test.go b/internal/processing/followrequest_test.go @@ -57,7 +57,7 @@ func (suite *FollowRequestTestSuite) TestFollowRequestAccept() { time.Sleep(1 * time.Second) // accept should be sent to some_user - sent, ok := suite.sentHTTPRequests[requestingAccount.InboxURI] + sent, ok := suite.httpClient.SentMessages[requestingAccount.InboxURI] suite.True(ok) accept := &struct { @@ -109,7 +109,7 @@ func (suite *FollowRequestTestSuite) TestFollowRequestReject() { time.Sleep(1 * time.Second) // reject should be sent to some_user - sent, ok := suite.sentHTTPRequests[requestingAccount.InboxURI] + sent, ok := suite.httpClient.SentMessages[requestingAccount.InboxURI] suite.True(ok) reject := &struct { diff --git a/internal/processing/fromfederator.go b/internal/processing/fromfederator.go @@ -26,6 +26,7 @@ import ( "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/messages" @@ -130,7 +131,11 @@ func (p *processor) processCreateStatusFromFederator(ctx context.Context, federa return err } - a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false) + a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: federatorMsg.ReceivingAccount.Username, + RemoteAccountID: remoteAccountID, + Blocking: true, + }) if err != nil { return err } @@ -172,7 +177,11 @@ func (p *processor) processCreateFaveFromFederator(ctx context.Context, federato return err } - a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false) + a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: federatorMsg.ReceivingAccount.Username, + RemoteAccountID: remoteAccountID, + Blocking: true, + }) if err != nil { return err } @@ -210,7 +219,11 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context, return err } - a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false) + a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: federatorMsg.ReceivingAccount.Username, + RemoteAccountID: remoteAccountID, + Blocking: true, + }) if err != nil { return err } @@ -267,7 +280,11 @@ func (p *processor) processCreateAnnounceFromFederator(ctx context.Context, fede return err } - a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false) + a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: federatorMsg.ReceivingAccount.Username, + RemoteAccountID: remoteAccountID, + Blocking: true, + }) if err != nil { return err } @@ -332,7 +349,11 @@ func (p *processor) processUpdateAccountFromFederator(ctx context.Context, feder return err } - if _, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, incomingAccountURL, false, true); err != nil { + if _, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: federatorMsg.ReceivingAccount.Username, + RemoteAccountID: incomingAccountURL, + Blocking: true, + }); err != nil { return fmt.Errorf("error enriching updated account from federator: %s", err) } diff --git a/internal/processing/fromfederator_test.go b/internal/processing/fromfederator_test.go @@ -423,7 +423,7 @@ func (suite *FromFederatorTestSuite) TestProcessFollowRequestLocked() { suite.Equal(originAccount.ID, notif.Account.ID) // no messages should have been sent out, since we didn't need to federate an accept - suite.Empty(suite.sentHTTPRequests) + suite.Empty(suite.httpClient.SentMessages) } func (suite *FromFederatorTestSuite) TestProcessFollowRequestUnlocked() { @@ -474,8 +474,8 @@ func (suite *FromFederatorTestSuite) TestProcessFollowRequestUnlocked() { suite.Equal(originAccount.ID, notif.Account.ID) // an accept message should be sent to satan's inbox - suite.Len(suite.sentHTTPRequests, 1) - acceptBytes := suite.sentHTTPRequests[originAccount.InboxURI] + suite.Len(suite.httpClient.SentMessages, 1) + acceptBytes := suite.httpClient.SentMessages[originAccount.InboxURI] accept := &struct { Actor string `json:"actor"` ID string `json:"id"` diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go @@ -19,12 +19,7 @@ package media_test import ( - "bytes" - "io" - "net/http" - "codeberg.org/gruf/go-store/kv" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -79,7 +74,7 @@ func (suite *MediaStandardTestSuite) SetupTest() { suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.transportController = suite.mockTransportController() + suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)) suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") @@ -89,40 +84,3 @@ func (suite *MediaStandardTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) testrig.StandardStorageTeardown(suite.storage) } - -func (suite *MediaStandardTestSuite) mockTransportController() transport.Controller { - do := func(req *http.Request) (*http.Response, error) { - logrus.Debugf("received request for %s", req.URL) - - responseBytes := []byte{} - responseType := "" - responseLength := 0 - - if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok { - responseBytes = attachment.Data - responseType = attachment.ContentType - } - - if len(responseBytes) != 0 { - // we found something, so print what we're going to return - logrus.Debugf("returning response %s", string(responseBytes)) - } - responseLength = len(responseBytes) - - reader := bytes.NewReader(responseBytes) - readCloser := io.NopCloser(reader) - response := &http.Response{ - StatusCode: 200, - Body: readCloser, - ContentLength: int64(responseLength), - Header: http.Header{ - "content-type": {responseType}, - }, - } - - return response, nil - } - fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - mockClient := testrig.NewMockHTTPClient(do) - return testrig.NewTestTransportController(mockClient, suite.db, fedWorker) -} diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go @@ -19,16 +19,8 @@ package processing_test import ( - "bytes" - "context" - "encoding/json" - "io" - "io/ioutil" - "net/http" - "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" - "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -50,6 +42,7 @@ type ProcessingStandardTestSuite struct { storage *kv.KVStore mediaManager media.Manager typeconverter typeutils.TypeConverter + httpClient *testrig.MockHTTPClient transportController transport.Controller federator federation.Federator oauthServer oauth.Server @@ -69,8 +62,6 @@ type ProcessingStandardTestSuite struct { testBlocks map[string]*gtsmodel.Block testActivities map[string]testrig.ActivityWithSignature - sentHTTPRequests map[string][]byte - processor processing.Processor } @@ -102,123 +93,12 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { suite.testActivities = testrig.NewTestActivities(suite.testAccounts) suite.storage = testrig.NewTestStorage() suite.typeconverter = testrig.NewTestTypeConverter(suite.db) - - // make an http client that stores POST requests it receives into a map, - // and also responds to correctly to dereference requests - suite.sentHTTPRequests = make(map[string][]byte) - httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - if req.Method == http.MethodPost && req.Body != nil { - requestBytes, err := ioutil.ReadAll(req.Body) - if err != nil { - panic(err) - } - if err := req.Body.Close(); err != nil { - panic(err) - } - suite.sentHTTPRequests[req.URL.String()] = requestBytes - } - - if req.URL.String() == suite.testAccounts["remote_account_1"].URI { - // the request is for remote account 1 - satan := suite.testAccounts["remote_account_1"] - - satanAS, err := suite.typeconverter.AccountToAS(context.Background(), satan) - if err != nil { - panic(err) - } - - satanI, err := streams.Serialize(satanAS) - if err != nil { - panic(err) - } - satanJson, err := json.Marshal(satanI) - if err != nil { - panic(err) - } - responseType := "application/activity+json" - - reader := bytes.NewReader(satanJson) - readCloser := io.NopCloser(reader) - response := &http.Response{ - StatusCode: 200, - Body: readCloser, - ContentLength: int64(len(satanJson)), - Header: http.Header{ - "content-type": {responseType}, - }, - } - return response, nil - } - - if req.URL.String() == suite.testAccounts["remote_account_2"].URI { - // the request is for remote account 2 - someAccount := suite.testAccounts["remote_account_2"] - - someAccountAS, err := suite.typeconverter.AccountToAS(context.Background(), someAccount) - if err != nil { - panic(err) - } - - someAccountI, err := streams.Serialize(someAccountAS) - if err != nil { - panic(err) - } - someAccountJson, err := json.Marshal(someAccountI) - if err != nil { - panic(err) - } - responseType := "application/activity+json" - - reader := bytes.NewReader(someAccountJson) - readCloser := io.NopCloser(reader) - response := &http.Response{ - StatusCode: 200, - Body: readCloser, - ContentLength: int64(len(someAccountJson)), - Header: http.Header{ - "content-type": {responseType}, - }, - } - return response, nil - } - - if req.URL.String() == "http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1" { - // the request is for the forwarded message - message := suite.testActivities["forwarded_message"].Activity.GetActivityStreamsObject().At(0).GetActivityStreamsNote() - messageI, err := streams.Serialize(message) - if err != nil { - panic(err) - } - messageJson, err := json.Marshal(messageI) - if err != nil { - panic(err) - } - responseType := "application/activity+json" - - reader := bytes.NewReader(messageJson) - readCloser := io.NopCloser(reader) - response := &http.Response{ - StatusCode: 200, - Body: readCloser, - ContentLength: int64(len(messageJson)), - Header: http.Header{ - "content-type": {responseType}, - }, - } - return response, nil - } - - r := ioutil.NopCloser(bytes.NewReader([]byte{})) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - }) + suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media") clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1) - suite.transportController = testrig.NewTestTransportController(httpClient, suite.db, fedWorker) + suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker) suite.oauthServer = testrig.NewTestOauthServer(suite.db) diff --git a/internal/processing/search.go b/internal/processing/search.go @@ -20,6 +20,7 @@ package processing import ( "context" + "errors" "fmt" "net/url" "strings" @@ -28,53 +29,70 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/util" ) -func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode) { +func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, search *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode) { l := logrus.WithFields(logrus.Fields{ "func": "SearchGet", - "query": searchQuery.Query, + "query": search.Query, }) - results := &apimodel.SearchResult{ + // tidy up the query and make sure it wasn't just spaces + query := strings.TrimSpace(search.Query) + if query == "" { + err := errors.New("search query was empty string after trimming space") + return nil, gtserror.NewErrorBadRequest(err, err.Error()) + } + + searchResult := &apimodel.SearchResult{ Accounts: []apimodel.Account{}, Statuses: []apimodel.Status{}, Hashtags: []apimodel.Tag{}, } foundAccounts := []*gtsmodel.Account{} foundStatuses := []*gtsmodel.Status{} - // foundHashtags := []*gtsmodel.Tag{} - - // convert the query to lowercase and trim leading/trailing spaces - query := strings.ToLower(strings.TrimSpace(searchQuery.Query)) var foundOne bool - // check if the query is something like @whatever_username@example.org -- this means it's a remote account - if _, domain, err := util.ExtractMentionParts(searchQuery.Query); err == nil && domain != "" { - l.Debug("search term is a mention, looking it up...") - foundAccount, err := p.searchAccountByMention(ctx, authed, searchQuery.Query, searchQuery.Resolve) - if err == nil && foundAccount != nil { + + /* + SEARCH BY MENTION + check if the query is something like @whatever_username@example.org -- this means it's a remote account + */ + maybeNamestring := query + if maybeNamestring[0] != '@' { + maybeNamestring = "@" + maybeNamestring + } + + if username, domain, err := util.ExtractNamestringParts(maybeNamestring); err == nil { + l.Debugf("search term %s is a mention, looking it up...", maybeNamestring) + if foundAccount, err := p.searchAccountByMention(ctx, authed, username, domain, search.Resolve); err == nil && foundAccount != nil { foundAccounts = append(foundAccounts, foundAccount) foundOne = true l.Debug("got an account by searching by mention") + } else if err != nil { + l.Debugf("error looking up account %s: %s", maybeNamestring, err) } } - // check if the query is a URI with a recognizable scheme and just do a lookup for that, straight up + /* + SEARCH BY URI + check if the query is a URI with a recognizable scheme and dereference it + */ if !foundOne { if uri, err := url.Parse(query); err == nil && (uri.Scheme == "https" || uri.Scheme == "http") { // 1. check if it's a status - if foundStatus, err := p.searchStatusByURI(ctx, authed, uri, searchQuery.Resolve); err == nil && foundStatus != nil { + if foundStatus, err := p.searchStatusByURI(ctx, authed, uri, search.Resolve); err == nil && foundStatus != nil { foundStatuses = append(foundStatuses, foundStatus) l.Debug("got a status by searching by URI") } // 2. check if it's an account - if foundAccount, err := p.searchAccountByURI(ctx, authed, uri, searchQuery.Resolve); err == nil && foundAccount != nil { + if foundAccount, err := p.searchAccountByURI(ctx, authed, uri, search.Resolve); err == nil && foundAccount != nil { foundAccounts = append(foundAccounts, foundAccount) l.Debug("got an account by searching by URI") } @@ -90,7 +108,7 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQue if blocked, err := p.db.IsBlocked(ctx, authed.Account.ID, foundAccount.ID, true); err == nil && !blocked { // all good, convert it and add it to the results if apiAcct, err := p.tc.AccountToAPIAccountPublic(ctx, foundAccount); err == nil && apiAcct != nil { - results.Accounts = append(results.Accounts, *apiAcct) + searchResult.Accounts = append(searchResult.Accounts, *apiAcct) } } } @@ -105,10 +123,10 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQue continue } - results.Statuses = append(results.Statuses, *apiStatus) + searchResult.Statuses = append(searchResult.Statuses, *apiStatus) } - return results, nil + return searchResult, nil } func (p *processor) searchStatusByURI(ctx context.Context, authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) { @@ -147,7 +165,10 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth, if resolve { // we don't have it locally so try and dereference it - account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, uri, true, true) + account, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: authed.Account.Username, + RemoteAccountID: uri, + }) if err != nil { return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err) } @@ -156,18 +177,14 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth, return nil, nil } -func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, mention string, resolve bool) (*gtsmodel.Account, error) { - // query is for a remote account - username, domain, err := util.ExtractMentionParts(mention) - if err != nil { - return nil, fmt.Errorf("searchAccountByMention: error extracting mention parts: %s", err) - } +func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) { + maybeAcct := >smodel.Account{} + var err error // if it's a local account we can skip a whole bunch of stuff - maybeAcct := >smodel.Account{} - if domain == config.GetHost() { + if domain == config.GetHost() || domain == config.GetAccountDomain() || domain == "" { maybeAcct, err = p.db.GetLocalAccountByUsername(ctx, username) - if err != nil { + if err != nil && err != db.ErrNoEntries { return nil, fmt.Errorf("searchAccountByMention: error getting local account by username: %s", err) } return maybeAcct, nil @@ -191,22 +208,15 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au // we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it if resolve { - // we're allowed to resolve it so let's try - // first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account - acctURI, err := p.federator.FingerRemoteAccount(ctx, authed.Account.Username, username, domain) + maybeAcct, err = p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: authed.Account.Username, + RemoteAccountUsername: username, + RemoteAccountHost: domain, + }) if err != nil { - // something went wrong doing the webfinger lookup so we can't process the request - return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err) - } - - if acctURI.Scheme == "https" || acctURI.Scheme == "http" { - acct, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true) - if err != nil { - logrus.Debugf("could not get remote account by mention %s with uri %s: %s", mention, acctURI, err) - return nil, err - } - return acct, nil + return nil, fmt.Errorf("searchAccountByMention: error getting remote account: %s", err) } + return maybeAcct, nil } return nil, nil diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go @@ -80,7 +80,7 @@ func (suite *StatusStandardTestSuite) SetupTest() { suite.db = testrig.NewTestDB() suite.typeConverter = testrig.NewTestTypeConverter(suite.db) suite.clientWorker = concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1) - suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker) diff --git a/internal/processing/util.go b/internal/processing/util.go @@ -21,12 +21,14 @@ package processing import ( "context" "fmt" - "strings" + "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/util" ) func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.ParseMentionFunc { @@ -37,98 +39,35 @@ func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel. return nil, fmt.Errorf("couldn't get mention origin account with id %s", originAccountID) } - // A mentioned account looks like "@test@example.org" or just "@test" for a local account - // -- we can guarantee this from the regex that targetAccounts should have been derived from. - // But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given). - - // 1. trim off the first @ - trimmed := strings.TrimPrefix(targetAccount, "@") - - // 2. split the username and domain - split := strings.Split(trimmed, "@") - - // 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong - - var local bool - switch len(split) { - case 1: - local = true - case 2: - local = false - default: - return nil, fmt.Errorf("mentioned account format '%s' was not valid", targetAccount) - } - - var username, domain string - username = split[0] - if !local { - domain = split[1] - } - - // 4. check we now have a proper username and domain - if username == "" || (!local && domain == "") { - return nil, fmt.Errorf("username or domain for '%s' was nil", targetAccount) + username, domain, err := util.ExtractNamestringParts(targetAccount) + if err != nil { + return nil, fmt.Errorf("couldn't extract namestring parts from %s: %s", targetAccount, err) } var mentionedAccount *gtsmodel.Account - - if local { + if domain == "" || domain == config.GetHost() || domain == config.GetAccountDomain() { localAccount, err := dbConn.GetLocalAccountByUsername(ctx, username) if err != nil { return nil, err } mentionedAccount = localAccount } else { - remoteAccount := >smodel.Account{} - - where := []db.Where{ - { - Key: "username", - Value: username, - CaseInsensitive: true, - }, - { - Key: "domain", - Value: domain, - CaseInsensitive: true, - }, + var requestingUsername string + if originAccount.Domain == "" { + requestingUsername = originAccount.Username + } + remoteAccount, err := federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{ + RequestingUsername: requestingUsername, + RemoteAccountUsername: username, + RemoteAccountHost: domain, + }) + if err != nil { + return nil, fmt.Errorf("error dereferencing account: %s", err) } - err := dbConn.GetWhere(ctx, where, remoteAccount) - if err == nil { - // the account was already in the database - mentionedAccount = remoteAccount - } else { - // we couldn't get it from the database - if err != db.ErrNoEntries { - // a serious error has happened so bail - return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err) - } - - // We just don't have the account, so try webfingering it. - // - // If the mention originates from our instance we should use the username of the origin account to do the dereferencing, - // otherwise we should just use our instance account (that is, provide an empty string), since obviously we can't use - // a remote account to do remote dereferencing! - var fingeringUsername string - if originAccount.Domain == "" { - fingeringUsername = originAccount.Username - } - - acctURI, err := federator.FingerRemoteAccount(ctx, fingeringUsername, username, domain) - if err != nil { - // something went wrong doing the webfinger lookup so we can't process the request - return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err) - } - - resolvedAccount, err := federator.GetRemoteAccount(ctx, fingeringUsername, acctURI, true, true) - if err != nil { - return nil, fmt.Errorf("error dereferencing account with uri %s: %s", acctURI.String(), err) - } + // we were able to resolve it! + mentionedAccount = remoteAccount - // we were able to resolve it! - mentionedAccount = resolvedAccount - } } mentionID, err := id.NewRandomULID() diff --git a/internal/regexes/regexes.go b/internal/regexes/regexes.go @@ -61,7 +61,7 @@ var ( return rgx }() - mentionName = `^@(\w+)(?:@([a-zA-Z0-9_\-\.:]+))?$` + mentionName = `^@([\w\-\.]+)(?:@([\w\-\.:]+))?$` // MentionName captures the username and domain part from a mention string // such as @whatever_user@example.org, returning whatever_user and example.org (without the @ symbols) MentionName = regexp.MustCompile(mentionName) diff --git a/internal/transport/derefinstance.go b/internal/transport/derefinstance.go @@ -206,7 +206,7 @@ func dereferenceByNodeInfo(c context.Context, t *transport, iri *url.URL) (*gtsm // see if there's a 'name' in the map if name, present := v["name"]; present { // name could be just a username, or could be a mention string eg @whatever@aaaa.com - username, _, err := util.ExtractMentionParts(name) + username, _, err := util.ExtractNamestringParts(name) if err == nil { // it was a mention string contactAccountUsername = username diff --git a/internal/typeutils/astointernal.go b/internal/typeutils/astointernal.go @@ -30,7 +30,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" ) -func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, update bool) (*gtsmodel.Account, error) { +func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error) { // first check if we actually already know this account uriProp := accountable.GetJSONLDId() if uriProp == nil || !uriProp.IsIRI() { @@ -63,7 +63,11 @@ func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable a acct.Username = username // Domain - acct.Domain = uri.Host + if accountDomain != "" { + acct.Domain = accountDomain + } else { + acct.Domain = uri.Host + } // avatar aka icon // if this one isn't extractable in a format we recognise we'll just skip it diff --git a/internal/typeutils/astointernal_test.go b/internal/typeutils/astointernal_test.go @@ -38,7 +38,7 @@ type ASToInternalTestSuite struct { func (suite *ASToInternalTestSuite) TestParsePerson() { testPerson := suite.testPeople["https://unknown-instance.com/users/brand_new_person"] - acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, false) + acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "", false) suite.NoError(err) suite.Equal("https://unknown-instance.com/users/brand_new_person", acct.URI) @@ -106,7 +106,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() { rep, ok := t.(ap.Accountable) suite.True(ok) - acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, false) + acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false) suite.NoError(err) fmt.Printf("%+v", acct) @@ -168,7 +168,7 @@ func (suite *ASToInternalTestSuite) TestParseOwncastService() { rep, ok := t.(ap.Accountable) suite.True(ok) - acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, false) + acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false) suite.NoError(err) suite.Equal("rgh", acct.Username) diff --git a/internal/typeutils/converter.go b/internal/typeutils/converter.go @@ -99,7 +99,10 @@ type TypeConverter interface { // If update is false, and the account is already known in the database, then the existing account entry will be returned. // If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account // will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile. - ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, update bool) (*gtsmodel.Account, error) + // + // If accountDomain is set (not an empty string) then this value will be used as the account's Domain. If not set, + // then the Host of the accountable's AP ID will be used instead. + ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error) // ASStatus converts a remote activitystreams 'status' representation into a gts model status. ASStatusToStatus(ctx context.Context, statusable ap.Statusable) (*gtsmodel.Status, error) // ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request. diff --git a/internal/util/namestring.go b/internal/util/namestring.go @@ -0,0 +1,59 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package util + +import ( + "fmt" + "strings" + + "github.com/superseriousbusiness/gotosocial/internal/regexes" +) + +// ExtractNamestringParts extracts the username test_user and +// the domain example.org from a string like @test_user@example.org. +// +// If nothing is matched, it will return an error. +func ExtractNamestringParts(mention string) (username, host string, err error) { + matches := regexes.MentionName.FindStringSubmatch(mention) + switch len(matches) { + case 2: + return matches[1], "", nil + case 3: + return matches[1], matches[2], nil + default: + return "", "", fmt.Errorf("couldn't match mention %s", mention) + } +} + +// ExtractWebfingerParts returns username test_user and +// domain example.org from a string like acct:test_user@example.org, +// or acct:@test_user@example.org. +// +// If nothing is extracted, it will return an error. +func ExtractWebfingerParts(webfinger string) (username, host string, err error) { + // remove the acct: prefix if it's present + webfinger = strings.TrimPrefix(webfinger, "acct:") + + // prepend an @ if necessary + if webfinger[0] != '@' { + webfinger = "@" + webfinger + } + + return ExtractNamestringParts(webfinger) +} diff --git a/internal/util/namestring_test.go b/internal/util/namestring_test.go @@ -0,0 +1,118 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package util_test + +import ( + "testing" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +type NamestringSuite struct { + suite.Suite +} + +func (suite *NamestringSuite) TestExtractWebfingerParts1() { + webfinger := "acct:stonerkitty.monster@stonerkitty.monster" + username, host, err := util.ExtractWebfingerParts(webfinger) + suite.NoError(err) + + suite.Equal("stonerkitty.monster", username) + suite.Equal("stonerkitty.monster", host) +} + +func (suite *NamestringSuite) TestExtractWebfingerParts2() { + webfinger := "@stonerkitty.monster@stonerkitty.monster" + username, host, err := util.ExtractWebfingerParts(webfinger) + suite.NoError(err) + + suite.Equal("stonerkitty.monster", username) + suite.Equal("stonerkitty.monster", host) +} + +func (suite *NamestringSuite) TestExtractWebfingerParts3() { + webfinger := "acct:someone@somewhere" + username, host, err := util.ExtractWebfingerParts(webfinger) + suite.NoError(err) + + suite.Equal("someone", username) + suite.Equal("somewhere", host) +} + +func (suite *NamestringSuite) TestExtractWebfingerParts4() { + webfinger := "@stoner-kitty.monster@stonerkitty.monster" + username, host, err := util.ExtractWebfingerParts(webfinger) + suite.NoError(err) + + suite.Equal("stoner-kitty.monster", username) + suite.Equal("stonerkitty.monster", host) +} + +func (suite *NamestringSuite) TestExtractWebfingerParts5() { + webfinger := "@stonerkitty.monster" + username, host, err := util.ExtractWebfingerParts(webfinger) + suite.NoError(err) + + suite.Equal("stonerkitty.monster", username) + suite.Empty(host) +} + +func (suite *NamestringSuite) TestExtractWebfingerParts6() { + webfinger := "@@stonerkitty.monster" + _, _, err := util.ExtractWebfingerParts(webfinger) + suite.EqualError(err, "couldn't match mention @@stonerkitty.monster") +} + +func (suite *NamestringSuite) TestExtractNamestringParts1() { + namestring := "@stonerkitty.monster@stonerkitty.monster" + username, host, err := util.ExtractNamestringParts(namestring) + suite.NoError(err) + + suite.Equal("stonerkitty.monster", username) + suite.Equal("stonerkitty.monster", host) +} + +func (suite *NamestringSuite) TestExtractNamestringParts2() { + namestring := "@stonerkitty.monster" + username, host, err := util.ExtractNamestringParts(namestring) + suite.NoError(err) + + suite.Equal("stonerkitty.monster", username) + suite.Empty(host) +} + +func (suite *NamestringSuite) TestExtractNamestringParts3() { + namestring := "@someone@somewhere" + username, host, err := util.ExtractWebfingerParts(namestring) + suite.NoError(err) + + suite.Equal("someone", username) + suite.Equal("somewhere", host) +} + +func (suite *NamestringSuite) TestExtractNamestringParts4() { + namestring := "" + _, _, err := util.ExtractNamestringParts(namestring) + suite.EqualError(err, "couldn't match mention ") +} + +func TestNamestringSuite(t *testing.T) { + suite.Run(t, &NamestringSuite{}) +} diff --git a/internal/util/statustools.go b/internal/util/statustools.go @@ -19,7 +19,6 @@ package util import ( - "fmt" "strings" "github.com/superseriousbusiness/gotosocial/internal/regexes" @@ -59,19 +58,3 @@ func DeriveEmojisFromText(text string) []string { } return UniqueStrings(emojis) } - -// ExtractMentionParts extracts the username test_user and the domain example.org -// from a mention string like @test_user@example.org. -// -// If nothing is matched, it will return an error. -func ExtractMentionParts(mention string) (username, domain string, err error) { - matches := regexes.MentionName.FindStringSubmatch(mention) - switch len(matches) { - case 2: - return matches[1], "", nil - case 3: - return matches[1], matches[2], nil - default: - return "", "", fmt.Errorf("couldn't match mention %s", mention) - } -} diff --git a/testrig/testmodels.go b/testrig/testmodels.go @@ -1909,6 +1909,25 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote { return map[string]vocab.ActivityStreamsNote{ + "http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1": NewAPNote( + URLMustParse("http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1"), + URLMustParse("http://example.org/@some_user/afaba698-5740-4e32-a702-af61aa543bc1"), + time.Now(), + "this is a public status, please forward it!", + "", + URLMustParse("http://example.org/users/some_user"), + []*url.URL{URLMustParse(pub.PublicActivityPubIRI)}, + nil, + false, + []vocab.ActivityStreamsMention{}, + []vocab.ActivityStreamsImage{ + newAPImage( + URLMustParse("http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpeg"), + "image/jpeg", + "trent reznor looking handsome as balls", + "LEDara58O=t5EMSOENEN9]}?aK%0"), + }, + ), "https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839": NewAPNote( URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839"), URLMustParse("https://unknown-instance.com/users/@brand_new_person/01FE4NTHKWW7THT67EF10EB839"), diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go @@ -20,10 +20,16 @@ package testrig import ( "bytes" - "io/ioutil" + "encoding/json" + "io" "net/http" + "strings" + "github.com/sirupsen/logrus" "github.com/superseriousbusiness/activity/pub" + "github.com/superseriousbusiness/activity/streams" + "github.com/superseriousbusiness/activity/streams/vocab" + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/concurrency" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" @@ -31,6 +37,9 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/transport" ) +const applicationJSON = "application/json" +const applicationActivityJSON = "application/activity+json" + // NewTestTransportController returns a test transport controller with the given http client. // // Obviously for testing purposes you should not be making actual http calls to other servers. @@ -44,33 +53,232 @@ func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *conc return transport.NewController(db, NewTestFederatingDB(db, fedWorker), &federation.Clock{}, client) } -// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface, -// but will always just execute the given `do` function, allowing responses to be mocked. +type MockHTTPClient struct { + do func(req *http.Request) (*http.Response, error) + + testRemoteStatuses map[string]vocab.ActivityStreamsNote + testRemotePeople map[string]vocab.ActivityStreamsPerson + testRemoteGroups map[string]vocab.ActivityStreamsGroup + testRemoteServices map[string]vocab.ActivityStreamsService + testRemoteAttachments map[string]RemoteAttachmentFile + + SentMessages map[string][]byte +} + +// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface. +// +// If do is nil, then a standard response set will be mocked out, which includes models stored in the +// testrig, and webfinger responses as well. // -// If 'do' is nil, then a no-op function will be used instead, that just returns status 200. +// If do is not nil, then the given do function will always be used, which allows callers +// to customize how the client is mocked. // // Note that you should never ever make ACTUAL http calls with this thing. -func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error)) pub.HttpClient { - if do == nil { - return &mockHTTPClient{ - do: func(req *http.Request) (*http.Response, error) { - r := ioutil.NopCloser(bytes.NewReader([]byte{})) - return &http.Response{ - StatusCode: 200, - Body: r, - }, nil - }, - } +func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relativeMediaPath string) *MockHTTPClient { + mockHTTPClient := &MockHTTPClient{} + + if do != nil { + mockHTTPClient.do = do + return mockHTTPClient } - return &mockHTTPClient{ - do: do, + + mockHTTPClient.testRemoteStatuses = NewTestFediStatuses() + mockHTTPClient.testRemotePeople = NewTestFediPeople() + mockHTTPClient.testRemoteGroups = NewTestFediGroups() + mockHTTPClient.testRemoteServices = NewTestFediServices() + mockHTTPClient.testRemoteAttachments = NewTestFediAttachments(relativeMediaPath) + + mockHTTPClient.SentMessages = make(map[string][]byte) + + mockHTTPClient.do = func(req *http.Request) (*http.Response, error) { + responseCode := http.StatusNotFound + responseBytes := []byte(`{"error":"404 not found"}`) + responseContentType := applicationJSON + responseContentLength := len(responseBytes) + + if req.Method == http.MethodPost { + b, err := io.ReadAll(req.Body) + if err != nil { + panic(err) + } + mockHTTPClient.SentMessages[req.URL.String()] = b + + responseCode = http.StatusOK + responseBytes = []byte(`{"ok":"accepted"}`) + responseContentType = applicationJSON + responseContentLength = len(responseBytes) + } else if strings.Contains(req.URL.String(), ".well-known/webfinger") { + responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req) + } else if note, ok := mockHTTPClient.testRemoteStatuses[req.URL.String()]; ok { + // the request is for a note that we have stored + noteI, err := streams.Serialize(note) + if err != nil { + panic(err) + } + noteJSON, err := json.Marshal(noteI) + if err != nil { + panic(err) + } + responseCode = http.StatusOK + responseBytes = noteJSON + responseContentType = applicationActivityJSON + responseContentLength = len(noteJSON) + } else if person, ok := mockHTTPClient.testRemotePeople[req.URL.String()]; ok { + // the request is for a person that we have stored + personI, err := streams.Serialize(person) + if err != nil { + panic(err) + } + personJSON, err := json.Marshal(personI) + if err != nil { + panic(err) + } + responseCode = http.StatusOK + responseBytes = personJSON + responseContentType = applicationActivityJSON + responseContentLength = len(personJSON) + } else if group, ok := mockHTTPClient.testRemoteGroups[req.URL.String()]; ok { + // the request is for a person that we have stored + groupI, err := streams.Serialize(group) + if err != nil { + panic(err) + } + groupJSON, err := json.Marshal(groupI) + if err != nil { + panic(err) + } + responseCode = http.StatusOK + responseBytes = groupJSON + responseContentType = applicationActivityJSON + responseContentLength = len(groupJSON) + } else if service, ok := mockHTTPClient.testRemoteServices[req.URL.String()]; ok { + serviceI, err := streams.Serialize(service) + if err != nil { + panic(err) + } + serviceJSON, err := json.Marshal(serviceI) + if err != nil { + panic(err) + } + responseCode = http.StatusOK + responseBytes = serviceJSON + responseContentType = applicationActivityJSON + responseContentLength = len(serviceJSON) + } else if attachment, ok := mockHTTPClient.testRemoteAttachments[req.URL.String()]; ok { + responseCode = http.StatusOK + responseBytes = attachment.Data + responseContentType = attachment.ContentType + responseContentLength = len(attachment.Data) + } + + logrus.Debugf("returning response %s", string(responseBytes)) + reader := bytes.NewReader(responseBytes) + readCloser := io.NopCloser(reader) + return &http.Response{ + StatusCode: responseCode, + Body: readCloser, + ContentLength: int64(responseContentLength), + Header: http.Header{ + "content-type": {responseContentType}, + }, + }, nil } -} -type mockHTTPClient struct { - do func(req *http.Request) (*http.Response, error) + return mockHTTPClient } -func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) { +func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) { return m.do(req) } + +func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) { + var wfr *apimodel.WellKnownResponse + + switch req.URL.String() { + case "https://unknown-instance.com/.well-known/webfinger?resource=acct:some_group@unknown-instance.com": + wfr = &apimodel.WellKnownResponse{ + Subject: "acct:some_group@unknown-instance.com", + Links: []apimodel.Link{ + { + Rel: "self", + Type: applicationActivityJSON, + Href: "https://unknown-instance.com/groups/some_group", + }, + }, + } + case "https://owncast.example.org/.well-known/webfinger?resource=acct:rgh@owncast.example.org": + wfr = &apimodel.WellKnownResponse{ + Subject: "acct:rgh@example.org", + Links: []apimodel.Link{ + { + Rel: "self", + Type: applicationActivityJSON, + Href: "https://owncast.example.org/federation/user/rgh", + }, + }, + } + case "https://unknown-instance.com/.well-known/webfinger?resource=acct:brand_new_person@unknown-instance.com": + wfr = &apimodel.WellKnownResponse{ + Subject: "acct:brand_new_person@unknown-instance.com", + Links: []apimodel.Link{ + { + Rel: "self", + Type: applicationActivityJSON, + Href: "https://unknown-instance.com/users/brand_new_person", + }, + }, + } + case "https://turnip.farm/.well-known/webfinger?resource=acct:turniplover6969@turnip.farm": + wfr = &apimodel.WellKnownResponse{ + Subject: "acct:turniplover6969@turnip.farm", + Links: []apimodel.Link{ + { + Rel: "self", + Type: applicationActivityJSON, + Href: "https://turnip.farm/users/turniplover6969", + }, + }, + } + case "https://fossbros-anonymous.io/.well-known/webfinger?resource=acct:foss_satan@fossbros-anonymous.io": + wfr = &apimodel.WellKnownResponse{ + Subject: "acct:foss_satan@fossbros-anonymous.io", + Links: []apimodel.Link{ + { + Rel: "self", + Type: applicationActivityJSON, + Href: "https://fossbros-anonymous.io/users/foss_satan", + }, + }, + } + case "https://example.org/.well-known/webfinger?resource=acct:some_user@example.org": + wfr = &apimodel.WellKnownResponse{ + Subject: "acct:some_user@example.org", + Links: []apimodel.Link{ + { + Rel: "self", + Type: applicationActivityJSON, + Href: "https://example.org/users/some_user", + }, + }, + } + } + + if wfr == nil { + logrus.Debugf("webfinger response not available for %s", req.URL) + responseCode = http.StatusNotFound + responseBytes = []byte(`{"error":"not found"}`) + responseContentType = applicationJSON + responseContentLength = len(responseBytes) + return + } + + wfrJSON, err := json.Marshal(wfr) + if err != nil { + panic(err) + } + responseCode = http.StatusOK + responseBytes = wfrJSON + responseContentType = applicationJSON + responseContentLength = len(wfrJSON) + return +}