commit 420e2fb22bc7aa4967ddadb11e444079efdf5117 parent cc5f2e98b763b76f1b0d705efcedaa2ea031ad09 Author: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Thu, 28 Apr 2022 13:23:11 +0100 replace async client API / federator msg processing with worker pools (#497) * replace async client API / federator msg processing with worker pools * appease our lord-and-saviour, the linter Diffstat:
64 files changed, 573 insertions(+), 336 deletions(-)
diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go @@ -63,6 +63,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" "github.com/superseriousbusiness/gotosocial/internal/gotosocial" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/oidc" "github.com/superseriousbusiness/gotosocial/internal/processing" @@ -70,6 +71,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/web" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // Start creates and starts a gotosocial server @@ -87,7 +89,14 @@ var Start action.GTSAction = func(ctx context.Context) error { return fmt.Errorf("error creating instance instance: %s", err) } - federatingDB := federatingdb.New(dbService) + // Create the client API and federator worker pools + // NOTE: these MUST NOT be used until they are passed to the + // processor and it is started. The reason being that the processor + // sets the Worker process functions and start the underlying pools + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + federatingDB := federatingdb.New(dbService, fedWorker) router, err := router.New(ctx, dbService) if err != nil { @@ -138,8 +147,8 @@ var Start action.GTSAction = func(ctx context.Context) error { } // create and start the message processor using the other services we've created so far - processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaManager, storage, dbService, emailSender) - if err := processor.Start(ctx); err != nil { + processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaManager, storage, dbService, emailSender, clientWorker, fedWorker) + if err := processor.Start(); err != nil { return fmt.Errorf("error starting processor: %s", err) } diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go @@ -55,8 +55,10 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger" "github.com/superseriousbusiness/gotosocial/internal/api/security" "github.com/superseriousbusiness/gotosocial/internal/gotosocial" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oidc" "github.com/superseriousbusiness/gotosocial/internal/web" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -71,6 +73,10 @@ var Start action.GTSAction = func(ctx context.Context) error { storageBackend := testrig.NewTestStorage() testrig.StandardStorageSetup(storageBackend, "./testrig/media") + // Create client API and federator worker pools + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + // build backend handlers oauthServer := testrig.NewTestOauthServer(dbService) transportController := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { @@ -79,14 +85,14 @@ var Start action.GTSAction = func(ctx context.Context) error { StatusCode: 200, Body: r, }, nil - }), dbService) + }), dbService, fedWorker) mediaManager := testrig.NewTestMediaManager(dbService, storageBackend) - federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager) + federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager, fedWorker) emailSender := testrig.NewEmailSender("./web/template/", nil) - processor := testrig.NewTestProcessor(dbService, storageBackend, federator, emailSender, mediaManager) - if err := processor.Start(ctx); err != nil { + processor := testrig.NewTestProcessor(dbService, storageBackend, federator, emailSender, mediaManager, clientWorker, fedWorker) + if err := processor.Start(); err != nil { return fmt.Errorf("error starting processor: %s", err) } diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go @@ -17,8 +17,10 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -60,13 +62,16 @@ func (suite *AccountStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.accountModule = account.New(suite.processor).(*account.Module) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go @@ -35,8 +35,10 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -78,13 +80,16 @@ func (suite *AdminStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.adminModule = admin.New(suite.processor).(*admin.Module) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go @@ -36,9 +36,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -74,12 +76,16 @@ func (suite *ServeFileTestSuite) SetupSuite() { // setup standard items testrig.InitTestConfig() testrig.InitTestLog() + + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage)) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage), fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage)) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker) suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go @@ -34,8 +34,10 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -74,12 +76,16 @@ func (suite *FollowRequestStandardTestSuite) SetupSuite() { func (suite *FollowRequestStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() + + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.followRequestModule = followrequest.New(suite.processor).(*followrequest.Module) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go @@ -43,9 +43,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -81,14 +83,18 @@ func (suite *MediaCreateTestSuite) SetupSuite() { // setup standard items testrig.InitTestConfig() testrig.InitTestLog() + + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) // setup module being tested suite.mediaModule = mediamodule.New(suite.processor).(*mediamodule.Module) diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go @@ -41,9 +41,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -79,14 +81,18 @@ func (suite *MediaUpdateTestSuite) SetupSuite() { // setup standard items testrig.InitTestConfig() testrig.InitTestLog() + + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + suite.db = testrig.NewTestDB() suite.storage = testrig.NewTestStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) suite.oauthServer = testrig.NewTestOauthServer(suite.db) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) // setup module being tested suite.mediaModule = mediamodule.New(suite.processor).(*mediamodule.Module) diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go @@ -37,8 +37,10 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -79,13 +81,17 @@ func (suite *StatusStandardTestSuite) SetupSuite() { func (suite *StatusStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() + + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.statusModule = status.New(suite.processor).(*status.Module) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") @@ -104,7 +110,6 @@ func (suite *StatusStandardTestSuite) testHttpClient() pub.HttpClient { fmt.Println(remoteAccountWebfingerURI) httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { - // respond correctly to a webfinger lookup if req.URL.String() == remoteAccountWebfingerURI { responseJson := fmt.Sprintf(` diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go @@ -27,8 +27,10 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -56,6 +58,8 @@ type UserStandardTestSuite struct { func (suite *UserStandardTestSuite) SetupTest() { testrig.InitTestLog() testrig.InitTestConfig() + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) suite.testTokens = testrig.NewTestTokens() suite.testClients = testrig.NewTestClients() suite.testApplications = testrig.NewTestApplications() @@ -65,10 +69,10 @@ func (suite *UserStandardTestSuite) SetupTest() { suite.storage = testrig.NewTestStorage() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.userModule = user.New(suite.processor).(*user.Module) testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media") diff --git a/internal/api/s2s/user/inboxpost_test.go b/internal/api/s2s/user/inboxpost_test.go @@ -36,6 +36,8 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" + "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -83,10 +85,13 @@ func (suite *InboxPostTestSuite) TestPostBlock() { suite.NoError(err) body := bytes.NewReader(bodyJson) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -183,10 +188,13 @@ func (suite *InboxPostTestSuite) TestPostUnblock() { suite.NoError(err) body := bytes.NewReader(bodyJson) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -273,10 +281,13 @@ func (suite *InboxPostTestSuite) TestPostUpdate() { suite.NoError(err) body := bytes.NewReader(bodyJson) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -392,11 +403,14 @@ func (suite *InboxPostTestSuite) TestPostDelete() { suite.NoError(err) body := bytes.NewReader(bodyJson) - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) - err = processor.Start(context.Background()) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) + err = processor.Start() suite.NoError(err) userModule := user.New(processor).(*user.Module) diff --git a/internal/api/s2s/user/outboxget_test.go b/internal/api/s2s/user/outboxget_test.go @@ -31,6 +31,8 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -44,10 +46,13 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() { signedRequest := derefRequests["foss_satan_dereference_zork_outbox"] targetAccount := suite.testAccounts["local_account_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -99,10 +104,13 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() { signedRequest := derefRequests["foss_satan_dereference_zork_outbox_first"] targetAccount := suite.testAccounts["local_account_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -154,10 +162,13 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() { signedRequest := derefRequests["foss_satan_dereference_zork_outbox_next"] targetAccount := suite.testAccounts["local_account_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request diff --git a/internal/api/s2s/user/repliesget_test.go b/internal/api/s2s/user/repliesget_test.go @@ -33,6 +33,8 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -47,10 +49,13 @@ func (suite *RepliesGetTestSuite) TestGetReplies() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -108,10 +113,13 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -172,10 +180,13 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request diff --git a/internal/api/s2s/user/statusget_test.go b/internal/api/s2s/user/statusget_test.go @@ -32,6 +32,8 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -46,10 +48,13 @@ func (suite *StatusGetTestSuite) TestGetStatus() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -111,10 +116,13 @@ func (suite *StatusGetTestSuite) TestGetStatusLowercase() { targetAccount := suite.testAccounts["local_account_1"] targetStatus := suite.testStatuses["local_account_1_status_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go @@ -28,9 +28,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -76,13 +78,16 @@ func (suite *UserStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.userModule = user.New(suite.processor).(*user.Module) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module) diff --git a/internal/api/s2s/user/userget_test.go b/internal/api/s2s/user/userget_test.go @@ -33,7 +33,9 @@ import ( "github.com/superseriousbusiness/activity/streams/vocab" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -47,10 +49,13 @@ func (suite *UserGetTestSuite) TestGetUser() { signedRequest := derefRequests["foss_satan_dereference_zork"] targetAccount := suite.testAccounts["local_account_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request @@ -125,10 +130,13 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() { derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts) signedRequest := derefRequests["foss_satan_dereference_zork_public_key"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) - federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) + federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker) emailSender := testrig.NewEmailSender("../../../../web/template/", nil) - processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager) + processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker) userModule := user.New(processor).(*user.Module) // setup request diff --git a/internal/api/s2s/webfinger/webfinger_test.go b/internal/api/s2s/webfinger/webfinger_test.go @@ -33,9 +33,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -79,13 +81,16 @@ func (suite *WebfingerStandardTestSuite) SetupTest() { testrig.InitTestLog() testrig.InitTestConfig() + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker) suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil) - suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager) + suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.securityModule = security.New(suite.db, suite.oauthServer).(*security.Module) diff --git a/internal/api/s2s/webfinger/webfingerget_test.go b/internal/api/s2s/webfinger/webfingerget_test.go @@ -32,7 +32,9 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/webfinger" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -69,7 +71,9 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() { func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender, clientWorker, fedWorker) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) targetAccount := accountDomainAccount() @@ -103,7 +107,9 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() { viper.Set(config.Keys.Host, "gts.example.org") viper.Set(config.Keys.AccountDomain, "example.org") - suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaManager(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender, clientWorker, fedWorker) suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module) targetAccount := accountDomainAccount() diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go @@ -32,7 +32,9 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -148,6 +150,7 @@ func (suite *DereferencerStandardTestSuite) mockTransportController() transport. return response, nil } + fedWorker := worker.New[messages.FromFederator](-1, -1) mockClient := testrig.NewMockHTTPClient(do) - return testrig.NewTestTransportController(mockClient, suite.db) + return testrig.NewTestTransportController(mockClient, suite.db, fedWorker) } diff --git a/internal/federation/federatingdb/accept.go b/internal/federation/federatingdb/accept.go @@ -48,9 +48,9 @@ func (f *federatingDB) Accept(ctx context.Context, accept vocab.ActivityStreamsA l.Debug("entering Accept") } - receivingAccount, _, fromFederatorChan := extractFromCtx(ctx) - if receivingAccount == nil || fromFederatorChan == nil { - // If the receiving account or federator channel wasn't set on the context, that means this request didn't pass + receivingAccount, _ := extractFromCtx(ctx) + if receivingAccount == nil { + // If the receiving account wasn't set on the context, that means this request didn't pass // through the API, but came from inside GtS as the result of another activity on this instance. That being so, // we can safely just ignore this activity, since we know we've already processed it elsewhere. return nil @@ -82,12 +82,12 @@ func (f *federatingDB) Accept(ctx context.Context, accept vocab.ActivityStreamsA return err } - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityAccept, GTSModel: follow, ReceivingAccount: receivingAccount, - } + }) return nil } @@ -117,12 +117,12 @@ func (f *federatingDB) Accept(ctx context.Context, accept vocab.ActivityStreamsA return err } - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityAccept, GTSModel: follow, ReceivingAccount: receivingAccount, - } + }) return nil } diff --git a/internal/federation/federatingdb/announce.go b/internal/federation/federatingdb/announce.go @@ -44,9 +44,9 @@ func (f *federatingDB) Announce(ctx context.Context, announce vocab.ActivityStre l.Debug("entering Announce") } - receivingAccount, _, fromFederatorChan := extractFromCtx(ctx) - if receivingAccount == nil || fromFederatorChan == nil { - // If the receiving account or federator channel wasn't set on the context, that means this request didn't pass + receivingAccount, _ := extractFromCtx(ctx) + if receivingAccount == nil { + // If the receiving account wasn't set on the context, that means this request didn't pass // through the API, but came from inside GtS as the result of another activity on this instance. That being so, // we can safely just ignore this activity, since we know we've already processed it elsewhere. return nil @@ -63,12 +63,12 @@ func (f *federatingDB) Announce(ctx context.Context, announce vocab.ActivityStre } // it's a new announce so pass it back to the processor async for dereferencing etc - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ActivityAnnounce, APActivityType: ap.ActivityCreate, GTSModel: boost, ReceivingAccount: receivingAccount, - } + }) return nil } diff --git a/internal/federation/federatingdb/create.go b/internal/federation/federatingdb/create.go @@ -61,9 +61,9 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error { l.Debug("entering Create") } - receivingAccount, requestingAccount, fromFederatorChan := extractFromCtx(ctx) - if receivingAccount == nil || fromFederatorChan == nil { - // If the receiving account or federator channel wasn't set on the context, that means this request didn't pass + receivingAccount, requestingAccount := extractFromCtx(ctx) + if receivingAccount == nil { + // If the receiving account wasn't set on the context, that means this request didn't pass // through the API, but came from inside GtS as the result of another activity on this instance. That being so, // we can safely just ignore this activity, since we know we've already processed it elsewhere. return nil @@ -72,16 +72,16 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error { switch asType.GetTypeName() { case ap.ActivityBlock: // BLOCK SOMETHING - return f.activityBlock(ctx, asType, receivingAccount, requestingAccount, fromFederatorChan) + return f.activityBlock(ctx, asType, receivingAccount, requestingAccount) case ap.ActivityCreate: // CREATE SOMETHING - return f.activityCreate(ctx, asType, receivingAccount, requestingAccount, fromFederatorChan) + return f.activityCreate(ctx, asType, receivingAccount, requestingAccount) case ap.ActivityFollow: // FOLLOW SOMETHING - return f.activityFollow(ctx, asType, receivingAccount, requestingAccount, fromFederatorChan) + return f.activityFollow(ctx, asType, receivingAccount, requestingAccount) case ap.ActivityLike: // LIKE SOMETHING - return f.activityLike(ctx, asType, receivingAccount, requestingAccount, fromFederatorChan) + return f.activityLike(ctx, asType, receivingAccount, requestingAccount) } return nil } @@ -90,7 +90,7 @@ func (f *federatingDB) Create(ctx context.Context, asType vocab.Type) error { BLOCK HANDLERS */ -func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, receiving *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error { +func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, receiving *gtsmodel.Account, requestingAccount *gtsmodel.Account) error { blockable, ok := asType.(vocab.ActivityStreamsBlock) if !ok { return errors.New("activityBlock: could not convert type to block") @@ -111,12 +111,12 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec return fmt.Errorf("activityBlock: database error inserting block: %s", err) } - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ActivityBlock, APActivityType: ap.ActivityCreate, GTSModel: block, ReceivingAccount: receiving, - } + }) return nil } @@ -124,7 +124,7 @@ func (f *federatingDB) activityBlock(ctx context.Context, asType vocab.Type, rec CREATE HANDLERS */ -func (f *federatingDB) activityCreate(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error { +func (f *federatingDB) activityCreate(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error { create, ok := asType.(vocab.ActivityStreamsCreate) if !ok { return errors.New("activityCreate: could not convert type to create") @@ -152,7 +152,7 @@ func (f *federatingDB) activityCreate(ctx context.Context, asType vocab.Type, re switch asObjectTypeName { case ap.ObjectNote: // CREATE A NOTE - if err := f.createNote(ctx, objectIter.GetActivityStreamsNote(), receivingAccount, requestingAccount, fromFederatorChan); err != nil { + if err := f.createNote(ctx, objectIter.GetActivityStreamsNote(), receivingAccount, requestingAccount); err != nil { errs = append(errs, err.Error()) } default: @@ -168,7 +168,7 @@ func (f *federatingDB) activityCreate(ctx context.Context, asType vocab.Type, re } // createNote handles a Create activity with a Note type. -func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStreamsNote, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error { +func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStreamsNote, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error { l := logrus.WithFields(logrus.Fields{ "func": "createNote", "receivingAccount": receivingAccount.URI, @@ -206,13 +206,13 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream return nil } // pass the note iri into the processor and have it do the dereferencing instead of doing it here - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityCreate, APIri: id.GetIRI(), GTSModel: nil, ReceivingAccount: receivingAccount, - } + }) return nil } @@ -241,12 +241,12 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream return fmt.Errorf("createNote: database error inserting status: %s", err) } - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityCreate, GTSModel: status, ReceivingAccount: receivingAccount, - } + }) return nil } @@ -255,7 +255,7 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream FOLLOW HANDLERS */ -func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error { +func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error { follow, ok := asType.(vocab.ActivityStreamsFollow) if !ok { return errors.New("activityFollow: could not convert type to follow") @@ -276,12 +276,12 @@ func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, re return fmt.Errorf("activityFollow: database error inserting follow request: %s", err) } - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityCreate, GTSModel: followRequest, ReceivingAccount: receivingAccount, - } + }) return nil } @@ -290,7 +290,7 @@ func (f *federatingDB) activityFollow(ctx context.Context, asType vocab.Type, re LIKE HANDLERS */ -func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) error { +func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) error { like, ok := asType.(vocab.ActivityStreamsLike) if !ok { return errors.New("activityLike: could not convert type to like") @@ -311,12 +311,12 @@ func (f *federatingDB) activityLike(ctx context.Context, asType vocab.Type, rece return fmt.Errorf("activityLike: database error inserting fave: %s", err) } - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ActivityLike, APActivityType: ap.ActivityCreate, GTSModel: fave, ReceivingAccount: receivingAccount, - } + }) return nil } diff --git a/internal/federation/federatingdb/create_test.go b/internal/federation/federatingdb/create_test.go @@ -25,7 +25,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/messages" ) type CreateTestSuite struct { @@ -35,9 +34,8 @@ type CreateTestSuite struct { func (suite *CreateTestSuite) TestCreateNote() { receivingAccount := suite.testAccounts["local_account_1"] requestingAccount := suite.testAccounts["remote_account_1"] - fromFederatorChan := make(chan messages.FromFederator, 10) - ctx := createTestContext(receivingAccount, requestingAccount, fromFederatorChan) + ctx := createTestContext(receivingAccount, requestingAccount) create := suite.testActivities["dm_for_zork"].Activity @@ -45,7 +43,7 @@ func (suite *CreateTestSuite) TestCreateNote() { suite.NoError(err) // should be a message heading to the processor now, which we can intercept here - msg := <-fromFederatorChan + msg := <-suite.fromFederator suite.Equal(ap.ObjectNote, msg.APObjectType) suite.Equal(ap.ActivityCreate, msg.APActivityType) @@ -65,9 +63,8 @@ func (suite *CreateTestSuite) TestCreateNote() { func (suite *CreateTestSuite) TestCreateNoteForward() { receivingAccount := suite.testAccounts["local_account_1"] requestingAccount := suite.testAccounts["remote_account_1"] - fromFederatorChan := make(chan messages.FromFederator, 10) - ctx := createTestContext(receivingAccount, requestingAccount, fromFederatorChan) + ctx := createTestContext(receivingAccount, requestingAccount) create := suite.testActivities["forwarded_message"].Activity @@ -75,7 +72,7 @@ func (suite *CreateTestSuite) TestCreateNoteForward() { suite.NoError(err) // should be a message heading to the processor now, which we can intercept here - msg := <-fromFederatorChan + msg := <-suite.fromFederator suite.Equal(ap.ObjectNote, msg.APObjectType) suite.Equal(ap.ActivityCreate, msg.APActivityType) diff --git a/internal/federation/federatingdb/db.go b/internal/federation/federatingdb/db.go @@ -25,7 +25,9 @@ import ( "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // DB wraps the pub.Database interface with a couple of custom functions for GoToSocial. @@ -42,14 +44,16 @@ type DB interface { type federatingDB struct { locks mutexes.MutexMap db db.DB + fedWorker *worker.Worker[messages.FromFederator] typeConverter typeutils.TypeConverter } // New returns a DB interface using the given database and config -func New(db db.DB) DB { +func New(db db.DB, fedWorker *worker.Worker[messages.FromFederator]) DB { fdb := federatingDB{ locks: mutexes.NewMap(-1, -1), // use defaults db: db, + fedWorker: fedWorker, typeConverter: typeutils.NewConverter(db), } return &fdb diff --git a/internal/federation/federatingdb/delete.go b/internal/federation/federatingdb/delete.go @@ -44,9 +44,9 @@ func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error { ) l.Debug("entering Delete") - receivingAccount, _, fromFederatorChan := extractFromCtx(ctx) - if receivingAccount == nil || fromFederatorChan == nil { - // If the receiving account or federator channel wasn't set on the context, that means this request didn't pass + receivingAccount, _ := extractFromCtx(ctx) + if receivingAccount == nil { + // If the receiving account wasn't set on the context, that means this request didn't pass // through the API, but came from inside GtS as the result of another activity on this instance. That being so, // we can safely just ignore this activity, since we know we've already processed it elsewhere. return nil @@ -61,24 +61,24 @@ func (f *federatingDB) Delete(ctx context.Context, id *url.URL) error { if err := f.db.DeleteByID(ctx, s.ID, >smodel.Status{}); err != nil { return fmt.Errorf("DELETE: err deleting status: %s", err) } - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityDelete, GTSModel: s, ReceivingAccount: receivingAccount, - } + }) } a, err := f.db.GetAccountByURI(ctx, id.String()) if err == nil { // it's an account l.Debugf("uri is for an account with id %s, passing delete message to the processor", a.ID) - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ObjectProfile, APActivityType: ap.ActivityDelete, GTSModel: a, ReceivingAccount: receivingAccount, - } + }) } return nil diff --git a/internal/federation/federatingdb/federatingdb_test.go b/internal/federation/federatingdb/federatingdb_test.go @@ -28,14 +28,17 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) type FederatingDBTestSuite struct { suite.Suite - db db.DB - tc typeutils.TypeConverter - federatingDB federatingdb.DB + db db.DB + tc typeutils.TypeConverter + fedWorker *worker.Worker[messages.FromFederator] + fromFederator chan messages.FromFederator + federatingDB federatingdb.DB testTokens map[string]*gtsmodel.Token testClients map[string]*gtsmodel.Client @@ -62,10 +65,17 @@ func (suite *FederatingDBTestSuite) SetupSuite() { func (suite *FederatingDBTestSuite) SetupTest() { testrig.InitTestLog() testrig.InitTestConfig() + suite.fedWorker = worker.New[messages.FromFederator](-1, -1) + suite.fromFederator = make(chan messages.FromFederator, 10) + suite.fedWorker.SetProcessor(func(ctx context.Context, msg messages.FromFederator) error { + suite.fromFederator <- msg + return nil + }) + _ = suite.fedWorker.Start() suite.db = testrig.NewTestDB() suite.testActivities = testrig.NewTestActivities(suite.testAccounts) suite.tc = testrig.NewTestTypeConverter(suite.db) - suite.federatingDB = testrig.NewTestFederatingDB(suite.db) + suite.federatingDB = testrig.NewTestFederatingDB(suite.db, suite.fedWorker) testrig.StandardDBSetup(suite.db, suite.testAccounts) } @@ -73,10 +83,9 @@ func (suite *FederatingDBTestSuite) TearDownTest() { testrig.StandardDBTeardown(suite.db) } -func createTestContext(receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) context.Context { +func createTestContext(receivingAccount *gtsmodel.Account, requestingAccount *gtsmodel.Account) context.Context { ctx := context.Background() ctx = context.WithValue(ctx, ap.ContextReceivingAccount, receivingAccount) ctx = context.WithValue(ctx, ap.ContextRequestingAccount, requestingAccount) - ctx = context.WithValue(ctx, ap.ContextFromFederatorChan, fromFederatorChan) return ctx } diff --git a/internal/federation/federatingdb/reject.go b/internal/federation/federatingdb/reject.go @@ -47,8 +47,8 @@ func (f *federatingDB) Reject(ctx context.Context, reject vocab.ActivityStreamsR l.Debug("entering Reject") } - receivingAccount, _, fromFederatorChan := extractFromCtx(ctx) - if receivingAccount == nil || fromFederatorChan == nil { + receivingAccount, _ := extractFromCtx(ctx) + if receivingAccount == nil { // If the receiving account or federator channel wasn't set on the context, that means this request didn't pass // through the API, but came from inside GtS as the result of another activity on this instance. That being so, // we can safely just ignore this activity, since we know we've already processed it elsewhere. diff --git a/internal/federation/federatingdb/reject_test.go b/internal/federation/federatingdb/reject_test.go @@ -26,7 +26,6 @@ import ( "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/uris" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -40,8 +39,7 @@ func (suite *RejectTestSuite) TestRejectFollowRequest() { // remote_account_2 rejects the follow request followingAccount := suite.testAccounts["local_account_1"] followedAccount := suite.testAccounts["remote_account_2"] - fromFederatorChan := make(chan messages.FromFederator, 10) - ctx := createTestContext(followingAccount, followedAccount, fromFederatorChan) + ctx := createTestContext(followingAccount, followedAccount) // put the follow request in the database fr := >smodel.FollowRequest{ @@ -84,7 +82,7 @@ func (suite *RejectTestSuite) TestRejectFollowRequest() { suite.NoError(err) // there should be nothing in the federator channel since nothing needs to be passed - suite.Empty(fromFederatorChan) + suite.Empty(suite.fromFederator) // the follow request should not be in the database anymore -- it's been rejected err = suite.db.GetByID(ctx, fr.ID, >smodel.FollowRequest{}) diff --git a/internal/federation/federatingdb/undo.go b/internal/federation/federatingdb/undo.go @@ -46,9 +46,9 @@ func (f *federatingDB) Undo(ctx context.Context, undo vocab.ActivityStreamsUndo) l.Debug("entering Undo") } - receivingAccount, _, fromFederatorChan := extractFromCtx(ctx) - if receivingAccount == nil || fromFederatorChan == nil { - // If the receiving account or federator channel wasn't set on the context, that means this request didn't pass + receivingAccount, _ := extractFromCtx(ctx) + if receivingAccount == nil { + // If the receiving account wasn't set on the context, that means this request didn't pass // through the API, but came from inside GtS as the result of another activity on this instance. That being so, // we can safely just ignore this activity, since we know we've already processed it elsewhere. return nil diff --git a/internal/federation/federatingdb/update.go b/internal/federation/federatingdb/update.go @@ -57,9 +57,9 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { l.Debug("entering Update") } - receivingAccount, _, fromFederatorChan := extractFromCtx(ctx) - if receivingAccount == nil || fromFederatorChan == nil { - // If the receiving account or federator channel wasn't set on the context, that means this request didn't pass + receivingAccount, _ := extractFromCtx(ctx) + if receivingAccount == nil { + // If the receiving account wasn't set on the context, that means this request didn't pass // through the API, but came from inside GtS as the result of another activity on this instance. That being so, // we can safely just ignore this activity, since we know we've already processed it elsewhere. return nil @@ -148,12 +148,12 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error { } // pass to the processor for further processing of eg., avatar/header - fromFederatorChan <- messages.FromFederator{ + f.fedWorker.Queue(messages.FromFederator{ APObjectType: ap.ObjectProfile, APActivityType: ap.ActivityUpdate, GTSModel: updatedAcct, ReceivingAccount: receivingAccount, - } + }) } return nil diff --git a/internal/federation/federatingdb/util.go b/internal/federation/federatingdb/util.go @@ -34,7 +34,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" - "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/uris" ) @@ -310,7 +309,7 @@ func (f *federatingDB) collectIRIs(ctx context.Context, iris []*url.URL) (vocab. // - The requesting account that posted to the inbox. // - A channel that messages for the processor can be placed into. // If a value is not present, nil will be returned for it. It's up to the caller to check this and respond appropriately. -func extractFromCtx(ctx context.Context) (receivingAccount, requestingAccount *gtsmodel.Account, fromFederatorChan chan messages.FromFederator) { +func extractFromCtx(ctx context.Context) (receivingAccount, requestingAccount *gtsmodel.Account) { receivingAccountI := ctx.Value(ap.ContextReceivingAccount) if receivingAccountI != nil { var ok bool @@ -329,15 +328,6 @@ func extractFromCtx(ctx context.Context) (receivingAccount, requestingAccount *g } } - fromFederatorChanI := ctx.Value(ap.ContextFromFederatorChan) - if fromFederatorChanI != nil { - var ok bool - fromFederatorChan, ok = fromFederatorChanI.(chan messages.FromFederator) - if !ok { - logrus.Panicf("extractFromCtx: context entry with key %s could not be asserted to chan messages.FromFederator", ap.ContextFromFederatorChan) - } - } - return } diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go @@ -34,7 +34,9 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -73,12 +75,14 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() { // the activity we're gonna use activity := suite.activities["dm_for_zork"] + fedWorker := worker.New[messages.FromFederator](-1, -1) + // setup transport controller with a no-op client so we don't make external calls tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) { return nil, nil - }), suite.db) + }), suite.db, fedWorker) // setup module being tested - federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage)) + federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage)) // setup request ctx := context.Background() @@ -105,9 +109,11 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() { sendingAccount := suite.accounts["remote_account_1"] inboxAccount := suite.accounts["local_account_1"] - tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) // now setup module being tested, with the mock transport controller - federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage)) + federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.typeConverter, testrig.NewTestMediaManager(suite.db, suite.storage)) request := httptest.NewRequest(http.MethodPost, "http://localhost:8080/users/the_mighty_zork/inbox", nil) // we need these headers for the request to be validated diff --git a/internal/processing/account/account.go b/internal/processing/account/account.go @@ -33,6 +33,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/visibility" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/oauth2/v4" ) @@ -81,28 +82,28 @@ type Processor interface { } type processor struct { - tc typeutils.TypeConverter - mediaManager media.Manager - fromClientAPI chan messages.FromClientAPI - oauthServer oauth.Server - filter visibility.Filter - formatter text.Formatter - db db.DB - federator federation.Federator - parseMention gtsmodel.ParseMentionFunc + tc typeutils.TypeConverter + mediaManager media.Manager + clientWorker *worker.Worker[messages.FromClientAPI] + oauthServer oauth.Server + filter visibility.Filter + formatter text.Formatter + db db.DB + federator federation.Federator + parseMention gtsmodel.ParseMentionFunc } // New returns a new account processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, clientWorker *worker.Worker[messages.FromClientAPI], federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor { return &processor{ - tc: tc, - mediaManager: mediaManager, - fromClientAPI: fromClientAPI, - oauthServer: oauthServer, - filter: visibility.NewFilter(db), - formatter: text.NewFormatter(db), - db: db, - federator: federator, - parseMention: parseMention, + tc: tc, + mediaManager: mediaManager, + clientWorker: clientWorker, + oauthServer: oauthServer, + filter: visibility.NewFilter(db), + formatter: text.NewFormatter(db), + db: db, + federator: federator, + parseMention: parseMention, } } diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go @@ -19,6 +19,8 @@ package account_test import ( + "context" + "codeberg.org/gruf/go-store/kv" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/activity/pub" @@ -33,6 +35,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/processing/account" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -78,6 +81,16 @@ func (suite *AccountStandardTestSuite) SetupTest() { testrig.InitTestLog() testrig.InitTestConfig() + fedWorker := worker.New[messages.FromFederator](-1, -1) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + clientWorker.SetProcessor(func(_ context.Context, msg messages.FromClientAPI) error { + suite.fromClientAPIChan <- msg + return nil + }) + + _ = fedWorker.Start() + _ = clientWorker.Start() + suite.db = testrig.NewTestDB() suite.tc = testrig.NewTestTypeConverter(suite.db) suite.storage = testrig.NewTestStorage() @@ -85,11 +98,11 @@ func (suite *AccountStandardTestSuite) SetupTest() { suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) suite.httpClient = testrig.NewMockHTTPClient(nil) - suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db) - suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager) + suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker) + suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker) suite.sentEmails = make(map[string]string) suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails) - suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator)) + suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, clientWorker, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator)) testrig.StandardDBSetup(suite.db, nil) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") } diff --git a/internal/processing/account/create.go b/internal/processing/account/create.go @@ -85,12 +85,12 @@ func (p *processor) Create(ctx context.Context, applicationToken oauth2.TokenInf // there are side effects for creating a new account (sending confirmation emails etc) // so pass a message to the processor so that it can do it asynchronously - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ObjectProfile, APActivityType: ap.ActivityCreate, GTSModel: user.Account, OriginAccount: user.Account, - } + }) return &apimodel.Token{ AccessToken: accessToken.GetAccess(), diff --git a/internal/processing/account/createblock.go b/internal/processing/account/createblock.go @@ -113,7 +113,7 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel // follow request status changed so send the UNDO activity to the channel for async processing if frChanged { - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityUndo, GTSModel: >smodel.Follow{ @@ -123,12 +123,12 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel }, OriginAccount: requestingAccount, TargetAccount: targetAccount, - } + }) } // follow status changed so send the UNDO activity to the channel for async processing if fChanged { - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityUndo, GTSModel: >smodel.Follow{ @@ -138,17 +138,17 @@ func (p *processor) BlockCreate(ctx context.Context, requestingAccount *gtsmodel }, OriginAccount: requestingAccount, TargetAccount: targetAccount, - } + }) } // handle the rest of the block process asynchronously - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityBlock, APActivityType: ap.ActivityCreate, GTSModel: block, OriginAccount: requestingAccount, TargetAccount: targetAccount, - } + }) return p.RelationshipGet(ctx, requestingAccount, targetAccountID) } diff --git a/internal/processing/account/createfollow.go b/internal/processing/account/createfollow.go @@ -101,13 +101,13 @@ func (p *processor) FollowCreate(ctx context.Context, requestingAccount *gtsmode } // otherwise we leave the follow request as it is and we handle the rest of the process asynchronously - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityCreate, GTSModel: fr, OriginAccount: requestingAccount, TargetAccount: targetAcct, - } + }) // return whatever relationship results from this return p.RelationshipGet(ctx, requestingAccount, form.ID) diff --git a/internal/processing/account/delete.go b/internal/processing/account/delete.go @@ -159,13 +159,13 @@ selectStatusesLoop: // pass the status delete through the client api channel for processing s.Account = account l.Debug("putting status in the client api channel") - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityDelete, GTSModel: s, OriginAccount: account, TargetAccount: account, - } + }) if err := p.db.DeleteByID(ctx, s.ID, s); err != nil { if err != db.ErrNoEntries { @@ -195,13 +195,13 @@ selectStatusesLoop: } l.Debug("putting boost undo in the client api channel") - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityAnnounce, APActivityType: ap.ActivityUndo, GTSModel: s, OriginAccount: b.Account, TargetAccount: account, - } + }) if err := p.db.DeleteByID(ctx, b.ID, b); err != nil { if err != db.ErrNoEntries { @@ -331,7 +331,7 @@ func (p *processor) DeleteLocal(ctx context.Context, account *gtsmodel.Account, } // put the delete in the processor queue to handle the rest of it asynchronously - p.fromClientAPI <- fromClientAPIMessage + p.clientWorker.Queue(fromClientAPIMessage) return nil } diff --git a/internal/processing/account/removeblock.go b/internal/processing/account/removeblock.go @@ -54,13 +54,13 @@ func (p *processor) BlockRemove(ctx context.Context, requestingAccount *gtsmodel // block status changed so send the UNDO activity to the channel for async processing if blockChanged { - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityBlock, APActivityType: ap.ActivityUndo, GTSModel: block, OriginAccount: requestingAccount, TargetAccount: targetAccount, - } + }) } // return whatever relationship results from all this diff --git a/internal/processing/account/removefollow.go b/internal/processing/account/removefollow.go @@ -80,7 +80,7 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode // follow request status changed so send the UNDO activity to the channel for async processing if frChanged { - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityUndo, GTSModel: >smodel.Follow{ @@ -90,12 +90,12 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode }, OriginAccount: requestingAccount, TargetAccount: targetAcct, - } + }) } // follow status changed so send the UNDO activity to the channel for async processing if fChanged { - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityUndo, GTSModel: >smodel.Follow{ @@ -105,7 +105,7 @@ func (p *processor) FollowRemove(ctx context.Context, requestingAccount *gtsmode }, OriginAccount: requestingAccount, TargetAccount: targetAcct, - } + }) } // return whatever relationship results from all this diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go @@ -117,12 +117,12 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form return nil, fmt.Errorf("could not update account %s: %s", account.ID, err) } - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ObjectProfile, APActivityType: ap.ActivityUpdate, GTSModel: updatedAccount, OriginAccount: updatedAccount, - } + }) acctSensitive, err := p.tc.AccountToAPIAccountSensitive(ctx, updatedAccount) if err != nil { diff --git a/internal/processing/admin/accountaction.go b/internal/processing/admin/accountaction.go @@ -34,12 +34,12 @@ func (p *processor) AccountAction(ctx context.Context, account *gtsmodel.Account case string(gtsmodel.AdminActionSuspend): adminAction.Type = gtsmodel.AdminActionSuspend // pass the account delete through the client api channel for processing - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActorPerson, APActivityType: ap.ActivityDelete, OriginAccount: account, TargetAccount: targetAccount, - } + }) default: return gtserror.NewErrorBadRequest(fmt.Errorf("admin action type %s is not supported for this endpoint", form.Type)) } diff --git a/internal/processing/admin/admin.go b/internal/processing/admin/admin.go @@ -29,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // Processor wraps a bunch of functions for processing admin actions. @@ -43,18 +44,18 @@ type Processor interface { } type processor struct { - tc typeutils.TypeConverter - mediaManager media.Manager - fromClientAPI chan messages.FromClientAPI - db db.DB + tc typeutils.TypeConverter + mediaManager media.Manager + clientWorker *worker.Worker[messages.FromClientAPI] + db db.DB } // New returns a new admin processor. -func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, fromClientAPI chan messages.FromClientAPI) Processor { +func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, clientWorker *worker.Worker[messages.FromClientAPI]) Processor { return &processor{ - tc: tc, - mediaManager: mediaManager, - fromClientAPI: fromClientAPI, - db: db, + tc: tc, + mediaManager: mediaManager, + clientWorker: clientWorker, + db: db, } } diff --git a/internal/processing/admin/createdomainblock.go b/internal/processing/admin/createdomainblock.go @@ -141,13 +141,13 @@ selectAccountsLoop: l.Debugf("putting delete for account %s in the clientAPI channel", a.Username) // pass the account delete through the client api channel for processing - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActorPerson, APActivityType: ap.ActivityDelete, GTSModel: block, OriginAccount: account, TargetAccount: a, - } + }) // if this is the last account in the slice, set the maxID appropriately for the next query if i == len(accounts)-1 { diff --git a/internal/processing/federation/federation.go b/internal/processing/federation/federation.go @@ -27,7 +27,6 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/visibility" ) @@ -81,20 +80,18 @@ type Processor interface { } type processor struct { - db db.DB - federator federation.Federator - tc typeutils.TypeConverter - filter visibility.Filter - fromFederator chan messages.FromFederator + db db.DB + federator federation.Federator + tc typeutils.TypeConverter + filter visibility.Filter } // New returns a new federation processor. -func New(db db.DB, tc typeutils.TypeConverter, federator federation.Federator, fromFederator chan messages.FromFederator) Processor { +func New(db db.DB, tc typeutils.TypeConverter, federator federation.Federator) Processor { return &processor{ - db: db, - federator: federator, - tc: tc, - filter: visibility.NewFilter(db), - fromFederator: fromFederator, + db: db, + federator: federator, + tc: tc, + filter: visibility.NewFilter(db), } } diff --git a/internal/processing/federation/postinbox.go b/internal/processing/federation/postinbox.go @@ -21,12 +21,8 @@ package federation import ( "context" "net/http" - - "github.com/superseriousbusiness/gotosocial/internal/ap" ) func (p *processor) PostInbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (bool, error) { - // pass the fromFederator channel through to postInbox, since it'll be needed later - contextWithChannel := context.WithValue(ctx, ap.ContextFromFederatorChan, p.fromFederator) - return p.federator.FederatingActor().PostInbox(contextWithChannel, w, r) + return p.federator.FederatingActor().PostInbox(ctx, w, r) } diff --git a/internal/processing/followrequest.go b/internal/processing/followrequest.go @@ -78,13 +78,13 @@ func (p *processor) FollowRequestAccept(ctx context.Context, auth *oauth.Auth, a follow.TargetAccount = followTargetAccount } - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityAccept, GTSModel: follow, OriginAccount: follow.Account, TargetAccount: follow.TargetAccount, - } + }) gtsR, err := p.db.GetRelationship(ctx, auth.Account.ID, accountID) if err != nil { @@ -121,13 +121,13 @@ func (p *processor) FollowRequestReject(ctx context.Context, auth *oauth.Auth, a followRequest.TargetAccount = a } - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityFollow, APActivityType: ap.ActivityReject, GTSModel: followRequest, OriginAccount: followRequest.Account, TargetAccount: followRequest.TargetAccount, - } + }) gtsR, err := p.db.GetRelationship(ctx, auth.Account.ID, accountID) if err != nil { diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go @@ -29,9 +29,11 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -120,6 +122,7 @@ func (suite *MediaStandardTestSuite) mockTransportController() transport.Control return response, nil } + fedWorker := worker.New[messages.FromFederator](-1, -1) mockClient := testrig.NewMockHTTPClient(do) - return testrig.NewTestTransportController(mockClient, suite.db) + return testrig.NewTestTransportController(mockClient, suite.db, fedWorker) } diff --git a/internal/processing/processor.go b/internal/processing/processor.go @@ -24,7 +24,6 @@ import ( "net/url" "codeberg.org/gruf/go-store/kv" - "github.com/sirupsen/logrus" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/email" @@ -45,6 +44,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/visibility" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // Processor should be passed to api modules (see internal/apimodule/...). It is used for @@ -55,7 +55,7 @@ import ( // for clean distribution of messages without slowing down the client API and harming the user experience. type Processor interface { // Start starts the Processor, reading from its channels and passing messages back and forth. - Start(ctx context.Context) error + Start() error // Stop stops the processor cleanly, finishing handling any remaining messages before closing down. Stop() error // ProcessFromClientAPI processes one message coming from the clientAPI channel, and triggers appropriate side effects. @@ -235,10 +235,10 @@ type Processor interface { // processor just implements the Processor interface type processor struct { - fromClientAPI chan messages.FromClientAPI - fromFederator chan messages.FromFederator + clientWorker *worker.Worker[messages.FromClientAPI] + fedWorker *worker.Worker[messages.FromFederator] + federator federation.Federator - stop chan interface{} tc typeutils.TypeConverter oauthServer oauth.Server mediaManager media.Manager @@ -268,26 +268,26 @@ func NewProcessor( mediaManager media.Manager, storage *kv.KVStore, db db.DB, - emailSender email.Sender) Processor { - - fromClientAPI := make(chan messages.FromClientAPI, 1000) - fromFederator := make(chan messages.FromFederator, 1000) + emailSender email.Sender, + clientWorker *worker.Worker[messages.FromClientAPI], + fedWorker *worker.Worker[messages.FromFederator], +) Processor { parseMentionFunc := GetParseMentionFunc(db, federator) - statusProcessor := status.New(db, tc, fromClientAPI, parseMentionFunc) + statusProcessor := status.New(db, tc, clientWorker, parseMentionFunc) streamingProcessor := streaming.New(db, oauthServer) - accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator, parseMentionFunc) - adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI) + accountProcessor := account.New(db, tc, mediaManager, oauthServer, clientWorker, federator, parseMentionFunc) + adminProcessor := admin.New(db, tc, mediaManager, clientWorker) mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage) userProcessor := user.New(db, emailSender) - federationProcessor := federationProcessor.New(db, tc, federator, fromFederator) + federationProcessor := federationProcessor.New(db, tc, federator) filter := visibility.NewFilter(db) return &processor{ - fromClientAPI: fromClientAPI, - fromFederator: fromFederator, + clientWorker: clientWorker, + fedWorker: fedWorker, + federator: federator, - stop: make(chan interface{}), tc: tc, oauthServer: oauthServer, mediaManager: mediaManager, @@ -307,36 +307,29 @@ func NewProcessor( } // Start starts the Processor, reading from its channels and passing messages back and forth. -func (p *processor) Start(ctx context.Context) error { - go func() { - DistLoop: - for { - select { - case clientMsg := <-p.fromClientAPI: - logrus.Tracef("received message FROM client API: %+v", clientMsg) - go func() { - if err := p.ProcessFromClientAPI(ctx, clientMsg); err != nil { - logrus.Error(err) - } - }() - case federatorMsg := <-p.fromFederator: - logrus.Tracef("received message FROM federator: %+v", federatorMsg) - go func() { - if err := p.ProcessFromFederator(ctx, federatorMsg); err != nil { - logrus.Error(err) - } - }() - case <-p.stop: - break DistLoop - } - } - }() +func (p *processor) Start() error { + // Setup and start the client API worker pool + p.clientWorker.SetProcessor(p.ProcessFromClientAPI) + if err := p.clientWorker.Start(); err != nil { + return err + } + + // Setup and start the federator worker pool + p.fedWorker.SetProcessor(p.ProcessFromFederator) + if err := p.fedWorker.Start(); err != nil { + return err + } + return nil } // Stop stops the processor cleanly, finishing handling any remaining messages before closing down. -// TODO: empty message buffer properly before stopping otherwise we'll lose federating messages. func (p *processor) Stop() error { - close(p.stop) + if err := p.clientWorker.Stop(); err != nil { + return err + } + if err := p.fedWorker.Stop(); err != nil { + return err + } return nil } diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go @@ -34,11 +34,13 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/processing" "github.com/superseriousbusiness/gotosocial/internal/timeline" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -215,17 +217,20 @@ func (suite *ProcessingStandardTestSuite) SetupTest() { }, nil }) - suite.transportController = testrig.NewTestTransportController(httpClient, suite.db) + clientWorker := worker.New[messages.FromClientAPI](-1, -1) + fedWorker := worker.New[messages.FromFederator](-1, -1) + + suite.transportController = testrig.NewTestTransportController(httpClient, suite.db, fedWorker) suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager) + suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker) suite.oauthServer = testrig.NewTestOauthServer(suite.db) suite.emailSender = testrig.NewEmailSender("../../web/template/", nil) - suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, suite.storage, suite.db, suite.emailSender) + suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaManager, suite.storage, suite.db, suite.emailSender, clientWorker, fedWorker) testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../testrig/media") - if err := suite.processor.Start(context.Background()); err != nil { + if err := suite.processor.Start(); err != nil { panic(err) } } diff --git a/internal/processing/status/boost.go b/internal/processing/status/boost.go @@ -65,13 +65,13 @@ func (p *processor) Boost(ctx context.Context, requestingAccount *gtsmodel.Accou } // send it back to the processor for async processing - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityAnnounce, APActivityType: ap.ActivityCreate, GTSModel: boostWrapperStatus, OriginAccount: requestingAccount, TargetAccount: targetStatus.Account, - } + }) // return the frontend representation of the new status to the submitter apiStatus, err := p.tc.StatusToAPIStatus(ctx, boostWrapperStatus, requestingAccount) diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go @@ -97,12 +97,12 @@ func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, appli } // send it back to the processor for async processing - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityCreate, GTSModel: newStatus, OriginAccount: account, - } + }) // return the frontend representation of the new status to the submitter apiStatus, err := p.tc.StatusToAPIStatus(ctx, newStatus, account) diff --git a/internal/processing/status/delete.go b/internal/processing/status/delete.go @@ -53,13 +53,13 @@ func (p *processor) Delete(ctx context.Context, requestingAccount *gtsmodel.Acco } // send it back to the processor for async processing - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityDelete, GTSModel: targetStatus, OriginAccount: requestingAccount, TargetAccount: requestingAccount, - } + }) return apiStatus, nil } diff --git a/internal/processing/status/fave.go b/internal/processing/status/fave.go @@ -84,13 +84,13 @@ func (p *processor) Fave(ctx context.Context, requestingAccount *gtsmodel.Accoun } // send it back to the processor for async processing - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityLike, APActivityType: ap.ActivityCreate, GTSModel: gtsFave, OriginAccount: requestingAccount, TargetAccount: targetStatus.Account, - } + }) } // return the apidon representation of the target status diff --git a/internal/processing/status/status.go b/internal/processing/status/status.go @@ -29,6 +29,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/text" "github.com/superseriousbusiness/gotosocial/internal/typeutils" "github.com/superseriousbusiness/gotosocial/internal/visibility" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // Processor wraps a bunch of functions for processing statuses. @@ -69,22 +70,22 @@ type Processor interface { } type processor struct { - tc typeutils.TypeConverter - db db.DB - filter visibility.Filter - formatter text.Formatter - fromClientAPI chan messages.FromClientAPI - parseMention gtsmodel.ParseMentionFunc + tc typeutils.TypeConverter + db db.DB + filter visibility.Filter + formatter text.Formatter + clientWorker *worker.Worker[messages.FromClientAPI] + parseMention gtsmodel.ParseMentionFunc } // New returns a new status processor. -func New(db db.DB, tc typeutils.TypeConverter, fromClientAPI chan messages.FromClientAPI, parseMention gtsmodel.ParseMentionFunc) Processor { +func New(db db.DB, tc typeutils.TypeConverter, clientWorker *worker.Worker[messages.FromClientAPI], parseMention gtsmodel.ParseMentionFunc) Processor { return &processor{ - tc: tc, - db: db, - filter: visibility.NewFilter(db), - formatter: text.NewFormatter(db), - fromClientAPI: fromClientAPI, - parseMention: parseMention, + tc: tc, + db: db, + filter: visibility.NewFilter(db), + formatter: text.NewFormatter(db), + clientWorker: clientWorker, + parseMention: parseMention, } } diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go @@ -30,18 +30,19 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/processing/status" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/typeutils" + "github.com/superseriousbusiness/gotosocial/internal/worker" "github.com/superseriousbusiness/gotosocial/testrig" ) type StatusStandardTestSuite struct { suite.Suite - db db.DB - typeConverter typeutils.TypeConverter - tc transport.Controller - storage *kv.KVStore - mediaManager media.Manager - federator federation.Federator - fromClientAPIChan chan messages.FromClientAPI + db db.DB + typeConverter typeutils.TypeConverter + tc transport.Controller + storage *kv.KVStore + mediaManager media.Manager + federator federation.Federator + clientWorker *worker.Worker[messages.FromClientAPI] // standard suite models testTokens map[string]*gtsmodel.Token @@ -74,14 +75,16 @@ func (suite *StatusStandardTestSuite) SetupTest() { testrig.InitTestConfig() testrig.InitTestLog() + fedWorker := worker.New[messages.FromFederator](-1, -1) + suite.db = testrig.NewTestDB() suite.typeConverter = testrig.NewTestTypeConverter(suite.db) - suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100) - suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db) + suite.clientWorker = worker.New[messages.FromClientAPI](-1, -1) + suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker) suite.storage = testrig.NewTestStorage() suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage) - suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager) - suite.status = status.New(suite.db, suite.typeConverter, suite.fromClientAPIChan, processing.GetParseMentionFunc(suite.db, suite.federator)) + suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker) + suite.status = status.New(suite.db, suite.typeConverter, suite.clientWorker, processing.GetParseMentionFunc(suite.db, suite.federator)) testrig.StandardDBSetup(suite.db, suite.testAccounts) testrig.StandardStorageSetup(suite.storage, "../../../testrig/media") diff --git a/internal/processing/status/unboost.go b/internal/processing/status/unboost.go @@ -91,13 +91,13 @@ func (p *processor) Unboost(ctx context.Context, requestingAccount *gtsmodel.Acc gtsBoost.BoostOf.Account = targetStatus.Account // send it back to the processor for async processing - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityAnnounce, APActivityType: ap.ActivityUndo, GTSModel: gtsBoost, OriginAccount: requestingAccount, TargetAccount: targetStatus.Account, - } + }) } apiStatus, err := p.tc.StatusToAPIStatus(ctx, targetStatus, requestingAccount) diff --git a/internal/processing/status/unfave.go b/internal/processing/status/unfave.go @@ -73,13 +73,13 @@ func (p *processor) Unfave(ctx context.Context, requestingAccount *gtsmodel.Acco } // send it back to the processor for async processing - p.fromClientAPI <- messages.FromClientAPI{ + p.clientWorker.Queue(messages.FromClientAPI{ APObjectType: ap.ActivityLike, APActivityType: ap.ActivityUndo, GTSModel: gtsFave, OriginAccount: requestingAccount, TargetAccount: targetStatus.Account, - } + }) } apiStatus, err := p.tc.StatusToAPIStatus(ctx, targetStatus, requestingAccount) diff --git a/internal/worker/workers.go b/internal/worker/workers.go @@ -0,0 +1,69 @@ +package worker + +import ( + "context" + "errors" + "runtime" + + "codeberg.org/gruf/go-runners" + "github.com/sirupsen/logrus" +) + +// Worker represents a proccessor for MsgType objects, using a worker pool to allocate resources. +type Worker[MsgType any] struct { + workers runners.WorkerPool + process func(context.Context, MsgType) error +} + +// New returns a new Worker[MsgType] with given number of workers and queue size +// (see runners.WorkerPool for more information on args). If args < 1 then suitable +// defaults are determined from the runtime's GOMAXPROCS variable. +func New[MsgType any](workers int, queue int) *Worker[MsgType] { + if workers < 1 { + workers = runtime.GOMAXPROCS(0) + } + if queue < 1 { + queue = workers * 100 + } + return &Worker[MsgType]{ + workers: runners.NewWorkerPool(workers, queue), + process: nil, + } +} + +// Start will attempt to start the underlying worker pool, or return error. +func (w *Worker[MsgType]) Start() error { + if w.process == nil { + return errors.New("nil Worker.process function") + } + if !w.workers.Start() { + return errors.New("failed to start Worker pool") + } + return nil +} + +// Stop will attempt to stop the underlying worker pool, or return error. +func (w *Worker[MsgType]) Stop() error { + if !w.workers.Stop() { + return errors.New("failed to stop Worker pool") + } + return nil +} + +// SetProcessor will set the Worker's processor function, which is called for each queued message. +func (w *Worker[MsgType]) SetProcessor(fn func(context.Context, MsgType) error) { + if w.process != nil { + logrus.Panic("Worker.process is already set") + } + w.process = fn +} + +// Queue will queue provided message to be processed with there's a free worker. +func (w *Worker[MsgType]) Queue(msg MsgType) { + logrus.Tracef("queueing %[1]T message; %+[1]v", msg) + w.workers.Enqueue(func(ctx context.Context) { + if err := w.process(ctx, msg); err != nil { + logrus.Error(err) + } + }) +} diff --git a/testrig/federatingdb.go b/testrig/federatingdb.go @@ -3,9 +3,11 @@ package testrig import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb" + "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // NewTestFederatingDB returns a federating DB with the underlying db -func NewTestFederatingDB(db db.DB) federatingdb.DB { - return federatingdb.New(db) +func NewTestFederatingDB(db db.DB, fedWorker *worker.Worker[messages.FromFederator]) federatingdb.DB { + return federatingdb.New(db, fedWorker) } diff --git a/testrig/federator.go b/testrig/federator.go @@ -23,10 +23,12 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // NewTestFederator returns a federator with the given database and (mock!!) transport controller. -func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager) federation.Federator { - return federation.NewFederator(db, NewTestFederatingDB(db), tc, NewTestTypeConverter(db), mediaManager) +func NewTestFederator(db db.DB, tc transport.Controller, storage *kv.KVStore, mediaManager media.Manager, fedWorker *worker.Worker[messages.FromFederator]) federation.Federator { + return federation.NewFederator(db, NewTestFederatingDB(db, fedWorker), tc, NewTestTypeConverter(db), mediaManager) } diff --git a/testrig/processor.go b/testrig/processor.go @@ -24,10 +24,12 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/email" "github.com/superseriousbusiness/gotosocial/internal/federation" "github.com/superseriousbusiness/gotosocial/internal/media" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/processing" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // NewTestProcessor returns a Processor suitable for testing purposes -func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager) processing.Processor { - return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), mediaManager, storage, db, emailSender) +func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender, mediaManager media.Manager, clientWorker *worker.Worker[messages.FromClientAPI], fedWorker *worker.Worker[messages.FromFederator]) processing.Processor { + return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), mediaManager, storage, db, emailSender, clientWorker, fedWorker) } diff --git a/testrig/testmodels.go b/testrig/testmodels.go @@ -42,6 +42,8 @@ import ( "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/messages" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // NewTestTokens returns a map of tokens keyed according to which account the token belongs to. @@ -505,7 +507,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account { } if diff := len(accounts) - len(preserializedKeys); diff > 0 { - var keyStrings = make([]string, diff) + keyStrings := make([]string, diff) for i := 0; i < diff; i++ { priv, _ := rsa.GenerateKey(rand.Reader, 2048) key, _ := x509.MarshalPKCS8PrivateKey(priv) @@ -1823,8 +1825,13 @@ func GetSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry }, } + // Create temporary federator worker for transport controller + fedWorker := worker.New[messages.FromFederator](-1, -1) + _ = fedWorker.Start() + defer func() { _ = fedWorker.Stop() }() + // use the client to create a new transport - c := NewTestTransportController(client, NewTestDB()) + c := NewTestTransportController(client, NewTestDB(), fedWorker) tp, err := c.NewTransport(pubKeyID, privkey) if err != nil { panic(err) @@ -1865,8 +1872,13 @@ func GetSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest }, } + // Create temporary federator worker for transport controller + fedWorker := worker.New[messages.FromFederator](-1, -1) + _ = fedWorker.Start() + defer func() { _ = fedWorker.Stop() }() + // use the client to create a new transport - c := NewTestTransportController(client, NewTestDB()) + c := NewTestTransportController(client, NewTestDB(), fedWorker) tp, err := c.NewTransport(pubKeyID, privkey) if err != nil { panic(err) @@ -1899,7 +1911,8 @@ func newAPPerson( avatarContentType string, headerURL *url.URL, headerContentType string, - manuallyApprovesFollowers bool) vocab.ActivityStreamsPerson { + manuallyApprovesFollowers bool, +) vocab.ActivityStreamsPerson { person := streams.NewActivityStreamsPerson() // id should be the activitypub URI of this user @@ -2082,7 +2095,8 @@ func newAPGroup( avatarContentType string, headerURL *url.URL, headerContentType string, - manuallyApprovesFollowers bool) vocab.ActivityStreamsGroup { + manuallyApprovesFollowers bool, +) vocab.ActivityStreamsGroup { group := streams.NewActivityStreamsGroup() // id should be the activitypub URI of this group @@ -2303,8 +2317,8 @@ func newAPNote( noteCC []*url.URL, noteSensitive bool, noteMentions []vocab.ActivityStreamsMention, - noteAttachments []vocab.ActivityStreamsImage) vocab.ActivityStreamsNote { - + noteAttachments []vocab.ActivityStreamsImage, +) vocab.ActivityStreamsNote { // create the note itself note := streams.NewActivityStreamsNote() diff --git a/testrig/transportcontroller.go b/testrig/transportcontroller.go @@ -26,7 +26,9 @@ import ( "github.com/superseriousbusiness/activity/pub" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/federation" + "github.com/superseriousbusiness/gotosocial/internal/messages" "github.com/superseriousbusiness/gotosocial/internal/transport" + "github.com/superseriousbusiness/gotosocial/internal/worker" ) // NewTestTransportController returns a test transport controller with the given http client. @@ -38,8 +40,8 @@ import ( // Unlike the other test interfaces provided in this package, you'll probably want to call this function // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular) // basis. -func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller { - return transport.NewController(db, NewTestFederatingDB(db), &federation.Clock{}, client) +func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *worker.Worker[messages.FromFederator]) transport.Controller { + return transport.NewController(db, NewTestFederatingDB(db, fedWorker), &federation.Clock{}, client) } // NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,