commit 37d310f981f3e26bc643aeabac134b591c84455a
parent 983e696bd6c77d06c79599b98185ed91c5ed62db
Author: tobi <31960611+tsmethurst@users.noreply.github.com>
Date: Tue, 29 Mar 2022 11:54:56 +0200
[feature] Dereference remote mentions when the account is not already known (#442)
* remove mention util function from db
* add ParseMentionFunc to gtsmodel
* add parseMentionFunc to processor
* refactor search to simplify it a bit
* add parseMentionFunc to account
* add parseMentionFunc to status
* some renaming for clarity
* test dereference of unknown mentioned account
Diffstat:
16 files changed, 420 insertions(+), 183 deletions(-)
diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go
@@ -19,8 +19,18 @@
package status_test
import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/activity/pub"
+ "github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@@ -73,7 +83,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
- suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage, suite.mediaManager)
+ suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db), suite.storage, suite.mediaManager)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager)
suite.statusModule = status.New(suite.processor).(*status.Module)
@@ -85,3 +95,89 @@ func (suite *StatusStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}
+
+func (suite *StatusStandardTestSuite) testHttpClient() pub.HttpClient {
+ remoteAccount := suite.testAccounts["remote_account_1"]
+ remoteAccountNamestring := fmt.Sprintf("acct:%s@%s", remoteAccount.Username, remoteAccount.Domain)
+ remoteAccountWebfingerURI := fmt.Sprintf("https://%s/.well-known/webfinger?resource=%s", remoteAccount.Domain, remoteAccountNamestring)
+
+ fmt.Println(remoteAccountWebfingerURI)
+
+ httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
+
+ // respond correctly to a webfinger lookup
+ if req.URL.String() == remoteAccountWebfingerURI {
+ responseJson := fmt.Sprintf(`
+ {
+ "subject": "%s",
+ "aliases": [
+ "%s",
+ "%s"
+ ],
+ "links": [
+ {
+ "rel": "http://webfinger.net/rel/profile-page",
+ "type": "text/html",
+ "href": "%s"
+ },
+ {
+ "rel": "self",
+ "type": "application/activity+json",
+ "href": "%s"
+ }
+ ]
+ }`, remoteAccountNamestring, remoteAccount.URI, remoteAccount.URL, remoteAccount.URL, remoteAccount.URI)
+ responseType := "application/json"
+
+ reader := bytes.NewReader([]byte(responseJson))
+ readCloser := io.NopCloser(reader)
+ response := &http.Response{
+ StatusCode: 200,
+ Body: readCloser,
+ ContentLength: int64(len(responseJson)),
+ Header: http.Header{
+ "content-type": {responseType},
+ },
+ }
+ return response, nil
+ }
+
+ // respond correctly to an account dereference
+ if req.URL.String() == remoteAccount.URI {
+ satanAS, err := suite.tc.AccountToAS(context.Background(), remoteAccount)
+ if err != nil {
+ panic(err)
+ }
+
+ satanI, err := streams.Serialize(satanAS)
+ if err != nil {
+ panic(err)
+ }
+ satanJson, err := json.Marshal(satanI)
+ if err != nil {
+ panic(err)
+ }
+ responseType := "application/activity+json"
+
+ reader := bytes.NewReader(satanJson)
+ readCloser := io.NopCloser(reader)
+ response := &http.Response{
+ StatusCode: 200,
+ Body: readCloser,
+ ContentLength: int64(len(satanJson)),
+ Header: http.Header{
+ "content-type": {responseType},
+ },
+ }
+ return response, nil
+ }
+
+ r := ioutil.NopCloser(bytes.NewReader([]byte{}))
+ return &http.Response{
+ StatusCode: 200,
+ Body: r,
+ }, nil
+ })
+
+ return httpClient
+}
diff --git a/internal/api/client/status/statuscreate_test.go b/internal/api/client/status/statuscreate_test.go
@@ -29,7 +29,6 @@ import (
"testing"
"github.com/gin-gonic/gin"
- "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
"github.com/superseriousbusiness/gotosocial/internal/api/model"
@@ -85,26 +84,69 @@ func (suite *StatusCreateTestSuite) TestPostNewStatus() {
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
statusReply := &model.Status{}
err = json.Unmarshal(b, statusReply)
- assert.NoError(suite.T(), err)
-
- assert.Equal(suite.T(), "hello hello", statusReply.SpoilerText)
- assert.Equal(suite.T(), "<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>", statusReply.Content)
- assert.True(suite.T(), statusReply.Sensitive)
- assert.Equal(suite.T(), model.VisibilityPrivate, statusReply.Visibility) // even though we set this status to mutuals only, it should serialize to private, because the mastodon api has no idea about mutuals_only
- assert.Len(suite.T(), statusReply.Tags, 1)
- assert.Equal(suite.T(), model.Tag{
+ suite.NoError(err)
+
+ suite.Equal("hello hello", statusReply.SpoilerText)
+ suite.Equal("<p>this is a brand new status! <a href=\"http://localhost:8080/tags/helloworld\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>helloworld</span></a></p>", statusReply.Content)
+ suite.True(statusReply.Sensitive)
+ suite.Equal(model.VisibilityPrivate, statusReply.Visibility) // even though we set this status to mutuals only, it should serialize to private, because the mastodon api has no idea about mutuals_only
+ suite.Len(statusReply.Tags, 1)
+ suite.Equal(model.Tag{
Name: "helloworld",
URL: "http://localhost:8080/tags/helloworld",
}, statusReply.Tags[0])
gtsTag := >smodel.Tag{}
err = suite.db.GetWhere(context.Background(), []db.Where{{Key: "name", Value: "helloworld"}}, gtsTag)
- assert.NoError(suite.T(), err)
- assert.Equal(suite.T(), statusReply.Account.ID, gtsTag.FirstSeenFromAccountID)
+ suite.NoError(err)
+ suite.Equal(statusReply.Account.ID, gtsTag.FirstSeenFromAccountID)
+}
+
+// mention an account that is not yet known to the instance -- it should be looked up and put in the db
+func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() {
+
+ // first remove remote account 1 from the database so it gets looked up again
+ remoteAccount := suite.testAccounts["remote_account_1"]
+ if err := suite.db.DeleteByID(context.Background(), remoteAccount.ID, >smodel.Account{}); err != nil {
+ panic(err)
+ }
+
+ t := suite.testTokens["local_account_1"]
+ oauthToken := oauth.DBTokenToToken(t)
+
+ // setup
+ recorder := httptest.NewRecorder()
+ ctx, _ := gin.CreateTestContext(recorder)
+ ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
+ ctx.Set(oauth.SessionAuthorizedToken, oauthToken)
+ ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
+ ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
+ ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil) // the endpoint we're hitting
+ ctx.Request.Header.Set("accept", "application/json")
+ ctx.Request.Form = url.Values{
+ "status": {"hello @foss_satan@fossbros-anonymous.io"},
+ "visibility": {string(model.VisibilityPublic)},
+ }
+ suite.statusModule.StatusCreatePOSTHandler(ctx)
+
+ suite.EqualValues(http.StatusOK, recorder.Code)
+
+ result := recorder.Result()
+ defer result.Body.Close()
+ b, err := ioutil.ReadAll(result.Body)
+ suite.NoError(err)
+
+ statusReply := &model.Status{}
+ err = json.Unmarshal(b, statusReply)
+ suite.NoError(err)
+
+ // if the status is properly formatted, that means the account has been put in the db
+ suite.Equal("<p>hello <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span></p>", statusReply.Content)
+ suite.Equal(model.VisibilityPublic, statusReply.Visibility)
}
func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() {
@@ -134,13 +176,13 @@ func (suite *StatusCreateTestSuite) TestPostAnotherNewStatus() {
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
statusReply := &model.Status{}
err = json.Unmarshal(b, statusReply)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
- assert.Equal(suite.T(), "<p><a href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>test</span></a> alright, should be able to post <a href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>links</span></a> with fragments in them now, let's see........<br><br><a href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"noopener nofollow noreferrer\" target=\"_blank\">docs.gotosocial.org/en/latest/user_guide/posts/#links</a><br><br><a href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>gotosocial</span></a><br><br>(tobi remember to pull the docker image challenge)</p>", statusReply.Content)
+ suite.Equal("<p><a href=\"http://localhost:8080/tags/test\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>test</span></a> alright, should be able to post <a href=\"http://localhost:8080/tags/links\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>links</span></a> with fragments in them now, let's see........<br><br><a href=\"https://docs.gotosocial.org/en/latest/user_guide/posts/#links\" rel=\"noopener nofollow noreferrer\" target=\"_blank\">docs.gotosocial.org/en/latest/user_guide/posts/#links</a><br><br><a href=\"http://localhost:8080/tags/gotosocial\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>gotosocial</span></a><br><br>(tobi remember to pull the docker image challenge)</p>", statusReply.Content)
}
func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
@@ -167,22 +209,22 @@ func (suite *StatusCreateTestSuite) TestPostNewStatusWithEmoji() {
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
statusReply := &model.Status{}
err = json.Unmarshal(b, statusReply)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
- assert.Equal(suite.T(), "", statusReply.SpoilerText)
- assert.Equal(suite.T(), "<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:<br>here's an emoji that isn't in the db: :test_emoji:</p>", statusReply.Content)
+ suite.Equal("", statusReply.SpoilerText)
+ suite.Equal("<p>here is a rainbow emoji a few times! :rainbow: :rainbow: :rainbow:<br>here's an emoji that isn't in the db: :test_emoji:</p>", statusReply.Content)
- assert.Len(suite.T(), statusReply.Emojis, 1)
+ suite.Len(statusReply.Emojis, 1)
apiEmoji := statusReply.Emojis[0]
gtsEmoji := testrig.NewTestEmojis()["rainbow"]
- assert.Equal(suite.T(), gtsEmoji.Shortcode, apiEmoji.Shortcode)
- assert.Equal(suite.T(), gtsEmoji.ImageURL, apiEmoji.URL)
- assert.Equal(suite.T(), gtsEmoji.ImageStaticURL, apiEmoji.StaticURL)
+ suite.Equal(gtsEmoji.Shortcode, apiEmoji.Shortcode)
+ suite.Equal(gtsEmoji.ImageURL, apiEmoji.URL)
+ suite.Equal(gtsEmoji.ImageStaticURL, apiEmoji.StaticURL)
}
// Try to reply to a status that doesn't exist
@@ -213,8 +255,8 @@ func (suite *StatusCreateTestSuite) TestReplyToNonexistentStatus() {
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
- assert.NoError(suite.T(), err)
- assert.Equal(suite.T(), `{"error":"bad request"}`, string(b))
+ suite.NoError(err)
+ suite.Equal(`{"error":"bad request"}`, string(b))
}
// Post a reply to the status of a local user that allows replies.
@@ -243,19 +285,19 @@ func (suite *StatusCreateTestSuite) TestReplyToLocalStatus() {
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
statusReply := &model.Status{}
err = json.Unmarshal(b, statusReply)
- assert.NoError(suite.T(), err)
-
- assert.Equal(suite.T(), "", statusReply.SpoilerText)
- assert.Equal(suite.T(), fmt.Sprintf("<p>hello <span class=\"h-card\"><a href=\"http://localhost:8080/@%s\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>%s</span></a></span> this reply should work!</p>", testrig.NewTestAccounts()["local_account_2"].Username, testrig.NewTestAccounts()["local_account_2"].Username), statusReply.Content)
- assert.False(suite.T(), statusReply.Sensitive)
- assert.Equal(suite.T(), model.VisibilityPublic, statusReply.Visibility)
- assert.Equal(suite.T(), testrig.NewTestStatuses()["local_account_2_status_1"].ID, statusReply.InReplyToID)
- assert.Equal(suite.T(), testrig.NewTestAccounts()["local_account_2"].ID, statusReply.InReplyToAccountID)
- assert.Len(suite.T(), statusReply.Mentions, 1)
+ suite.NoError(err)
+
+ suite.Equal("", statusReply.SpoilerText)
+ suite.Equal(fmt.Sprintf("<p>hello <span class=\"h-card\"><a href=\"http://localhost:8080/@%s\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>%s</span></a></span> this reply should work!</p>", testrig.NewTestAccounts()["local_account_2"].Username, testrig.NewTestAccounts()["local_account_2"].Username), statusReply.Content)
+ suite.False(statusReply.Sensitive)
+ suite.Equal(model.VisibilityPublic, statusReply.Visibility)
+ suite.Equal(testrig.NewTestStatuses()["local_account_2_status_1"].ID, statusReply.InReplyToID)
+ suite.Equal(testrig.NewTestAccounts()["local_account_2"].ID, statusReply.InReplyToAccountID)
+ suite.Len(statusReply.Mentions, 1)
}
// Take a media file which is currently not associated with a status, and attach it to a new status.
@@ -286,33 +328,33 @@ func (suite *StatusCreateTestSuite) TestAttachNewMediaSuccess() {
result := recorder.Result()
defer result.Body.Close()
b, err := ioutil.ReadAll(result.Body)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
statusResponse := &model.Status{}
err = json.Unmarshal(b, statusResponse)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
- assert.Equal(suite.T(), "", statusResponse.SpoilerText)
- assert.Equal(suite.T(), "<p>here's an image attachment</p>", statusResponse.Content)
- assert.False(suite.T(), statusResponse.Sensitive)
- assert.Equal(suite.T(), model.VisibilityPublic, statusResponse.Visibility)
+ suite.Equal("", statusResponse.SpoilerText)
+ suite.Equal("<p>here's an image attachment</p>", statusResponse.Content)
+ suite.False(statusResponse.Sensitive)
+ suite.Equal(model.VisibilityPublic, statusResponse.Visibility)
// there should be one media attachment
- assert.Len(suite.T(), statusResponse.MediaAttachments, 1)
+ suite.Len(statusResponse.MediaAttachments, 1)
// get the updated media attachment from the database
gtsAttachment, err := suite.db.GetAttachmentByID(context.Background(), statusResponse.MediaAttachments[0].ID)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
// convert it to a api attachment
gtsAttachmentAsapi, err := suite.tc.AttachmentToAPIAttachment(context.Background(), gtsAttachment)
- assert.NoError(suite.T(), err)
+ suite.NoError(err)
// compare it with what we have now
- assert.EqualValues(suite.T(), statusResponse.MediaAttachments[0], gtsAttachmentAsapi)
+ suite.EqualValues(statusResponse.MediaAttachments[0], gtsAttachmentAsapi)
// the status id of the attachment should now be set to the id of the status we just created
- assert.Equal(suite.T(), statusResponse.ID, gtsAttachment.StatusID)
+ suite.Equal(statusResponse.ID, gtsAttachment.StatusID)
}
func TestStatusCreateTestSuite(t *testing.T) {
diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go
@@ -382,86 +382,6 @@ func tweakConnectionValues(sqldb *sql.DB) {
CONVERSION FUNCTIONS
*/
-// TODO: move these to the type converter, it's bananas that they're here and not there
-
-func (ps *bunDBService) MentionStringsToMentions(ctx context.Context, targetAccounts []string, originAccountID string, statusID string) ([]*gtsmodel.Mention, error) {
- ogAccount := >smodel.Account{}
- if err := ps.conn.NewSelect().Model(ogAccount).Where("id = ?", originAccountID).Scan(ctx); err != nil {
- return nil, err
- }
-
- menchies := []*gtsmodel.Mention{}
- for _, a := range targetAccounts {
- // A mentioned account looks like "@test@example.org" or just "@test" for a local account
- // -- we can guarantee this from the regex that targetAccounts should have been derived from.
- // But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given).
-
- // 1. trim off the first @
- t := strings.TrimPrefix(a, "@")
-
- // 2. split the username and domain
- s := strings.Split(t, "@")
-
- // 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong
- var local bool
- switch len(s) {
- case 1:
- local = true
- case 2:
- local = false
- default:
- return nil, fmt.Errorf("mentioned account format '%s' was not valid", a)
- }
-
- var username, domain string
- username = s[0]
- if !local {
- domain = s[1]
- }
-
- // 4. check we now have a proper username and domain
- if username == "" || (!local && domain == "") {
- return nil, fmt.Errorf("username or domain for '%s' was nil", a)
- }
-
- // okay we're good now, we can start pulling accounts out of the database
- mentionedAccount := >smodel.Account{}
- var err error
-
- // match username + account, case insensitive
- if local {
- // local user -- should have a null domain
- err = ps.conn.NewSelect().Model(mentionedAccount).Where("LOWER(?) = LOWER(?)", bun.Ident("username"), username).Where("? IS NULL", bun.Ident("domain")).Scan(ctx)
- } else {
- // remote user -- should have domain defined
- err = ps.conn.NewSelect().Model(mentionedAccount).Where("LOWER(?) = LOWER(?)", bun.Ident("username"), username).Where("LOWER(?) = LOWER(?)", bun.Ident("domain"), domain).Scan(ctx)
- }
-
- if err != nil {
- if err == sql.ErrNoRows {
- // no result found for this username/domain so just don't include it as a mencho and carry on about our business
- logrus.Debugf("no account found with username '%s' and domain '%s', skipping it", username, domain)
- continue
- }
- // a serious error has happened so bail
- return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err)
- }
-
- // id, createdAt and updatedAt will be populated by the db, so we have everything we need!
- menchies = append(menchies, >smodel.Mention{
- StatusID: statusID,
- OriginAccountID: ogAccount.ID,
- OriginAccountURI: ogAccount.URI,
- TargetAccountID: mentionedAccount.ID,
- NameString: a,
- TargetAccountURI: mentionedAccount.URI,
- TargetAccountURL: mentionedAccount.URL,
- OriginAccount: mentionedAccount,
- })
- }
- return menchies, nil
-}
-
func (ps *bunDBService) TagStringsToTags(ctx context.Context, tags []string, originAccountID string) ([]*gtsmodel.Tag, error) {
protocol := viper.GetString(config.Keys.Protocol)
host := viper.GetString(config.Keys.Host)
diff --git a/internal/db/db.go b/internal/db/db.go
@@ -48,15 +48,6 @@ type DB interface {
USEFUL CONVERSION FUNCTIONS
*/
- // MentionStringsToMentions takes a slice of deduplicated, lowercase account names in the form "@test@whatever.example.org" for a remote account,
- // or @test for a local account, which have been mentioned in a status.
- // It takes the id of the account that wrote the status, and the id of the status itself, and then
- // checks in the database for the mentioned accounts, and returns a slice of mentions generated based on the given parameters.
- //
- // Note: this func doesn't/shouldn't do any manipulation of the accounts in the DB, it's just for checking
- // if they exist in the db and conveniently returning them if they do.
- MentionStringsToMentions(ctx context.Context, targetAccounts []string, originAccountID string, statusID string) ([]*gtsmodel.Mention, error)
-
// TagStringsToTags takes a slice of deduplicated, lowercase tags in the form "somehashtag", which have been
// used in a status. It takes the id of the account that wrote the status, and the id of the status itself, and then
// returns a slice of *model.Tag corresponding to the given tags. If the tag already exists in database, that tag
diff --git a/internal/gtsmodel/mention.go b/internal/gtsmodel/mention.go
@@ -18,7 +18,10 @@
package gtsmodel
-import "time"
+import (
+ "context"
+ "time"
+)
// Mention refers to the 'tagging' or 'mention' of a user within a status.
type Mention struct {
@@ -57,3 +60,15 @@ type Mention struct {
TargetAccountURL string `validate:"-" bun:"-"`
// A pointer to the gtsmodel account of the mentioned account.
}
+
+// ParseMentionFunc describes a function that takes a lowercase account name
+// in the form "@test@whatever.example.org" for a remote account, or "@test"
+// for a local account, and returns a fully populated mention for that account,
+// with the given origin status ID and origin account ID.
+//
+// If the account is remote and not yet found in the database, then ParseMentionFunc
+// will try to webfinger the remote account and put it in the database before returning.
+//
+// Mentions generated by this function are not put in the database, that's still up to
+// the caller to do.
+type ParseMentionFunc func(ctx context.Context, targetAccount string, originAccountID string, statusID string) (*Mention, error)
diff --git a/internal/processing/account/account.go b/internal/processing/account/account.go
@@ -87,10 +87,11 @@ type processor struct {
formatter text.Formatter
db db.DB
federator federation.Federator
+ parseMention gtsmodel.ParseMentionFunc
}
// New returns a new account processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauthServer oauth.Server, fromClientAPI chan messages.FromClientAPI, federator federation.Federator, parseMention gtsmodel.ParseMentionFunc) Processor {
return &processor{
tc: tc,
mediaManager: mediaManager,
@@ -100,5 +101,6 @@ func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, oauth
formatter: text.NewFormatter(db),
db: db,
federator: federator,
+ parseMention: parseMention,
}
}
diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go
@@ -29,6 +29,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
+ "github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/processing/account"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
@@ -88,7 +89,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)
- suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator)
+ suite.accountProcessor = account.New(suite.db, suite.tc, suite.mediaManager, suite.oauthServer, suite.fromClientAPIChan, suite.federator, processing.GetParseMentionFunc(suite.db, suite.federator))
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
}
diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go
@@ -199,10 +199,14 @@ func (p *processor) processNote(ctx context.Context, note string, accountID stri
return "", err
}
- mentionStrings := util.DeriveMentionsFromText(note)
- mentions, err := p.db.MentionStringsToMentions(ctx, mentionStrings, accountID, "")
- if err != nil {
- return "", err
+ mentionStrings := util.DeriveMentionNamesFromText(note)
+ mentions := []*gtsmodel.Mention{}
+ for _, mentionString := range mentionStrings {
+ mention, err := p.parseMention(ctx, mentionString, accountID, "")
+ if err != nil {
+ continue
+ }
+ mentions = append(mentions, mention)
}
// TODO: support emojis in account notes
diff --git a/internal/processing/processor.go b/internal/processing/processor.go
@@ -267,12 +267,14 @@ func NewProcessor(
storage *kv.KVStore,
db db.DB,
emailSender email.Sender) Processor {
+
fromClientAPI := make(chan messages.FromClientAPI, 1000)
fromFederator := make(chan messages.FromFederator, 1000)
+ parseMentionFunc := GetParseMentionFunc(db, federator)
- statusProcessor := status.New(db, tc, fromClientAPI)
+ statusProcessor := status.New(db, tc, fromClientAPI, parseMentionFunc)
streamingProcessor := streaming.New(db, oauthServer)
- accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator)
+ accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator, parseMentionFunc)
adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI)
mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage)
userProcessor := user.New(db, emailSender)
diff --git a/internal/processing/search.go b/internal/processing/search.go
@@ -194,20 +194,15 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au
// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
if resolve {
// we're allowed to resolve it so let's try
-
// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
acctURI, err := p.federator.FingerRemoteAccount(ctx, authed.Account.Username, username, domain)
if err != nil {
// something went wrong doing the webfinger lookup so we can't process the request
- return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err)
+ return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
}
- // we don't have it locally so try and dereference it
- account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true)
- if err != nil {
- return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err)
- }
- return account, nil
+ // return the attempt to get the remove account
+ return p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true)
}
return nil, nil
diff --git a/internal/processing/status/status.go b/internal/processing/status/status.go
@@ -74,15 +74,17 @@ type processor struct {
filter visibility.Filter
formatter text.Formatter
fromClientAPI chan messages.FromClientAPI
+ parseMention gtsmodel.ParseMentionFunc
}
// New returns a new status processor.
-func New(db db.DB, tc typeutils.TypeConverter, fromClientAPI chan messages.FromClientAPI) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, fromClientAPI chan messages.FromClientAPI, parseMention gtsmodel.ParseMentionFunc) Processor {
return &processor{
tc: tc,
db: db,
filter: visibility.NewFilter(db),
formatter: text.NewFormatter(db),
fromClientAPI: fromClientAPI,
+ parseMention: parseMention,
}
}
diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go
@@ -19,11 +19,16 @@
package status_test
import (
+ "codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/messages"
+ "github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/processing/status"
+ "github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@@ -32,6 +37,10 @@ type StatusStandardTestSuite struct {
suite.Suite
db db.DB
typeConverter typeutils.TypeConverter
+ tc transport.Controller
+ storage *kv.KVStore
+ mediaManager media.Manager
+ federator federation.Federator
fromClientAPIChan chan messages.FromClientAPI
// standard suite models
@@ -62,17 +71,23 @@ func (suite *StatusStandardTestSuite) SetupSuite() {
}
func (suite *StatusStandardTestSuite) SetupTest() {
- testrig.InitTestLog()
testrig.InitTestConfig()
+ testrig.InitTestLog()
suite.db = testrig.NewTestDB()
suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
- suite.status = status.New(suite.db, suite.typeConverter, suite.fromClientAPIChan)
+ suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+ suite.storage = testrig.NewTestStorage()
+ suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
+ suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager)
+ suite.status = status.New(suite.db, suite.typeConverter, suite.fromClientAPIChan, processing.GetParseMentionFunc(suite.db, suite.federator))
- testrig.StandardDBSetup(suite.db, nil)
+ testrig.StandardDBSetup(suite.db, suite.testAccounts)
+ testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
}
func (suite *StatusStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
+ testrig.StandardStorageTeardown(suite.storage)
}
diff --git a/internal/processing/status/util.go b/internal/processing/status/util.go
@@ -23,10 +23,10 @@ import (
"errors"
"fmt"
+ "github.com/sirupsen/logrus"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
- "github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/text"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
@@ -192,27 +192,30 @@ func (p *processor) ProcessLanguage(ctx context.Context, form *apimodel.Advanced
}
func (p *processor) ProcessMentions(ctx context.Context, form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
- menchies := []string{}
- gtsMenchies, err := p.db.MentionStringsToMentions(ctx, util.DeriveMentionsFromText(form.Status), accountID, status.ID)
- if err != nil {
- return fmt.Errorf("error generating mentions from status: %s", err)
- }
- for _, menchie := range gtsMenchies {
- menchieID, err := id.NewRandomULID()
+ mentionedAccountNames := util.DeriveMentionNamesFromText(form.Status)
+ mentions := []*gtsmodel.Mention{}
+ mentionIDs := []string{}
+
+ for _, mentionedAccountName := range mentionedAccountNames {
+ gtsMention, err := p.parseMention(ctx, mentionedAccountName, accountID, status.ID)
if err != nil {
- return err
+ logrus.Errorf("ProcessMentions: error parsing mention %s from status: %s", mentionedAccountName, err)
+ continue
}
- menchie.ID = menchieID
- if err := p.db.Put(ctx, menchie); err != nil {
- return fmt.Errorf("error putting mentions in db: %s", err)
+ if err := p.db.Put(ctx, gtsMention); err != nil {
+ logrus.Errorf("ProcessMentions: error putting mention in db: %s", err)
}
- menchies = append(menchies, menchie.ID)
+
+ mentions = append(mentions, gtsMention)
+ mentionIDs = append(mentionIDs, gtsMention.ID)
}
+
// add full populated gts menchies to the status for passing them around conveniently
- status.Mentions = gtsMenchies
+ status.Mentions = mentions
// add just the ids of the mentioned accounts to the status for putting in the db
- status.MentionIDs = menchies
+ status.MentionIDs = mentionIDs
+
return nil
}
diff --git a/internal/processing/util.go b/internal/processing/util.go
@@ -0,0 +1,151 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package processing
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/federation"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/id"
+)
+
+func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.ParseMentionFunc {
+ return func(ctx context.Context, targetAccount string, originAccountID string, statusID string) (*gtsmodel.Mention, error) {
+ // get the origin account first since we'll need it to create the mention
+ originAccount, err := dbConn.GetAccountByID(ctx, originAccountID)
+ if err != nil {
+ return nil, fmt.Errorf("couldn't get mention origin account with id %s", originAccountID)
+ }
+
+ // A mentioned account looks like "@test@example.org" or just "@test" for a local account
+ // -- we can guarantee this from the regex that targetAccounts should have been derived from.
+ // But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given).
+
+ // 1. trim off the first @
+ trimmed := strings.TrimPrefix(targetAccount, "@")
+
+ // 2. split the username and domain
+ split := strings.Split(trimmed, "@")
+
+ // 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong
+
+ var local bool
+ switch len(split) {
+ case 1:
+ local = true
+ case 2:
+ local = false
+ default:
+ return nil, fmt.Errorf("mentioned account format '%s' was not valid", targetAccount)
+ }
+
+ var username, domain string
+ username = split[0]
+ if !local {
+ domain = split[1]
+ }
+
+ // 4. check we now have a proper username and domain
+ if username == "" || (!local && domain == "") {
+ return nil, fmt.Errorf("username or domain for '%s' was nil", targetAccount)
+ }
+
+ var mentionedAccount *gtsmodel.Account
+
+ if local {
+ localAccount, err := dbConn.GetLocalAccountByUsername(ctx, username)
+ if err != nil {
+ return nil, err
+ }
+ mentionedAccount = localAccount
+ } else {
+ remoteAccount := >smodel.Account{}
+
+ where := []db.Where{
+ {
+ Key: "username",
+ Value: username,
+ CaseInsensitive: true,
+ },
+ {
+ Key: "domain",
+ Value: domain,
+ CaseInsensitive: true,
+ },
+ }
+
+ err := dbConn.GetWhere(ctx, where, remoteAccount)
+ if err == nil {
+ // the account was already in the database
+ mentionedAccount = remoteAccount
+ } else {
+ // we couldn't get it from the database
+ if err != db.ErrNoEntries {
+ // a serious error has happened so bail
+ return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err)
+ }
+
+ // We just don't have the account, so try webfingering it.
+ //
+ // If the mention originates from our instance we should use the username of the origin account to do the dereferencing,
+ // otherwise we should just use our instance account (that is, provide an empty string), since obviously we can't use
+ // a remote account to do remote dereferencing!
+ var fingeringUsername string
+ if originAccount.Domain == "" {
+ fingeringUsername = originAccount.Username
+ }
+
+ acctURI, err := federator.FingerRemoteAccount(ctx, fingeringUsername, username, domain)
+ if err != nil {
+ // something went wrong doing the webfinger lookup so we can't process the request
+ return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
+ }
+
+ resolvedAccount, err := federator.GetRemoteAccount(ctx, fingeringUsername, acctURI, true, true)
+ if err != nil {
+ return nil, fmt.Errorf("error dereferencing account with uri %s: %s", acctURI.String(), err)
+ }
+
+ // we were able to resolve it!
+ mentionedAccount = resolvedAccount
+ }
+ }
+
+ mentionID, err := id.NewRandomULID()
+ if err != nil {
+ return nil, err
+ }
+
+ return >smodel.Mention{
+ ID: mentionID,
+ StatusID: statusID,
+ OriginAccountID: originAccount.ID,
+ OriginAccountURI: originAccount.URI,
+ TargetAccountID: mentionedAccount.ID,
+ NameString: targetAccount,
+ TargetAccountURI: mentionedAccount.URI,
+ TargetAccountURL: mentionedAccount.URL,
+ OriginAccount: mentionedAccount,
+ }, nil
+ }
+}
diff --git a/internal/util/statustools.go b/internal/util/statustools.go
@@ -25,13 +25,11 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/regexes"
)
-// DeriveMentionsFromText takes a plaintext (ie., not html-formatted) text,
-// and applies a regex to it to return a deduplicated list of accounts
-// mentioned in that text.
-//
-// It will look for fully-qualified account names in the form "@user@example.org".
-// or the form "@username" for local users.
-func DeriveMentionsFromText(text string) []string {
+// DeriveMentionNamesFromText takes a plaintext (ie., not html-formatted) text,
+// and applies a regex to it to return a deduplicated list of account names
+// mentioned in that text, in the format "@user@example.org" or "@username" for
+// local users.
+func DeriveMentionNamesFromText(text string) []string {
mentionedAccounts := []string{}
for _, m := range regexes.MentionFinder.FindAllStringSubmatch(text, -1) {
mentionedAccounts = append(mentionedAccounts, m[1])
diff --git a/internal/util/statustools_test.go b/internal/util/statustools_test.go
@@ -37,7 +37,7 @@ https://localhost:8080/@the_mighty_zork/statuses/01FGVP55XMF2K6316MQRX6PFG1
that link shouldn't come out formatted as a mention!`
- menchies := util.DeriveMentionsFromText(statusText)
+ menchies := util.DeriveMentionNamesFromText(statusText)
suite.Empty(menchies)
}
@@ -56,7 +56,7 @@ func (suite *StatusTestSuite) TestDeriveMentionsOK() {
`
- menchies := util.DeriveMentionsFromText(statusText)
+ menchies := util.DeriveMentionNamesFromText(statusText)
assert.Len(suite.T(), menchies, 6)
assert.Equal(suite.T(), "@dumpsterqueer@example.org", menchies[0])
assert.Equal(suite.T(), "@someone_else@testing.best-horse.com", menchies[1])
@@ -68,7 +68,7 @@ func (suite *StatusTestSuite) TestDeriveMentionsOK() {
func (suite *StatusTestSuite) TestDeriveMentionsEmpty() {
statusText := ``
- menchies := util.DeriveMentionsFromText(statusText)
+ menchies := util.DeriveMentionNamesFromText(statusText)
assert.Len(suite.T(), menchies, 0)
}
@@ -126,7 +126,7 @@ func (suite *StatusTestSuite) TestDeriveMultiple() {
Text`
- ms := util.DeriveMentionsFromText(statusText)
+ ms := util.DeriveMentionNamesFromText(statusText)
hs := util.DeriveHashtagsFromText(statusText)
es := util.DeriveEmojisFromText(statusText)
