commit 0fe853b1ee644703e2273ed4ce331cfd377af268 parent cb54324430e7c4640a7aa5aac24932b5e91b71c4 Author: tsmethurst <tobi.smethurst@klarrio.com> Date: Sat, 29 May 2021 19:36:54 +0200 first implementation of search feature Diffstat:
20 files changed, 763 insertions(+), 19 deletions(-)
diff --git a/PROGRESS.md b/PROGRESS.md @@ -100,7 +100,7 @@ * [ ] Timelines * [ ] /api/v1/timelines/public GET (See the public/federated timeline) * [ ] /api/v1/timelines/tag/:hashtag GET (Get public statuses that use hashtag) - * [ ] /api/v1/timelines/home GET (View statuses from followed users) + * [x] /api/v1/timelines/home GET (View statuses from followed users) * [ ] /api/v1/timelines/list/:list_id GET (Get statuses in given list) * [ ] Conversations * [ ] /api/v1/conversations GET (Get a list of direct message convos) @@ -121,8 +121,8 @@ * [ ] Streaming * [ ] /api/v1/streaming WEBSOCKETS (Stream live events to user via websockets) * [ ] Notifications - * [ ] /api/v1/notifications GET (Get list of notifications) - * [ ] /api/v1/notifications/:id GET (Get a single notification) + * [x] /api/v1/notifications GET (Get list of notifications) + * [x] /api/v1/notifications/:id GET (Get a single notification) * [ ] /api/v1/notifications/clear POST (Clear all notifications) * [ ] /api/v1/notifications/:id POST (Clear a single notification) * [ ] Push @@ -130,8 +130,8 @@ * [ ] /api/v1/push/subscription GET (Get current subscription) * [ ] /api/v1/push/subscription PUT (Change notification types) * [ ] /api/v1/push/subscription DELETE (Delete current subscription) - * [ ] Search - * [ ] /api/v2/search GET (Get search query results) + * [x] Search + * [x] /api/v2/search GET (Get search query results) * [ ] Instance * [x] /api/v1/instance GET (Get instance information) * [ ] /api/v1/instance PATCH (Update instance information) @@ -174,7 +174,7 @@ * [ ] Federation modes * [ ] 'Slow' federation * [ ] Reputation scoring system for instances - * [ ] 'Greedy' federation + * [x] 'Greedy' federation * [ ] No federation (insulate this instance from the Fediverse) * [ ] Allowlist * [x] Secure HTTP signatures (creation and validation) diff --git a/internal/api/client/search/search.go b/internal/api/client/search/search.go @@ -0,0 +1,88 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package search + +import ( + "net/http" + + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/api" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/message" + "github.com/superseriousbusiness/gotosocial/internal/router" +) + +const ( + // BasePath is the base path for serving v1 of the search API + BasePathV1 = "/api/v1/search" + + // BasePathV2 is the base path for serving v2 of the search API + BasePathV2 = "/api/v2/search" + + // AccountIDKey -- If provided, statuses returned will be authored only by this account + AccountIDKey = "account_id" + // MaxIDKey -- Return results older than this id + MaxIDKey = "max_id" + // MinIDKey -- Return results immediately newer than this id + MinIDKey = "min_id" + // TypeKey -- Enum(accounts, hashtags, statuses) + TypeKey = "type" + // ExcludeUnreviewedKey -- Filter out unreviewed tags? Defaults to false. Use true when trying to find trending tags. + ExcludeUnreviewedKey = "exclude_unreviewed" + // QueryKey -- The search query + QueryKey = "q" + // ResolveKey -- Attempt WebFinger lookup. Defaults to false. + ResolveKey = "resolve" + // LimitKey -- Maximum number of results to load, per type. Defaults to 20. Max 40. + LimitKey = "limit" + // OffsetKey -- Offset in search results. Used for pagination. Defaults to 0. + OffsetKey = "offset" + // FollowingKey -- Only include accounts that the user is following. Defaults to false. + FollowingKey = "following" + + // TypeAccounts -- + TypeAccounts = "accounts" + // TypeHashtags -- + TypeHashtags = "hashtags" + // TypeStatuses -- + TypeStatuses = "statuses" +) + +// Module implements the ClientAPIModule interface for everything related to searching +type Module struct { + config *config.Config + processor message.Processor + log *logrus.Logger +} + +// New returns a new search module +func New(config *config.Config, processor message.Processor, log *logrus.Logger) api.ClientModule { + return &Module{ + config: config, + processor: processor, + log: log, + } +} + +// Route attaches all routes from this module to the given router +func (m *Module) Route(r router.Router) error { + r.AttachHandler(http.MethodGet, BasePathV1, m.SearchGETHandler) + r.AttachHandler(http.MethodGet, BasePathV2, m.SearchGETHandler) + return nil +} diff --git a/internal/api/client/search/searchget.go b/internal/api/client/search/searchget.go @@ -0,0 +1,151 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package search + +import ( + "fmt" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/oauth" +) + +// SearchGETHandler handles searches for local and remote accounts, statuses, and hashtags. +// It corresponds to the mastodon endpoint described here: https://docs.joinmastodon.org/methods/search/ +func (m *Module) SearchGETHandler(c *gin.Context) { + l := m.log.WithFields(logrus.Fields{ + "func": "SearchGETHandler", + "request_uri": c.Request.RequestURI, + "user_agent": c.Request.UserAgent(), + "origin_ip": c.ClientIP(), + }) + l.Debugf("entering function") + + authed, err := oauth.Authed(c, true, true, true, true) // we don't really need an app here but we want everything else + if err != nil { + l.Errorf("error authing search request: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "not authed"}) + return + } + + accountID := c.Query(AccountIDKey) + maxID := c.Query(MaxIDKey) + minID := c.Query(MinIDKey) + searchType := c.Query(TypeKey) + + excludeUnreviewed := false + excludeUnreviewedString := c.Query(ExcludeUnreviewedKey) + if excludeUnreviewedString != "" { + var err error + excludeUnreviewed, err = strconv.ParseBool(excludeUnreviewedString) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", excludeUnreviewedString, err)}) + return + } + } + + query := c.Query(QueryKey) + if query == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "query parameter q was empty"}) + return + } + + resolve := false + resolveString := c.Query(ResolveKey) + if resolveString != "" { + var err error + resolve, err = strconv.ParseBool(resolveString) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", resolveString, err)}) + return + } + } + + limit := 20 + limitString := c.Query(LimitKey) + if limitString != "" { + i, err := strconv.ParseInt(limitString, 10, 64) + if err != nil { + l.Debugf("error parsing limit string: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse limit query param"}) + return + } + limit = int(i) + } + if limit > 40 { + limit = 40 + } + if limit < 1 { + limit = 1 + } + + offset := 0 + offsetString := c.Query(OffsetKey) + if offsetString != "" { + i, err := strconv.ParseInt(offsetString, 10, 64) + if err != nil { + l.Debugf("error parsing offset string: %s", err) + c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse offset query param"}) + return + } + offset = int(i) + } + if limit > 40 { + limit = 40 + } + if limit < 1 { + limit = 1 + } + + following := false + followingString := c.Query(FollowingKey) + if followingString != "" { + var err error + following, err = strconv.ParseBool(followingString) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("couldn't parse param %s: %s", followingString, err)}) + return + } + } + + searchQuery := &model.SearchQuery{ + AccountID: accountID, + MaxID: maxID, + MinID: minID, + Type: searchType, + ExcludeUnreviewed: excludeUnreviewed, + Query: query, + Resolve: resolve, + Limit: limit, + Offset: offset, + Following: following, + } + + results, errWithCode := m.processor.SearchGet(authed, searchQuery) + if errWithCode != nil { + l.Debugf("error searching: %s", errWithCode.Error()) + c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()}) + return + } + + c.JSON(http.StatusOK, results) +} diff --git a/internal/api/model/search.go b/internal/api/model/search.go @@ -0,0 +1,52 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package model + +// SearchQuery corresponds to search parameters as submitted through the client API. +// See https://docs.joinmastodon.org/methods/search/ +type SearchQuery struct { + // If provided, statuses returned will be authored only by this account + AccountID string + // Return results older than this id + MaxID string + // Return results immediately newer than this id + MinID string + // Enum(accounts, hashtags, statuses) + Type string + // Filter out unreviewed tags? Defaults to false. Use true when trying to find trending tags. + ExcludeUnreviewed bool + // The search query + Query string + // Attempt WebFinger lookup. Defaults to false. + Resolve bool + // Maximum number of results to load, per type. Defaults to 20. Max 40. + Limit int + // Offset in search results. Used for pagination. Defaults to 0. + Offset int + // Only include accounts that the user is following. Defaults to false. + Following bool +} + +// SearchResult corresponds to a search result, containing accounts, statuses, and hashtags. +// See https://docs.joinmastodon.org/methods/search/ +type SearchResult struct { + Accounts []Account `json:"accounts"` + Statuses []Status `json:"statuses"` + Hashtags []Tag `json:"hashtags"` +} diff --git a/internal/db/db.go b/internal/db/db.go @@ -45,8 +45,9 @@ func (e ErrAlreadyExists) Error() string { } type Where struct { - Key string - Value interface{} + Key string + Value interface{} + CaseInsensitive bool } // DB provides methods for interacting with an underlying database or other storage mechanism (for now, just postgres). diff --git a/internal/db/pg/pg.go b/internal/db/pg/pg.go @@ -223,7 +223,12 @@ func (ps *postgresService) GetWhere(where []db.Where, i interface{}) error { q := ps.conn.Model(i) for _, w := range where { - q = q.Where("? = ?", pg.Safe(w.Key), w.Value) + if w.CaseInsensitive { + q = q.Where("LOWER(?) = LOWER(?)", pg.Safe(w.Key), w.Value) + } else { + q = q.Where("? = ?", pg.Safe(w.Key), w.Value) + } + } if err := q.Select(); err != nil { diff --git a/internal/federation/commonbehavior.go b/internal/federation/commonbehavior.go @@ -25,6 +25,7 @@ import ( "net/url" "github.com/go-fed/activity/pub" + "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/util" @@ -59,7 +60,7 @@ import ( func (f *federator) AuthenticateGetInbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, bool, error) { // IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through // the CLIENT API, not through the federation API, so we just do nothing here. - return nil, false, nil + return ctx, false, nil } // AuthenticateGetOutbox delegates the authentication of a GET to an @@ -84,7 +85,7 @@ func (f *federator) AuthenticateGetInbox(ctx context.Context, w http.ResponseWri func (f *federator) AuthenticateGetOutbox(ctx context.Context, w http.ResponseWriter, r *http.Request) (context.Context, bool, error) { // IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through // the CLIENT API, not through the federation API, so we just do nothing here. - return nil, false, nil + return ctx, false, nil } // GetOutbox returns the OrderedCollection inbox of the actor for this @@ -98,7 +99,7 @@ func (f *federator) AuthenticateGetOutbox(ctx context.Context, w http.ResponseWr func (f *federator) GetOutbox(ctx context.Context, r *http.Request) (vocab.ActivityStreamsOrderedCollectionPage, error) { // IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through // the CLIENT API, not through the federation API, so we just do nothing here. - return nil, nil + return streams.NewActivityStreamsOrderedCollectionPage(), nil } // NewTransport returns a new Transport on behalf of a specific actor. diff --git a/internal/federation/federatingdb/followers.go b/internal/federation/federatingdb/followers.go @@ -10,6 +10,7 @@ import ( "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" ) // Followers obtains the Followers Collection for an actor with the @@ -28,8 +29,17 @@ func (f *federatingDB) Followers(c context.Context, actorIRI *url.URL) (follower l.Debugf("entering FOLLOWERS function with actorIRI %s", actorIRI.String()) acct := >smodel.Account{} - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil { - return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err) + + if util.IsUserPath(actorIRI) { + if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil { + return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err) + } + } else if util.IsFollowersPath(actorIRI) { + if err := f.db.GetWhere([]db.Where{{Key: "followers_uri", Value: actorIRI.String()}}, acct); err != nil { + return nil, fmt.Errorf("db error getting account with followers uri %s: %s", actorIRI.String(), err) + } + } else { + return nil, fmt.Errorf("could not parse actor IRI %s as users or followers path", actorIRI.String()) } acctFollowers := []gtsmodel.Follow{} diff --git a/internal/federation/federatingdb/following.go b/internal/federation/federatingdb/following.go @@ -10,6 +10,7 @@ import ( "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/util" ) // Following obtains the Following Collection for an actor with the @@ -28,8 +29,16 @@ func (f *federatingDB) Following(c context.Context, actorIRI *url.URL) (followin l.Debugf("entering FOLLOWING function with actorIRI %s", actorIRI.String()) acct := >smodel.Account{} - if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil { - return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err) + if util.IsUserPath(actorIRI) { + if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: actorIRI.String()}}, acct); err != nil { + return nil, fmt.Errorf("db error getting account with uri %s: %s", actorIRI.String(), err) + } + } else if util.IsFollowingPath(actorIRI) { + if err := f.db.GetWhere([]db.Where{{Key: "following_uri", Value: actorIRI.String()}}, acct); err != nil { + return nil, fmt.Errorf("db error getting account with following uri %s: %s", actorIRI.String(), err) + } + } else { + return nil, fmt.Errorf("could not parse actor IRI %s as users or following path", actorIRI.String()) } acctFollowing := []gtsmodel.Follow{} diff --git a/internal/federation/federatingdb/lock.go b/internal/federation/federatingdb/lock.go @@ -42,6 +42,10 @@ func (f *federatingDB) Lock(c context.Context, id *url.URL) error { // Strategy: create a new lock, if stored, continue. Otherwise, lock the // existing mutex. + if id == nil { + return errors.New("Lock: id was nil") + } + mu := &sync.Mutex{} mu.Lock() // Optimistically lock if we do store it. i, loaded := f.locks.LoadOrStore(id.String(), mu) @@ -59,6 +63,9 @@ func (f *federatingDB) Lock(c context.Context, id *url.URL) error { func (f *federatingDB) Unlock(c context.Context, id *url.URL) error { // Once Go-Fed is done calling Database methods, the relevant `id` // entries are unlocked. + if id == nil { + return errors.New("Unlock: id was nil") + } i, ok := f.locks.Load(id.String()) if !ok { diff --git a/internal/federation/federatingprotocol.go b/internal/federation/federatingprotocol.go @@ -26,6 +26,7 @@ import ( "net/url" "github.com/go-fed/activity/pub" + "github.com/go-fed/activity/streams" "github.com/go-fed/activity/streams/vocab" "github.com/sirupsen/logrus" "github.com/superseriousbusiness/gotosocial/internal/db" @@ -310,7 +311,7 @@ func (f *federator) MaxDeliveryRecursionDepth(ctx context.Context) int { // logic to be used, but the implementation must not modify it. func (f *federator) FilterForwarding(ctx context.Context, potentialRecipients []*url.URL, a pub.Activity) ([]*url.URL, error) { // TODO - return nil, nil + return []*url.URL{}, nil } // GetInbox returns the OrderedCollection inbox of the actor for this @@ -324,5 +325,5 @@ func (f *federator) FilterForwarding(ctx context.Context, potentialRecipients [] func (f *federator) GetInbox(ctx context.Context, r *http.Request) (vocab.ActivityStreamsOrderedCollectionPage, error) { // IMPLEMENTATION NOTE: For GoToSocial, we serve GETS to outboxes and inboxes through // the CLIENT API, not through the federation API, so we just do nothing here. - return nil, nil + return streams.NewActivityStreamsOrderedCollectionPage(), nil } diff --git a/internal/federation/federator.go b/internal/federation/federator.go @@ -40,6 +40,9 @@ type Federator interface { // AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources. // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. AuthenticateFederatedRequest(username string, r *http.Request) (*url.URL, error) + // FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that + // account, or an error if it doesn't exist or can't be retrieved. + FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) // DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI). // The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments. DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) diff --git a/internal/federation/finger.go b/internal/federation/finger.go @@ -0,0 +1,69 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package federation + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "strings" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" +) + +func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) { + + t, err := f.GetTransportForUser(requestingUsername) + if err != nil { + return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) + } + + b, err := t.Finger(context.Background(), targetUsername, targetDomain) + if err != nil { + return nil, fmt.Errorf("FingerRemoteAccount: error doing request on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) + } + + resp := &apimodel.WebfingerAccountResponse{} + if err := json.Unmarshal(b, resp); err != nil { + return nil, fmt.Errorf("FingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err) + } + + if len(resp.Links) == 0 { + return nil, fmt.Errorf("FingerRemoteAccount: no links found in webfinger response %s", string(b)) + } + + // look through the links for the first one that matches "application/activity+json", this is what we need + for _, l := range resp.Links { + if strings.EqualFold(l.Type, "application/activity+json") { + if l.Href == "" || l.Rel != "self" { + continue + } + accountURI, err := url.Parse(l.Href) + if err != nil { + return nil, fmt.Errorf("FingerRemoteAccount: couldn't parse url %s: %s", l.Href, err) + } + // found it! + return accountURI, nil + } + } + + return nil, errors.New("FingerRemoteAccount: no match found in webfinger response") +} diff --git a/internal/gotosocial/actions.go b/internal/gotosocial/actions.go @@ -38,6 +38,7 @@ import ( "github.com/superseriousbusiness/gotosocial/internal/api/client/instance" mediaModule "github.com/superseriousbusiness/gotosocial/internal/api/client/media" "github.com/superseriousbusiness/gotosocial/internal/api/client/notification" + "github.com/superseriousbusiness/gotosocial/internal/api/client/search" "github.com/superseriousbusiness/gotosocial/internal/api/client/status" "github.com/superseriousbusiness/gotosocial/internal/api/client/timeline" "github.com/superseriousbusiness/gotosocial/internal/api/s2s/user" @@ -122,6 +123,7 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr usersModule := user.New(c, processor, log) timelineModule := timeline.New(c, processor, log) notificationModule := notification.New(c, processor, log) + searchModule := search.New(c, processor, log) mm := mediaModule.New(c, processor, log) fileServerModule := fileserver.New(c, processor, log) adminModule := admin.New(c, processor, log) @@ -146,6 +148,7 @@ var Run action.GTSAction = func(ctx context.Context, c *config.Config, log *logr usersModule, timelineModule, notificationModule, + searchModule, } for _, m := range apis { diff --git a/internal/message/fromfederatorprocess.go b/internal/message/fromfederatorprocess.go @@ -105,7 +105,9 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er } if err := p.db.Put(incomingAnnounce); err != nil { - return fmt.Errorf("error adding dereferenced announce to the db: %s", err) + if _, ok := err.(db.ErrAlreadyExists); !ok { + return fmt.Errorf("error adding dereferenced announce to the db: %s", err) + } } if err := p.notifyAnnounce(incomingAnnounce); err != nil { diff --git a/internal/message/processor.go b/internal/message/processor.go @@ -109,6 +109,9 @@ type Processor interface { // NotificationsGet NotificationsGet(authed *oauth.Auth, limit int, maxID string) ([]*apimodel.Notification, ErrorWithCode) + // SearchGet performs a search with the given params, resolving/dereferencing remotely as desired + SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode) + // StatusCreate processes the given form to create a new status, returning the api model representation of that status if it's OK. StatusCreate(authed *oauth.Auth, form *apimodel.AdvancedStatusCreateForm) (*apimodel.Status, error) // StatusDelete processes the delete of a given status, returning the deleted status if the delete goes through. diff --git a/internal/message/searchprocess.go b/internal/message/searchprocess.go @@ -0,0 +1,292 @@ +/* + GoToSocial + Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package message + +import ( + "errors" + "net/url" + "strings" + + apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" + "github.com/superseriousbusiness/gotosocial/internal/oauth" + "github.com/superseriousbusiness/gotosocial/internal/util" +) + +func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, ErrorWithCode) { + results := &apimodel.SearchResult{ + Accounts: []apimodel.Account{}, + Statuses: []apimodel.Status{}, + Hashtags: []apimodel.Tag{}, + } + foundAccounts := []*gtsmodel.Account{} + foundStatuses := []*gtsmodel.Status{} + // foundHashtags := []*gtsmodel.Tag{} + + // convert the query to lowercase and trim leading/trailing spaces + query := strings.ToLower(strings.TrimSpace(searchQuery.Query)) + + // check if the query is a URI and just do a lookup for that, straight up + if uri, err := url.Parse(query); err == nil { + // 1. check if it's a status + foundStatus, err := p.searchStatusByURI(authed, uri, searchQuery.Resolve) + if err != nil { + return nil, NewErrorInternalError(err) + } + if foundStatus != nil { + foundStatuses = append(foundStatuses, foundStatus) + } + + // 2. check if it's an account + foundAccount, err := p.searchAccountByURI(authed, uri, searchQuery.Resolve) + if err != nil { + return nil, NewErrorInternalError(err) + } + if foundAccount != nil { + foundAccounts = append(foundAccounts, foundAccount) + } + } + + // check if the query is something like @whatever_username@example.org -- this means it's a remote account + if util.IsMention(searchQuery.Query) { + foundAccount, err := p.searchAccountByMention(authed, searchQuery.Query, searchQuery.Resolve) + if err != nil { + return nil, NewErrorInternalError(err) + } + if foundAccount != nil { + foundAccounts = append(foundAccounts, foundAccount) + } + } + + /* + FROM HERE ON we have our search results, it's just a matter of filtering them according to what this user is allowed to see, + and then converting them into our frontend format. + */ + for _, foundAccount := range foundAccounts { + // make sure there's no block in either direction between the account and the requester + if blocked, err := p.db.Blocked(authed.Account.ID, foundAccount.ID); err == nil && !blocked { + // all good, convert it and add it to the results + acctMasto, err := p.tc.AccountToMastoPublic(foundAccount) + if err != nil { + return nil, NewErrorInternalError(err) + } + results.Accounts = append(results.Accounts, *acctMasto) + } + } + + for _, foundStatus := range foundStatuses { + statusOwner := >smodel.Account{} + if err := p.db.GetByID(foundStatus.AccountID, statusOwner); err != nil { + continue + } + + relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(foundStatus) + if err != nil { + continue + } + if visible, err := p.db.StatusVisible(foundStatus, statusOwner, authed.Account, relevantAccounts); !visible || err != nil { + continue + } + + statusMasto, err := p.tc.StatusToMasto(foundStatus, statusOwner, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, nil) + if err != nil { + continue + } + + results.Statuses = append(results.Statuses, *statusMasto) + } + + return results, nil +} + +func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (foundStatus *gtsmodel.Status, err error) { + // 1. check if it's a status + maybeStatus := >smodel.Status{} + if err = p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil { + // we have it and it's a status + foundStatus = maybeStatus + return + } else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil { + // we have it and it's a status + foundStatus = maybeStatus + return + } + + // we don't have it locally so dereference it if we're allowed to + if resolve { + statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri) + if err == nil { + // it IS a status! + + // extract the status owner's IRI from the statusable + var statusOwnerURI *url.URL + statusAttributedTo := statusable.GetActivityStreamsAttributedTo() + for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() { + if i.IsIRI() { + statusOwnerURI = i.GetIRI() + break + } + } + if statusOwnerURI == nil { + return nil, NewErrorInternalError(errors.New("couldn't extract ownerAccountURI from statusable")) + } + + // make sure the status owner exists in the db by searching for it + _, err := p.searchAccountByURI(authed, statusOwnerURI, resolve) + if err != nil { + return nil, err + } + + // we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly + + // first turn it into a gtsmodel.Status + status, err := p.tc.ASStatusToStatus(statusable) + if err != nil { + return nil, NewErrorInternalError(err) + } + + // put it in the DB so it gets a UUID + if err := p.db.Put(status); err != nil { + return nil, NewErrorInternalError(err) + } + + // properly dereference everything in the status (media attachments etc) + if err := p.dereferenceStatusFields(status, authed.Account.Username); err != nil { + return nil, NewErrorInternalError(err) + } + + // update with the nicely dereferenced status + if err := p.db.UpdateByID(status.ID, status); err != nil { + return nil, NewErrorInternalError(err) + } + + foundStatus = status + } + } + return +} + +func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (foundAccount *gtsmodel.Account, err error) { + maybeAccount := >smodel.Account{} + if err = p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil { + // we have it and it's an account + foundAccount = maybeAccount + return + } else if err = p.db.GetWhere([]db.Where{{Key: "url", Value: uri.String(), CaseInsensitive: true}}, maybeAccount); err == nil { + // we have it and it's an account + foundAccount = maybeAccount + return + } + if resolve { + // we don't have it locally so try and dereference it + accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri) + if err == nil { + // it IS an account! + account, err := p.tc.ASRepresentationToAccount(accountable, false) + if err != nil { + return nil, NewErrorInternalError(err) + } + + if err := p.db.Put(account); err != nil { + return nil, NewErrorInternalError(err) + } + + if err := p.dereferenceAccountFields(account, authed.Account.Username, false); err != nil { + return nil, NewErrorInternalError(err) + } + + foundAccount = account + } + } + return +} + +func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, resolve bool) (foundAccount *gtsmodel.Account, err error) { + // query is for a remote account + username, domain, err := util.ExtractMentionParts(mention) + if err != nil { + return nil, NewErrorBadRequest(err) + } + + // if it's a local account we can skip a whole bunch of stuff + maybeAcct := >smodel.Account{} + if domain == p.config.Host { + if p.db.GetLocalAccountByUsername(username, maybeAcct); err != nil { + return + } + foundAccount = maybeAcct + return + } + + // it's not a local account so first we'll check if it's in the database already... + where := []db.Where{ + {Key: "username", Value: username, CaseInsensitive: true}, + {Key: "domain", Value: domain, CaseInsensitive: true}, + } + err = p.db.GetWhere(where, maybeAcct) + if err == nil { + // we've got it stored locally already! + foundAccount = maybeAcct + return + } + + if _, ok := err.(db.ErrNoEntries); !ok { + // if it's not errNoEntries there's been a real database error so bail at this point + return nil, NewErrorInternalError(err) + } + + // we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it + if resolve { + // we're allowed to resolve it so let's try + + // first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account + acctURI, err := p.federator.FingerRemoteAccount(authed.Account.Username, username, domain) + if err != nil { + // something went wrong doing the webfinger lookup so we can't process the request + return nil, NewErrorInternalError(err) + } + + // dereference the account based on the URI we retrieved from the webfinger lookup + accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI) + if err != nil { + // something went wrong doing the dereferencing so we can't process the request + return nil, NewErrorInternalError(err) + } + + // convert the dereferenced account to the gts model of that account + foundAccount, err = p.tc.ASRepresentationToAccount(accountable, false) + if err != nil { + // something went wrong doing the conversion to a gtsmodel.Account so we can't process the request + return nil, NewErrorInternalError(err) + } + + // put this new account in our database + if err := p.db.Put(foundAccount); err != nil { + return nil, NewErrorInternalError(err) + } + + // properly dereference all the fields on the account immediately + if err := p.dereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil { + return nil, NewErrorInternalError(err) + } + } + + return +} diff --git a/internal/transport/transport.go b/internal/transport/transport.go @@ -19,6 +19,8 @@ import ( type Transport interface { pub.Transport DereferenceMedia(c context.Context, iri *url.URL, expectedContentType string) ([]byte, error) + // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. + Finger(c context.Context, targetUsername string, targetDomains string) ([]byte, error) } // transport implements the Transport interface @@ -83,3 +85,42 @@ func (t *transport) DereferenceMedia(c context.Context, iri *url.URL, expectedCo } return ioutil.ReadAll(resp.Body) } + +func (t *transport) Finger(c context.Context, targetUsername string, targetDomain string) ([]byte, error) { + l := t.log.WithField("func", "Finger") + urlString := fmt.Sprintf("https://%s/.well-known/webfinger?resource=acct:%s@%s", targetDomain, targetUsername, targetDomain) + l.Debugf("performing GET to %s", urlString) + + iri, err := url.Parse(urlString) + if err != nil { + return nil, fmt.Errorf("Finger: error parsing url %s: %s", urlString, err) + } + + l.Debugf("performing GET to %s", iri.String()) + + req, err := http.NewRequest("GET", iri.String(), nil) + if err != nil { + return nil, err + } + req = req.WithContext(c) + req.Header.Add("Accept", "application/json") + req.Header.Add("Accept", "application/jrd+json") + req.Header.Add("Date", t.clock.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05")+" GMT") + req.Header.Add("User-Agent", fmt.Sprintf("%s %s", t.appAgent, t.gofedAgent)) + req.Header.Set("Host", iri.Host) + t.getSignerMu.Lock() + err = t.getSigner.SignRequest(t.privkey, t.pubKeyID, req, nil) + t.getSignerMu.Unlock() + if err != nil { + return nil, err + } + resp, err := t.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("GET request to %s failed (%d): %s", iri.String(), resp.StatusCode, resp.Status) + } + return ioutil.ReadAll(resp.Body) +} diff --git a/internal/typeutils/internaltoas.go b/internal/typeutils/internaltoas.go @@ -252,6 +252,7 @@ func (c *converter) AccountToAS(a *gtsmodel.Account) (vocab.ActivityStreamsPerso headerImage.SetActivityStreamsUrl(headerURLProperty) headerProperty.AppendActivityStreamsImage(headerImage) + person.SetActivityStreamsImage(headerProperty) } return person, nil diff --git a/internal/util/statustools.go b/internal/util/statustools.go @@ -77,6 +77,11 @@ func ExtractMentionParts(mention string) (username, domain string, err error) { return } +// IsMention returns true if the passed string looks like @whatever@example.org +func IsMention(mention string) bool { + return mentionNameRegex.MatchString(strings.ToLower(mention)) +} + // unique returns a deduplicated version of a given string slice. func unique(s []string) []string { keys := make(map[string]bool)