commit 268f252e0d517f2693b30d03fb8a68a0764a43bc parent 31639c9b80a6877f96516ecedee865458e684212 Author: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Mon, 12 Sep 2022 13:03:23 +0200 [feature] Fetch + display custom emoji in statuses from remote instances (#807) * start implementing remote emoji fetcher * update status where pk * aaa * tidy up a little * check size limits for emojis * thank you linter, i love you <3 * update swagger docs * add emoji dereference test * make emoji max sizes configurable * normalize db.ErrAlreadyExists Diffstat:
28 files changed, 424 insertions(+), 48 deletions(-)
diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml @@ -2724,7 +2724,9 @@ paths: pattern: \w{2,30} required: true type: string - - description: A png or gif image of the emoji. Animated pngs work too! + - description: |- + A png or gif image of the emoji. Animated pngs work too! + To ensure compatibility with other fedi implementations, emoji size limit is 50kb by default. in: formData name: image required: true diff --git a/docs/configuration/media.md b/docs/configuration/media.md @@ -39,4 +39,20 @@ media-description-max-chars: 500 # Examples: [30, 60, 7, 0] # Default: 30 media-remote-cache-days: 30 + +# Int. Max size in bytes of emojis uploaded to this instance via the admin API. +# The default is the same as the Mastodon size limit for emojis (50kb), which allows +# for good interoperability. Raising this limit may cause issues with federation +# of your emojis to other instances, so beware. +# Examples: [51200, 102400] +# Default: 51200 +media-emoji-local-max-size: 51200 + +# Int. Max size in bytes of emojis to download from other instances. +# By default this is 100kb, or twice the size of the default for media-emoji-local-max-size. +# This strikes a good balance between decent interoperability with instances that have +# higher emoji size limits, and not taking up too much space in storage. +# Examples: [51200, 102400] +# Default: 51200 +media-emoji-remote-max-size: 102400 ``` diff --git a/example/config.yaml b/example/config.yaml @@ -211,7 +211,7 @@ accounts-reason-required: true ##### MEDIA CONFIG ##### ######################## -# Config pertaining to user media uploads (videos, image, image descriptions). +# Config pertaining to media uploads (videos, image, image descriptions, emoji). # Int. Maximum allowed image upload size in bytes. # Examples: [2097152, 10485760] @@ -244,6 +244,22 @@ media-description-max-chars: 500 # Default: 30 media-remote-cache-days: 30 +# Int. Max size in bytes of emojis uploaded to this instance via the admin API. +# The default is the same as the Mastodon size limit for emojis (50kb), which allows +# for good interoperability. Raising this limit may cause issues with federation +# of your emojis to other instances, so beware. +# Examples: [51200, 102400] +# Default: 51200 +media-emoji-local-max-size: 51200 + +# Int. Max size in bytes of emojis to download from other instances. +# By default this is 100kb, or twice the size of the default for media-emoji-local-max-size. +# This strikes a good balance between decent interoperability with instances that have +# higher emoji size limits, and not taking up too much space in storage. +# Examples: [51200, 102400] +# Default: 51200 +media-emoji-remote-max-size: 102400 + ########################## ##### STORAGE CONFIG ##### ########################## diff --git a/internal/api/client/admin/emojicreate.go b/internal/api/client/admin/emojicreate.go @@ -26,6 +26,7 @@ import ( "github.com/gin-gonic/gin" "github.com/superseriousbusiness/gotosocial/internal/api" "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/internal/validate" @@ -56,7 +57,9 @@ import ( // required: true // - name: image // in: formData -// description: A png or gif image of the emoji. Animated pngs work too! +// description: |- +// A png or gif image of the emoji. Animated pngs work too! +// To ensure compatibility with other fedi implementations, emoji size limit is 50kb by default. // type: file // required: true // @@ -126,5 +129,10 @@ func validateCreateEmoji(form *model.EmojiCreateRequest) error { return errors.New("no emoji given") } + maxSize := config.GetMediaEmojiLocalMaxSize() + if form.Image.Size > int64(maxSize) { + return fmt.Errorf("emoji image too large: image is %dKB but size limit for custom emojis is %dKB", form.Image.Size/1024, maxSize/1024) + } + return validate.EmojiShortcode(form.Shortcode) } diff --git a/internal/config/config.go b/internal/config/config.go @@ -79,6 +79,8 @@ type Configuration struct { MediaDescriptionMinChars int `name:"media-description-min-chars" usage:"Min required chars for an image description"` MediaDescriptionMaxChars int `name:"media-description-max-chars" usage:"Max permitted chars for an image description"` MediaRemoteCacheDays int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."` + MediaEmojiLocalMaxSize int `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."` + MediaEmojiRemoteMaxSize int `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."` StorageBackend string `name:"storage-backend" usage:"Storage backend to use for media attachments"` StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."` diff --git a/internal/config/defaults.go b/internal/config/defaults.go @@ -58,6 +58,8 @@ var Defaults = Configuration{ MediaDescriptionMinChars: 0, MediaDescriptionMaxChars: 500, MediaRemoteCacheDays: 30, + MediaEmojiLocalMaxSize: 51200, // 50kb + MediaEmojiRemoteMaxSize: 102400, // 100kb StorageBackend: "local", StorageLocalBasePath: "/gotosocial/storage", diff --git a/internal/config/flags.go b/internal/config/flags.go @@ -75,6 +75,8 @@ func AddServerFlags(cmd *cobra.Command) { cmd.Flags().Int(MediaDescriptionMinCharsFlag(), cfg.MediaDescriptionMinChars, fieldtag("MediaDescriptionMinChars", "usage")) cmd.Flags().Int(MediaDescriptionMaxCharsFlag(), cfg.MediaDescriptionMaxChars, fieldtag("MediaDescriptionMaxChars", "usage")) cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage")) + cmd.Flags().Int(MediaEmojiLocalMaxSizeFlag(), cfg.MediaEmojiLocalMaxSize, fieldtag("MediaEmojiLocalMaxSize", "usage")) + cmd.Flags().Int(MediaEmojiRemoteMaxSizeFlag(), cfg.MediaEmojiRemoteMaxSize, fieldtag("MediaEmojiRemoteMaxSize", "usage")) // Storage cmd.Flags().String(StorageBackendFlag(), cfg.StorageBackend, fieldtag("StorageBackend", "usage")) diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go @@ -793,6 +793,56 @@ func GetMediaRemoteCacheDays() int { return global.GetMediaRemoteCacheDays() } // SetMediaRemoteCacheDays safely sets the value for global configuration 'MediaRemoteCacheDays' field func SetMediaRemoteCacheDays(v int) { global.SetMediaRemoteCacheDays(v) } +// GetMediaEmojiLocalMaxSize safely fetches the Configuration value for state's 'MediaEmojiLocalMaxSize' field +func (st *ConfigState) GetMediaEmojiLocalMaxSize() (v int) { + st.mutex.Lock() + v = st.config.MediaEmojiLocalMaxSize + st.mutex.Unlock() + return +} + +// SetMediaEmojiLocalMaxSize safely sets the Configuration value for state's 'MediaEmojiLocalMaxSize' field +func (st *ConfigState) SetMediaEmojiLocalMaxSize(v int) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.MediaEmojiLocalMaxSize = v + st.reloadToViper() +} + +// MediaEmojiLocalMaxSizeFlag returns the flag name for the 'MediaEmojiLocalMaxSize' field +func MediaEmojiLocalMaxSizeFlag() string { return "media-emoji-local-max-size" } + +// GetMediaEmojiLocalMaxSize safely fetches the value for global configuration 'MediaEmojiLocalMaxSize' field +func GetMediaEmojiLocalMaxSize() int { return global.GetMediaEmojiLocalMaxSize() } + +// SetMediaEmojiLocalMaxSize safely sets the value for global configuration 'MediaEmojiLocalMaxSize' field +func SetMediaEmojiLocalMaxSize(v int) { global.SetMediaEmojiLocalMaxSize(v) } + +// GetMediaEmojiRemoteMaxSize safely fetches the Configuration value for state's 'MediaEmojiRemoteMaxSize' field +func (st *ConfigState) GetMediaEmojiRemoteMaxSize() (v int) { + st.mutex.Lock() + v = st.config.MediaEmojiRemoteMaxSize + st.mutex.Unlock() + return +} + +// SetMediaEmojiRemoteMaxSize safely sets the Configuration value for state's 'MediaEmojiRemoteMaxSize' field +func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v int) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.MediaEmojiRemoteMaxSize = v + st.reloadToViper() +} + +// MediaEmojiRemoteMaxSizeFlag returns the flag name for the 'MediaEmojiRemoteMaxSize' field +func MediaEmojiRemoteMaxSizeFlag() string { return "media-emoji-remote-max-size" } + +// GetMediaEmojiRemoteMaxSize safely fetches the value for global configuration 'MediaEmojiRemoteMaxSize' field +func GetMediaEmojiRemoteMaxSize() int { return global.GetMediaEmojiRemoteMaxSize() } + +// SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field +func SetMediaEmojiRemoteMaxSize(v int) { global.SetMediaEmojiRemoteMaxSize(v) } + // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field func (st *ConfigState) GetStorageBackend() (v string) { st.mutex.Lock() diff --git a/internal/config/validate.go b/internal/config/validate.go @@ -67,6 +67,14 @@ func Validate() error { errs = append(errs, fmt.Errorf("%s must be set", WebAssetBaseDirFlag())) } + if m := GetMediaEmojiLocalMaxSize(); m < 0 { + errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiLocalMaxSizeFlag())) + } + + if m := GetMediaEmojiRemoteMaxSize(); m < 0 { + errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiRemoteMaxSizeFlag())) + } + if len(errs) > 0 { errStrings := []string{} for _, err := range errs { diff --git a/internal/config/validate_test.go b/internal/config/validate_test.go @@ -141,6 +141,16 @@ func (suite *ConfigValidateTestSuite) TestValidateConfigBadProtocolNoHost() { suite.EqualError(err, "host must be set; protocol must be set to either http or https, provided value was foo") } +func (suite *ConfigValidateTestSuite) TestValidateConfigBadEmojiSizes() { + testrig.InitTestConfig() + + config.SetMediaEmojiLocalMaxSize(-10) + config.SetMediaEmojiRemoteMaxSize(-50) + + err := config.Validate() + suite.EqualError(err, "media-emoji-local-max-size must not be less than 0; media-emoji-remote-max-size must not be less than 0") +} + func TestConfigValidateTestSuite(t *testing.T) { suite.Run(t, &ConfigValidateTestSuite{}) } diff --git a/internal/db/bundb/errors.go b/internal/db/bundb/errors.go @@ -19,7 +19,7 @@ func processPostgresError(err error) db.Error { // (https://www.postgresql.org/docs/10/errcodes-appendix.html) switch pgErr.Code { case "23505" /* unique_violation */ : - return db.NewErrAlreadyExists(pgErr.Message) + return db.ErrAlreadyExists default: return err } @@ -36,7 +36,7 @@ func processSQLiteError(err error) db.Error { // Handle supplied error code: switch sqliteErr.Code() { case sqlite3.SQLITE_CONSTRAINT_UNIQUE, sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY: - return db.NewErrAlreadyExists(err.Error()) + return db.ErrAlreadyExists default: return err } diff --git a/internal/db/bundb/status.go b/internal/db/bundb/status.go @@ -22,6 +22,7 @@ import ( "container/list" "context" "database/sql" + "errors" "time" "github.com/superseriousbusiness/gotosocial/internal/cache" @@ -175,6 +176,57 @@ func (s *statusDB) PutStatus(ctx context.Context, status *gtsmodel.Status) db.Er }) } +func (s *statusDB) UpdateStatus(ctx context.Context, status *gtsmodel.Status) (*gtsmodel.Status, db.Error) { + err := s.conn.RunInTx(ctx, func(tx bun.Tx) error { + // create links between this status and any emojis it uses + for _, i := range status.EmojiIDs { + if _, err := tx.NewInsert().Model(>smodel.StatusToEmoji{ + StatusID: status.ID, + EmojiID: i, + }).Exec(ctx); err != nil { + err = s.conn.errProc(err) + if !errors.Is(err, db.ErrAlreadyExists) { + return err + } + } + } + + // create links between this status and any tags it uses + for _, i := range status.TagIDs { + if _, err := tx.NewInsert().Model(>smodel.StatusToTag{ + StatusID: status.ID, + TagID: i, + }).Exec(ctx); err != nil { + err = s.conn.errProc(err) + if !errors.Is(err, db.ErrAlreadyExists) { + return err + } + } + } + + // change the status ID of the media attachments to this status + for _, a := range status.Attachments { + a.StatusID = status.ID + a.UpdatedAt = time.Now() + if _, err := tx.NewUpdate().Model(a). + Where("id = ?", a.ID). + Exec(ctx); err != nil { + return err + } + } + + // Finally, update the status itself + if _, err := tx.NewUpdate().Model(status).WherePK().Exec(ctx); err != nil { + return err + } + + s.cache.Put(status) + return nil + }) + + return status, err +} + func (s *statusDB) GetStatusParents(ctx context.Context, status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, db.Error) { parents := []*gtsmodel.Status{} s.statusParent(ctx, status, &parents, onlyDirect) diff --git a/internal/db/emoji.go b/internal/db/emoji.go @@ -35,4 +35,6 @@ type Emoji interface { // GetEmojiByShortcodeDomain gets an emoji based on its shortcode and domain. // For local emoji, domain should be an empty string. GetEmojiByShortcodeDomain(ctx context.Context, shortcode string, domain string) (*gtsmodel.Emoji, Error) + // GetEmojiByURI returns one emoji based on its ActivityPub URI. + GetEmojiByURI(ctx context.Context, uri string) (*gtsmodel.Emoji, Error) } diff --git a/internal/db/error.go b/internal/db/error.go @@ -28,19 +28,8 @@ var ( ErrNoEntries Error = fmt.Errorf("no entries") // ErrMultipleEntries is returned when a caller expected ONE entry for a query, but multiples were found. ErrMultipleEntries Error = fmt.Errorf("multiple entries") + // ErrAlreadyExists is returned when a conflict was encountered in the db when doing an insert. + ErrAlreadyExists Error = fmt.Errorf("already exists") // ErrUnknown denotes an unknown database error. ErrUnknown Error = fmt.Errorf("unknown error") ) - -// ErrAlreadyExists is returned when a caller tries to insert a database entry that already exists in the db. -type ErrAlreadyExists struct { - message string -} - -func (e *ErrAlreadyExists) Error() string { - return e.message -} - -func NewErrAlreadyExists(msg string) error { - return &ErrAlreadyExists{message: msg} -} diff --git a/internal/db/status.go b/internal/db/status.go @@ -38,6 +38,9 @@ type Status interface { // PutStatus stores one status in the database. PutStatus(ctx context.Context, status *gtsmodel.Status) Error + // UpdateStatus updates one status in the database and returns it to the caller. + UpdateStatus(ctx context.Context, status *gtsmodel.Status) (*gtsmodel.Status, Error) + // CountStatusReplies returns the amount of replies recorded for a status, or an error if something goes wrong CountStatusReplies(ctx context.Context, status *gtsmodel.Status) (int, Error) diff --git a/internal/federation/dereferencing/dereferencer.go b/internal/federation/dereferencing/dereferencer.go @@ -41,6 +41,7 @@ type Dereferencer interface { GetRemoteInstance(ctx context.Context, username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) GetRemoteMedia(ctx context.Context, requestingUsername string, accountID string, remoteURL string, ai *media.AdditionalMediaInfo) (*media.ProcessingMedia, error) + GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo) (*media.ProcessingEmoji, error) DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error DereferenceThread(ctx context.Context, username string, statusIRI *url.URL) error diff --git a/internal/federation/dereferencing/emoji.go b/internal/federation/dereferencing/emoji.go @@ -0,0 +1,51 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package dereferencing + +import ( + "context" + "fmt" + "io" + "net/url" + + "github.com/superseriousbusiness/gotosocial/internal/media" +) + +func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, remoteURL string, shortcode string, id string, emojiURI string, ai *media.AdditionalEmojiInfo) (*media.ProcessingEmoji, error) { + t, err := d.transportController.NewTransportForUsername(ctx, requestingUsername) + if err != nil { + return nil, fmt.Errorf("GetRemoteEmoji: error creating transport: %s", err) + } + + derefURI, err := url.Parse(remoteURL) + if err != nil { + return nil, fmt.Errorf("GetRemoteEmoji: error parsing url: %s", err) + } + + dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { + return t.DereferenceMedia(innerCtx, derefURI) + } + + processingMedia, err := d.mediaManager.ProcessEmoji(ctx, dataFunc, nil, shortcode, id, emojiURI, ai) + if err != nil { + return nil, fmt.Errorf("GetRemoteEmoji: error processing emoji: %s", err) + } + + return processingMedia, nil +} diff --git a/internal/federation/dereferencing/emoji_test.go b/internal/federation/dereferencing/emoji_test.go @@ -0,0 +1,95 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package dereferencing_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/suite" + "github.com/superseriousbusiness/gotosocial/internal/media" +) + +type EmojiTestSuite struct { + DereferencerStandardTestSuite +} + +func (suite *EmojiTestSuite) TestDereferenceEmojiBlocking() { + ctx := context.Background() + fetchingAccount := suite.testAccounts["local_account_1"] + emojiImageRemoteURL := "http://example.org/media/emojis/1781772.gif" + emojiImageStaticRemoteURL := "http://example.org/media/emojis/1781772.gif" + emojiURI := "http://example.org/emojis/1781772" + emojiShortcode := "peglin" + emojiID := "01GCBMGNZBKMEE1KTZ6PMJEW5D" + emojiDomain := "example.org" + emojiDisabled := false + emojiVisibleInPicker := false + + ai := &media.AdditionalEmojiInfo{ + Domain: &emojiDomain, + ImageRemoteURL: &emojiImageRemoteURL, + ImageStaticRemoteURL: &emojiImageStaticRemoteURL, + Disabled: &emojiDisabled, + VisibleInPicker: &emojiVisibleInPicker, + } + + processingEmoji, err := suite.dereferencer.GetRemoteEmoji(ctx, fetchingAccount.Username, emojiImageRemoteURL, emojiShortcode, emojiID, emojiURI, ai) + suite.NoError(err) + + // make a blocking call to load the emoji from the in-process media + emoji, err := processingEmoji.LoadEmoji(ctx) + suite.NoError(err) + suite.NotNil(emoji) + + suite.Equal(emojiID, emoji.ID) + suite.WithinDuration(time.Now(), emoji.CreatedAt, 10*time.Second) + suite.WithinDuration(time.Now(), emoji.UpdatedAt, 10*time.Second) + suite.Equal(emojiShortcode, emoji.Shortcode) + suite.Equal(emojiDomain, emoji.Domain) + suite.Equal(emojiImageRemoteURL, emoji.ImageRemoteURL) + suite.Equal(emojiImageStaticRemoteURL, emoji.ImageStaticRemoteURL) + suite.Contains(emoji.ImageURL, "/emoji/original/01GCBMGNZBKMEE1KTZ6PMJEW5D.gif") + suite.Contains(emoji.ImageStaticURL, "emoji/static/01GCBMGNZBKMEE1KTZ6PMJEW5D.png") + suite.Contains(emoji.ImagePath, "/emoji/original/01GCBMGNZBKMEE1KTZ6PMJEW5D.gif") + suite.Contains(emoji.ImageStaticPath, "/emoji/static/01GCBMGNZBKMEE1KTZ6PMJEW5D.png") + suite.Equal("image/gif", emoji.ImageContentType) + suite.Equal("image/png", emoji.ImageStaticContentType) + suite.Equal(37796, emoji.ImageFileSize) + suite.Equal(7951, emoji.ImageStaticFileSize) + suite.WithinDuration(time.Now(), emoji.ImageUpdatedAt, 10*time.Second) + suite.False(*emoji.Disabled) + suite.Equal(emojiURI, emoji.URI) + suite.False(*emoji.VisibleInPicker) + suite.Empty(emoji.CategoryID) + + // ensure that emoji is now in storage + stored, err := suite.storage.Get(ctx, emoji.ImagePath) + suite.NoError(err) + suite.Len(stored, emoji.ImageFileSize) + + storedStatic, err := suite.storage.Get(ctx, emoji.ImageStaticPath) + suite.NoError(err) + suite.Len(storedStatic, emoji.ImageStaticFileSize) +} + +func TestEmojiTestSuite(t *testing.T) { + suite.Run(t, new(EmojiTestSuite)) +} diff --git a/internal/federation/dereferencing/status.go b/internal/federation/dereferencing/status.go @@ -26,10 +26,10 @@ import ( "net/url" "strings" - "codeberg.org/gruf/go-kv" "github.com/superseriousbusiness/activity/streams" "github.com/superseriousbusiness/activity/streams/vocab" "github.com/superseriousbusiness/gotosocial/internal/ap" + "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -46,11 +46,7 @@ func (d *deref) EnrichRemoteStatus(ctx context.Context, username string, status return nil, err } - if err := d.db.UpdateByPrimaryKey(ctx, status); err != nil { - return nil, fmt.Errorf("EnrichRemoteStatus: error updating status: %s", err) - } - - return status, nil + return d.db.UpdateStatus(ctx, status) } // GetRemoteStatus completely dereferences a remote status, converts it to a GtS model status, @@ -225,12 +221,6 @@ func (d *deref) dereferenceStatusable(ctx context.Context, username string, remo // and attach them to the status. The status itself will not be added to the database yet, // that's up the caller to do. func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Status, requestingUsername string, includeParent bool) error { - l := log.WithFields(kv.Fields{ - - {"status", status}, - }...) - l.Debug("entering function") - statusIRI, err := url.Parse(status.URI) if err != nil { return fmt.Errorf("populateStatusFields: couldn't parse status URI %s: %s", status.URI, err) @@ -262,7 +252,9 @@ func (d *deref) populateStatusFields(ctx context.Context, status *gtsmodel.Statu // TODO // 3. Emojis - // TODO + if err := d.populateStatusEmojis(ctx, status, requestingUsername); err != nil { + return fmt.Errorf("populateStatusFields: error populating status emojis: %s", err) + } // 4. Mentions // TODO: do we need to handle removing empty mention objects and just using mention IDs slice? @@ -413,6 +405,64 @@ func (d *deref) populateStatusAttachments(ctx context.Context, status *gtsmodel. return nil } +func (d *deref) populateStatusEmojis(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error { + // At this point we should know: + // * the AP uri of the emoji + // * the domain of the emoji + // * the shortcode of the emoji + // * the remote URL of the image + // This should be enough to dereference the emoji + + gotEmojis := make([]*gtsmodel.Emoji, 0, len(status.Emojis)) + emojiIDs := make([]string, 0, len(status.Emojis)) + + for _, e := range status.Emojis { + var gotEmoji *gtsmodel.Emoji + var err error + + // check if we've already got this emoji in the db + if gotEmoji, err = d.db.GetEmojiByURI(ctx, e.URI); err != nil && err != db.ErrNoEntries { + log.Errorf("populateStatusEmojis: error checking database for emoji %s: %s", e.URI, err) + continue + } + + if gotEmoji == nil { + // it's new! go get it! + newEmojiID, err := id.NewRandomULID() + if err != nil { + log.Errorf("populateStatusEmojis: error generating id for remote emoji %s: %s", e.URI, err) + continue + } + + processingEmoji, err := d.GetRemoteEmoji(ctx, requestingUsername, e.ImageRemoteURL, e.Shortcode, newEmojiID, e.URI, &media.AdditionalEmojiInfo{ + Domain: &e.Domain, + ImageRemoteURL: &e.ImageRemoteURL, + ImageStaticRemoteURL: &e.ImageRemoteURL, + Disabled: e.Disabled, + VisibleInPicker: e.VisibleInPicker, + }) + + if err != nil { + log.Errorf("populateStatusEmojis: couldn't get remote emoji %s: %s", e.URI, err) + continue + } + + if gotEmoji, err = processingEmoji.LoadEmoji(ctx); err != nil { + log.Errorf("populateStatusEmojis: couldn't load remote emoji %s: %s", e.URI, err) + continue + } + } + + // if we get here, we either had the emoji already or we successfully fetched it + gotEmojis = append(gotEmojis, gotEmoji) + emojiIDs = append(emojiIDs, gotEmoji.ID) + } + + status.Emojis = gotEmojis + status.EmojiIDs = emojiIDs + return nil +} + func (d *deref) populateStatusRepliedTo(ctx context.Context, status *gtsmodel.Status, requestingUsername string) error { if status.InReplyToURI != "" && status.InReplyToID == "" { statusURI, err := url.Parse(status.InReplyToURI) diff --git a/internal/federation/federatingdb/create.go b/internal/federation/federatingdb/create.go @@ -226,8 +226,7 @@ func (f *federatingDB) createNote(ctx context.Context, note vocab.ActivityStream status.ID = statusID if err := f.db.PutStatus(ctx, status); err != nil { - var alreadyExistsError *db.ErrAlreadyExists - if errors.As(err, &alreadyExistsError) { + if errors.Is(err, db.ErrAlreadyExists) { // the status already exists in the database, which means we've already handled everything else, // so we can just return nil here and be done with it. return nil diff --git a/internal/gtsmodel/emoji.go b/internal/gtsmodel/emoji.go @@ -20,7 +20,7 @@ package gtsmodel import "time" -// Emoji represents a custom emoji that's been uploaded through the admin UI, and is useable by instance denizens. +// Emoji represents a custom emoji that's been uploaded through the admin UI or downloaded from a remote instance. type Emoji struct { ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go @@ -28,6 +28,7 @@ import ( "sync/atomic" "time" + "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" @@ -170,6 +171,11 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { return fmt.Errorf("store: error executing data function: %s", err) } + maxSize := config.GetMediaEmojiRemoteMaxSize() + if fileSize > maxSize { + return fmt.Errorf("store: emoji size (%db) is larger than allowed emojiRemoteMaxSize (%db)", fileSize, maxSize) + } + // defer closing the reader when we're done with it defer func() { if rc, ok := reader.(io.ReadCloser); ok { diff --git a/internal/processing/status/util.go b/internal/processing/status/util.go @@ -234,8 +234,7 @@ func (p *processor) ProcessTags(ctx context.Context, form *apimodel.AdvancedStat } for _, tag := range gtsTags { if err := p.db.Put(ctx, tag); err != nil { - var alreadyExistsError *db.ErrAlreadyExists - if !errors.As(err, &alreadyExistsError) { + if !errors.Is(err, db.ErrAlreadyExists) { return fmt.Errorf("error putting tags in db: %s", err) } } diff --git a/test/cliparsing.sh b/test/cliparsing.sh @@ -5,7 +5,7 @@ set -e echo "STARTING CLI TESTS" echo "TEST_1 Make sure defaults are set correctly." -TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_1="$(go run ./cmd/gotosocial/... debug config)" if [ "${TEST_1}" != "${TEST_1_EXPECTED}" ]; then echo "TEST_1 not equal TEST_1_EXPECTED" @@ -15,7 +15,7 @@ else fi echo "TEST_2 Override db-address from default using cli flag." -TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_2="$(go run ./cmd/gotosocial/... --db-address some.db.address debug config)" if [ "${TEST_2}" != "${TEST_2_EXPECTED}" ]; then echo "TEST_2 not equal TEST_2_EXPECTED" @@ -25,7 +25,7 @@ else fi echo "TEST_3 Override db-address from default using env var." -TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_3="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... debug config)" if [ "${TEST_3}" != "${TEST_3_EXPECTED}" ]; then echo "TEST_3 not equal TEST_3_EXPECTED" @@ -35,7 +35,7 @@ else fi echo "TEST_4 Override db-address from default using both env var and cli flag. The cli flag should take priority." -TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_4="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... --db-address some.other.db.address debug config)" if [ "${TEST_4}" != "${TEST_4_EXPECTED}" ]; then echo "TEST_4 not equal TEST_4_EXPECTED" @@ -45,7 +45,7 @@ else fi echo "TEST_5 Test loading a config file by passing an env var." -TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_5="$(GTS_CONFIG_PATH=./test/test.yaml go run ./cmd/gotosocial/... debug config)" if [ "${TEST_5}" != "${TEST_5_EXPECTED}" ]; then echo "TEST_5 not equal TEST_5_EXPECTED" @@ -55,7 +55,7 @@ else fi echo "TEST_6 Test loading a config file by passing cli flag." -TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_6="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)" if [ "${TEST_6}" != "${TEST_6_EXPECTED}" ]; then echo "TEST_6 not equal TEST_6_EXPECTED" @@ -65,7 +65,7 @@ else fi echo "TEST_7 Test loading a config file and overriding one of the variables with a cli flag." -TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_7="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)" if [ "${TEST_7}" != "${TEST_7_EXPECTED}" ]; then echo "TEST_7 not equal TEST_7_EXPECTED" @@ -75,7 +75,7 @@ else fi echo "TEST_8 Test loading a config file and overriding one of the variables with an env var." -TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_8="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)" if [ "${TEST_8}" != "${TEST_8_EXPECTED}" ]; then echo "TEST_8 not equal TEST_8_EXPECTED" @@ -85,7 +85,7 @@ else fi echo "TEST_9 Test loading a config file and overriding one of the variables with both an env var and a cli flag. The cli flag should have priority." -TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_9="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)" if [ "${TEST_9}" != "${TEST_9_EXPECTED}" ]; then echo "TEST_9 not equal TEST_9_EXPECTED" @@ -95,7 +95,7 @@ else fi echo "TEST_10 Test loading a config file from json." -TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","email":"","host":"gts.example.org","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":false,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_10="$(go run ./cmd/gotosocial/... --config-path ./test/test.json debug config)" if [ "${TEST_10}" != "${TEST_10_EXPECTED}" ]; then echo "TEST_10 not equal TEST_10_EXPECTED" @@ -105,7 +105,7 @@ else fi echo "TEST_11 Test loading a partial config file. Default values should be used apart from those set in the config file." -TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' +TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"advanced-cookies-samesite":"lax","application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","email":"","host":"","instance-expose-peers":false,"instance-expose-suspended":false,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-db-queries":false,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-emoji-local-max-size":51200,"media-emoji-remote-max-size":102400,"media-image-max-size":10485760,"media-remote-cache-days":30,"media-video-max-size":41943040,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"password":"","path":"","port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","storage-s3-access-key":"","storage-s3-bucket":"","storage-s3-endpoint":"","storage-s3-secret-key":"","storage-s3-use-ssl":true,"syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"username":"","web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}' TEST_11="$(go run ./cmd/gotosocial/... --config-path ./test/test2.yaml debug config)" if [ "${TEST_11}" != "${TEST_11_EXPECTED}" ]; then echo "TEST_11 not equal TEST_11_EXPECTED" diff --git a/test/envparsing.sh b/test/envparsing.sh @@ -2,7 +2,7 @@ set -eu -EXPECTED='{"account-domain":"peepee","accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-expose-peers":true,"instance-expose-suspended":true,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' +EXPECTED='{"account-domain":"peepee","accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-expose-peers":true,"instance-expose-suspended":true,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' # Set all the environment variables to # ensure that these are parsed without panic @@ -35,6 +35,8 @@ GTS_MEDIA_VIDEO_MAX_SIZE=420 \ GTS_MEDIA_DESCRIPTION_MIN_CHARS=69 \ GTS_MEDIA_DESCRIPTION_MAX_CHARS=5000 \ GTS_MEDIA_REMOTE_CACHE_DAYS=30 \ +GTS_MEDIA_EMOJI_LOCAL_MAX_SIZE=420 \ +GTS_MEDIA_EMOJI_REMOTE_MAX_SIZE=420 \ GTS_STORAGE_BACKEND='local' \ GTS_STORAGE_LOCAL_BASE_PATH='/root/store' \ GTS_STORAGE_S3_ACCESS_KEY='minio' \ diff --git a/testrig/config.go b/testrig/config.go @@ -64,6 +64,8 @@ var testDefaults = config.Configuration{ MediaDescriptionMinChars: 0, MediaDescriptionMaxChars: 500, MediaRemoteCacheDays: 30, + MediaEmojiLocalMaxSize: 51200, // 50kb + MediaEmojiRemoteMaxSize: 102400, // 100kb // the testrig only uses in-memory storage, so we can // safely set this value to 'test' to avoid running storage diff --git a/testrig/media/peglin.gif b/testrig/media/peglin.gif Binary files differ. diff --git a/testrig/testmodels.go b/testrig/testmodels.go @@ -1954,6 +1954,11 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile panic(err) } + peglinBytes, err := os.ReadFile(fmt.Sprintf("%s/peglin.gif", relativePath)) + if err != nil { + panic(err) + } + return map[string]RemoteAttachmentFile{ "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg": { Data: beeBytes, @@ -1967,6 +1972,10 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile Data: massiveFuckingTurnipBytes, ContentType: "image/jpeg", }, + "http://example.org/media/emojis/1781772.gif": { + Data: peglinBytes, + ContentType: "image/gif", + }, } }
