commit 1659f75ae6e491355e1d32f0f5e8b956ef70a797 parent eabb9062686a38246ff7496270327cccc4b2f3fe Author: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Thu, 22 Dec 2022 11:48:28 +0100 [feature] For video attachments, store + return fps, bitrate, duration (#1282) * start messing about with different mp4 metadata extraction * heyyooo it works * add test cow * move useful multierror to gtserror package * error out if video doesn't seem to be a real mp4 * test parsing mkv in disguise as mp4 * tidy up error handling * remove extraneous line * update framerate formatting * use float32 for aspect * fixy mctesterson Diffstat:
19 files changed, 429 insertions(+), 104 deletions(-)
diff --git a/internal/db/bundb/migrations/20221220134514_mp4_jiggery_pokery.go b/internal/db/bundb/migrations/20221220134514_mp4_jiggery_pokery.go @@ -0,0 +1,59 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package migrations + +import ( + "context" + "strings" + + "github.com/uptrace/bun" +) + +func init() { + up := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + _, err := tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? REAL", bun.Ident("media_attachments"), bun.Ident("original_duration")) + if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? REAL", bun.Ident("media_attachments"), bun.Ident("original_framerate")) + if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) { + return err + } + + _, err = tx.ExecContext(ctx, "ALTER TABLE ? ADD COLUMN ? INTEGER", bun.Ident("media_attachments"), bun.Ident("original_bitrate")) + if err != nil && !(strings.Contains(err.Error(), "already exists") || strings.Contains(err.Error(), "duplicate column name") || strings.Contains(err.Error(), "SQLSTATE 42701")) { + return err + } + + return nil + }) + } + + down := func(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return nil + }) + } + + if err := Migrations.Register(up, down); err != nil { + panic(err) + } +} diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go @@ -66,7 +66,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { suite.NotEmpty(attachment.ID) suite.NotEmpty(attachment.CreatedAt) suite.NotEmpty(attachment.UpdatedAt) - suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect) + suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect) suite.Equal(2071680, attachment.FileMeta.Original.Size) suite.Equal(1245, attachment.FileMeta.Original.Height) suite.Equal(1664, attachment.FileMeta.Original.Width) @@ -92,7 +92,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { suite.NotEmpty(dbAttachment.ID) suite.NotEmpty(dbAttachment.CreatedAt) suite.NotEmpty(dbAttachment.UpdatedAt) - suite.Equal(1.336546184738956, dbAttachment.FileMeta.Original.Aspect) + suite.EqualValues(1.3365462, dbAttachment.FileMeta.Original.Aspect) suite.Equal(2071680, dbAttachment.FileMeta.Original.Size) suite.Equal(1245, dbAttachment.FileMeta.Original.Height) suite.Equal(1664, dbAttachment.FileMeta.Original.Width) @@ -147,7 +147,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() { suite.NotEmpty(attachment.ID) suite.NotEmpty(attachment.CreatedAt) suite.NotEmpty(attachment.UpdatedAt) - suite.Equal(1.336546184738956, attachment.FileMeta.Original.Aspect) + suite.EqualValues(1.3365462, attachment.FileMeta.Original.Aspect) suite.Equal(2071680, attachment.FileMeta.Original.Size) suite.Equal(1245, attachment.FileMeta.Original.Height) suite.Equal(1664, attachment.FileMeta.Original.Width) diff --git a/internal/gtserror/multi.go b/internal/gtserror/multi.go @@ -0,0 +1,45 @@ +/* + GoToSocial + Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +package gtserror + +import ( + "errors" + "fmt" + "strings" +) + +// MultiError allows encapsulating multiple errors under a singular instance, +// which is useful when you only want to log on errors, not return early / bubble up. +type MultiError []string + +func (e *MultiError) Append(err error) { + *e = append(*e, err.Error()) +} + +func (e *MultiError) Appendf(format string, args ...any) { + *e = append(*e, fmt.Sprintf(format, args...)) +} + +// Combine converts this multiError to a singular error instance, returning nil if empty. +func (e MultiError) Combine() error { + if len(e) == 0 { + return nil + } + return errors.New(`"` + strings.Join(e, `","`) + `"`) +} diff --git a/internal/gtsmodel/mediaattachment.go b/internal/gtsmodel/mediaattachment.go @@ -99,15 +99,18 @@ type Small struct { Width int `validate:"required_with=Height Size Aspect"` // width in pixels Height int `validate:"required_with=Width Size Aspect"` // height in pixels Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height) - Aspect float64 `validate:"required_with=Widhth Height Size"` // aspect ratio (width / height) + Aspect float32 `validate:"required_with=Width Height Size"` // aspect ratio (width / height) } // Original can be used for original metadata for any media type type Original struct { - Width int `validate:"required_with=Height Size Aspect"` // width in pixels - Height int `validate:"required_with=Width Size Aspect"` // height in pixels - Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height) - Aspect float64 `validate:"required_with=Widhth Height Size"` // aspect ratio (width / height) + Width int `validate:"required_with=Height Size Aspect"` // width in pixels + Height int `validate:"required_with=Width Size Aspect"` // height in pixels + Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height) + Aspect float32 `validate:"required_with=Width Height Size"` // aspect ratio (width / height) + Duration *float32 `validate:"-"` // video-specific: duration of the video in seconds + Framerate *float32 `validate:"-"` // video-specific: fps + Bitrate *uint64 `validate:"-"` // video-specific: bitrate } // Focus describes the 'center' of the image for display purposes. diff --git a/internal/media/image.go b/internal/media/image.go @@ -48,7 +48,7 @@ func decodeGif(r io.Reader) (*mediaMeta, error) { width := gif.Config.Width height := gif.Config.Height size := width * height - aspect := float64(width) / float64(height) + aspect := float32(width) / float32(height) return &mediaMeta{ width: width, @@ -85,7 +85,7 @@ func decodeImage(r io.Reader, contentType string) (*mediaMeta, error) { width := i.Bounds().Size().X height := i.Bounds().Size().Y size := width * height - aspect := float64(width) / float64(height) + aspect := float32(width) / float32(height) return &mediaMeta{ width: width, @@ -167,7 +167,7 @@ func deriveThumbnailFromImage(r io.Reader, contentType string, createBlurhash bo thumbX := thumb.Bounds().Size().X thumbY := thumb.Bounds().Size().Y size := thumbX * thumbY - aspect := float64(thumbX) / float64(thumbY) + aspect := float32(thumbX) / float32(thumbY) im := &mediaMeta{ width: thumbX, diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go @@ -407,9 +407,13 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() { suite.Equal(accountID, attachment.AccountID) // file meta should be correctly derived from the video - suite.EqualValues(gtsmodel.Original{ - Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334, - }, attachment.FileMeta.Original) + suite.Equal(338, attachment.FileMeta.Original.Width) + suite.Equal(240, attachment.FileMeta.Original.Height) + suite.Equal(81120, attachment.FileMeta.Original.Size) + suite.EqualValues(1.4083333, attachment.FileMeta.Original.Aspect) + suite.EqualValues(6.5862, *attachment.FileMeta.Original.Duration) + suite.EqualValues(29.000029, *attachment.FileMeta.Original.Framerate) + suite.EqualValues(0x3b3e1, *attachment.FileMeta.Original.Bitrate) suite.EqualValues(gtsmodel.Small{ Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334, }, attachment.FileMeta.Small) @@ -448,6 +452,108 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() { suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes) } +func (suite *ManagerTestSuite) TestLongerMp4ProcessBlocking() { + ctx := context.Background() + + data := func(_ context.Context) (io.ReadCloser, int64, error) { + // load bytes from a test video + b, err := os.ReadFile("./test/longer-mp4-original.mp4") + if err != nil { + panic(err) + } + return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil + } + + accountID := "01FS1X72SK9ZPW0J1QQ68BD264" + + // process the media with no additional info provided + processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil) + suite.NoError(err) + // fetch the attachment id from the processing media + attachmentID := processingMedia.AttachmentID() + + // do a blocking call to fetch the attachment + attachment, err := processingMedia.LoadAttachment(ctx) + suite.NoError(err) + suite.NotNil(attachment) + + // make sure it's got the stuff set on it that we expect + // the attachment ID and accountID we expect + suite.Equal(attachmentID, attachment.ID) + suite.Equal(accountID, attachment.AccountID) + + // file meta should be correctly derived from the video + suite.Equal(600, attachment.FileMeta.Original.Width) + suite.Equal(330, attachment.FileMeta.Original.Height) + suite.Equal(198000, attachment.FileMeta.Original.Size) + suite.EqualValues(1.8181819, attachment.FileMeta.Original.Aspect) + suite.EqualValues(16.6, *attachment.FileMeta.Original.Duration) + suite.EqualValues(10, *attachment.FileMeta.Original.Framerate) + suite.EqualValues(0xc8fb, *attachment.FileMeta.Original.Bitrate) + suite.EqualValues(gtsmodel.Small{ + Width: 600, Height: 330, Size: 198000, Aspect: 1.8181819, + }, attachment.FileMeta.Small) + suite.Equal("video/mp4", attachment.File.ContentType) + suite.Equal("image/jpeg", attachment.Thumbnail.ContentType) + suite.Equal(109549, attachment.File.FileSize) + suite.Equal("", attachment.Blurhash) + + // now make sure the attachment is in the database + dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) + suite.NoError(err) + suite.NotNil(dbAttachment) + + // make sure the processed file is in storage + processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path) + suite.NoError(err) + suite.NotEmpty(processedFullBytes) + + // load the processed bytes from our test folder, to compare + processedFullBytesExpected, err := os.ReadFile("./test/longer-mp4-processed.mp4") + suite.NoError(err) + suite.NotEmpty(processedFullBytesExpected) + + // the bytes in storage should be what we expected + suite.Equal(processedFullBytesExpected, processedFullBytes) + + // now do the same for the thumbnail and make sure it's what we expected + processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path) + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytes) + + processedThumbnailBytesExpected, err := os.ReadFile("./test/longer-mp4-thumbnail.jpg") + suite.NoError(err) + suite.NotEmpty(processedThumbnailBytesExpected) + + suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes) +} + +func (suite *ManagerTestSuite) TestNotAnMp4ProcessBlocking() { + // try to load an 'mp4' that's actually an mkv in disguise + + ctx := context.Background() + + data := func(_ context.Context) (io.ReadCloser, int64, error) { + // load bytes from a test video + b, err := os.ReadFile("./test/not-an.mp4") + if err != nil { + panic(err) + } + return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil + } + + accountID := "01FS1X72SK9ZPW0J1QQ68BD264" + + // pre processing should go fine but... + processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil) + suite.NoError(err) + + // we should get an error while loading + attachment, err := processingMedia.LoadAttachment(ctx) + suite.EqualError(err, "\"video width could not be discovered\",\"video height could not be discovered\",\"video duration could not be discovered\",\"video framerate could not be discovered\",\"video bitrate could not be discovered\"") + suite.Nil(attachment) +} + func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven() { ctx := context.Background() diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go @@ -249,16 +249,32 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { } // set appropriate fields on the attachment based on the image we derived + + // generic fields + p.attachment.File.UpdatedAt = time.Now() p.attachment.FileMeta.Original = gtsmodel.Original{ Width: decoded.width, Height: decoded.height, Size: decoded.size, Aspect: decoded.aspect, } - p.attachment.File.UpdatedAt = time.Now() - p.attachment.Processing = gtsmodel.ProcessingStatusProcessed + + // nullable fields + if decoded.duration != 0 { + i := decoded.duration + p.attachment.FileMeta.Original.Duration = &i + } + if decoded.framerate != 0 { + i := decoded.framerate + p.attachment.FileMeta.Original.Framerate = &i + } + if decoded.bitrate != 0 { + i := decoded.bitrate + p.attachment.FileMeta.Original.Bitrate = &i + } // we're done processing the full-size image + p.attachment.Processing = gtsmodel.ProcessingStatusProcessed atomic.StoreInt32(&p.fullSizeState, int32(complete)) log.Tracef("finished processing full size image for attachment %s", p.attachment.URL) fallthrough diff --git a/internal/media/test/longer-mp4-original.mp4 b/internal/media/test/longer-mp4-original.mp4 Binary files differ. diff --git a/internal/media/test/longer-mp4-processed.mp4 b/internal/media/test/longer-mp4-processed.mp4 Binary files differ. diff --git a/internal/media/test/longer-mp4-thumbnail.jpg b/internal/media/test/longer-mp4-thumbnail.jpg Binary files differ. diff --git a/internal/media/test/not-an.mp4 b/internal/media/test/not-an.mp4 Binary files differ. diff --git a/internal/media/types.go b/internal/media/types.go @@ -137,7 +137,12 @@ type mediaMeta struct { width int height int size int - aspect float64 + aspect float32 blurhash string small []byte + + // video-specific properties + duration float32 + framerate float32 + bitrate uint64 } diff --git a/internal/media/video.go b/internal/media/video.go @@ -20,7 +20,6 @@ package media import ( "bytes" - "errors" "fmt" "image" "image/color" @@ -30,6 +29,7 @@ import ( "os" "github.com/abema/go-mp4" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/log" ) @@ -61,60 +61,80 @@ func decodeVideo(r io.Reader, contentType string) (*mediaMeta, error) { return nil, fmt.Errorf("could not copy video reader into temporary file %s: %w", tempFileName, err) } - // define some vars we need to pull the width/height out of the video var ( - height int - width int - readHandler = getReadHandler(&height, &width) + width int + height int + duration float32 + framerate float32 + bitrate uint64 ) - // do the actual decoding here, providing the temporary file we created as readseeker - if _, err := mp4.ReadBoxStructure(tempFile, readHandler); err != nil { - return nil, fmt.Errorf("parsing video data: %w", err) + // probe the video file to extract useful metadata from it; for methodology, see: + // https://github.com/abema/go-mp4/blob/7d8e5a7c5e644e0394261b0cf72fef79ce246d31/mp4tool/probe/probe.go#L85-L154 + info, err := mp4.Probe(tempFile) + if err != nil { + return nil, fmt.Errorf("could not probe temporary video file %s: %w", tempFileName, err) } - // width + height should now be updated by the readHandler - return &mediaMeta{ - width: width, - height: height, - size: height * width, - aspect: float64(width) / float64(height), - }, nil -} + for _, tr := range info.Tracks { + if tr.AVC == nil { + continue + } -// getReadHandler returns a handler function that updates the underling -// values of the given height and width int pointers to the hightest and -// widest points of the video. -func getReadHandler(height *int, width *int) func(h *mp4.ReadHandle) (interface{}, error) { - return func(rh *mp4.ReadHandle) (interface{}, error) { - if rh.BoxInfo.Type == mp4.BoxTypeTkhd() { - box, _, err := rh.ReadPayload() - if err != nil { - return nil, fmt.Errorf("could not read mp4 payload: %w", err) - } - - tkhd, ok := box.(*mp4.Tkhd) - if !ok { - return nil, errors.New("box was not of type *mp4.Tkhd") - } - - // if height + width of this box are greater than what - // we have stored, then update our stored values - if h := int(tkhd.GetHeight()); h > *height { - *height = h - } - - if w := int(tkhd.GetWidth()); w > *width { - *width = w - } + if w := int(tr.AVC.Width); w > width { + width = w } - if rh.BoxInfo.IsSupportedType() { - return rh.Expand() + if h := int(tr.AVC.Height); h > height { + height = h } - return nil, nil + if br := tr.Samples.GetBitrate(tr.Timescale); br > bitrate { + bitrate = br + } else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > bitrate { + bitrate = br + } + + if d := float32(tr.Duration) / float32(tr.Timescale); d > duration { + duration = d + framerate = float32(len(tr.Samples)) / duration + } } + + var errs gtserror.MultiError + if width == 0 { + errs = append(errs, "video width could not be discovered") + } + + if height == 0 { + errs = append(errs, "video height could not be discovered") + } + + if duration == 0 { + errs = append(errs, "video duration could not be discovered") + } + + if framerate == 0 { + errs = append(errs, "video framerate could not be discovered") + } + + if bitrate == 0 { + errs = append(errs, "video bitrate could not be discovered") + } + + if errs != nil { + return nil, errs.Combine() + } + + return &mediaMeta{ + width: width, + height: height, + duration: duration, + framerate: framerate, + bitrate: bitrate, + size: height * width, + aspect: float32(width) / float32(height), + }, nil } func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) { @@ -134,7 +154,7 @@ func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) { width: width, height: height, size: width * height, - aspect: float64(width) / float64(height), + aspect: float32(width) / float32(height), small: out.Bytes(), }, nil } diff --git a/internal/typeutils/converter_test.go b/internal/typeutils/converter_test.go @@ -469,11 +469,12 @@ const ( type TypeUtilsTestSuite struct { suite.Suite - db db.DB - testAccounts map[string]*gtsmodel.Account - testStatuses map[string]*gtsmodel.Status - testPeople map[string]vocab.ActivityStreamsPerson - testEmojis map[string]*gtsmodel.Emoji + db db.DB + testAccounts map[string]*gtsmodel.Account + testStatuses map[string]*gtsmodel.Status + testAttachments map[string]*gtsmodel.MediaAttachment + testPeople map[string]vocab.ActivityStreamsPerson + testEmojis map[string]*gtsmodel.Emoji typeconverter typeutils.TypeConverter } @@ -485,6 +486,7 @@ func (suite *TypeUtilsTestSuite) SetupSuite() { suite.db = testrig.NewTestDB() suite.testAccounts = testrig.NewTestAccounts() suite.testStatuses = testrig.NewTestStatuses() + suite.testAttachments = testrig.NewTestAttachments() suite.testPeople = testrig.NewTestFediPeople() suite.testEmojis = testrig.NewTestEmojis() suite.typeconverter = typeutils.NewConverter(suite.db) diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go @@ -22,11 +22,14 @@ import ( "context" "errors" "fmt" + "math" + "strconv" "strings" "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" @@ -299,26 +302,38 @@ func (c *converter) AttachmentToAPIAttachment(ctx context.Context, a *gtsmodel.M } // nullable fields - if a.URL != "" { - i := a.URL + if i := a.URL; i != "" { apiAttachment.URL = &i } - if a.RemoteURL != "" { - i := a.RemoteURL + if i := a.RemoteURL; i != "" { apiAttachment.RemoteURL = &i } - if a.Thumbnail.RemoteURL != "" { - i := a.Thumbnail.RemoteURL + if i := a.Thumbnail.RemoteURL; i != "" { apiAttachment.PreviewRemoteURL = &i } - if a.Description != "" { - i := a.Description + if i := a.Description; i != "" { apiAttachment.Description = &i } + if i := a.FileMeta.Original.Duration; i != nil { + apiAttachment.Meta.Original.Duration = *i + } + + if i := a.FileMeta.Original.Framerate; i != nil { + // the masto api expects this as a string in + // the format `integer/1`, so 30fps is `30/1` + round := math.Round(float64(*i)) + fr := strconv.FormatInt(int64(round), 10) + apiAttachment.Meta.Original.FrameRate = fr + "/1" + } + + if i := a.FileMeta.Original.Bitrate; i != nil { + apiAttachment.Meta.Original.Bitrate = int(*i) + } + return apiAttachment, nil } @@ -789,7 +804,7 @@ func (c *converter) DomainBlockToAPIDomainBlock(ctx context.Context, b *gtsmodel // convertAttachmentsToAPIAttachments will convert a slice of GTS model attachments to frontend API model attachments, falling back to IDs if no GTS models supplied. func (c *converter) convertAttachmentsToAPIAttachments(ctx context.Context, attachments []*gtsmodel.MediaAttachment, attachmentIDs []string) ([]model.Attachment, error) { - var errs multiError + var errs gtserror.MultiError if len(attachments) == 0 { // GTS model attachments were not populated @@ -826,7 +841,7 @@ func (c *converter) convertAttachmentsToAPIAttachments(ctx context.Context, atta // convertEmojisToAPIEmojis will convert a slice of GTS model emojis to frontend API model emojis, falling back to IDs if no GTS models supplied. func (c *converter) convertEmojisToAPIEmojis(ctx context.Context, emojis []*gtsmodel.Emoji, emojiIDs []string) ([]model.Emoji, error) { - var errs multiError + var errs gtserror.MultiError if len(emojis) == 0 { // GTS model attachments were not populated @@ -863,7 +878,7 @@ func (c *converter) convertEmojisToAPIEmojis(ctx context.Context, emojis []*gtsm // convertMentionsToAPIMentions will convert a slice of GTS model mentions to frontend API model mentions, falling back to IDs if no GTS models supplied. func (c *converter) convertMentionsToAPIMentions(ctx context.Context, mentions []*gtsmodel.Mention, mentionIDs []string) ([]model.Mention, error) { - var errs multiError + var errs gtserror.MultiError if len(mentions) == 0 { var err error @@ -895,7 +910,7 @@ func (c *converter) convertMentionsToAPIMentions(ctx context.Context, mentions [ // convertTagsToAPITags will convert a slice of GTS model tags to frontend API model tags, falling back to IDs if no GTS models supplied. func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.Tag, tagIDs []string) ([]model.Tag, error) { - var errs multiError + var errs gtserror.MultiError if len(tags) == 0 { // GTS model tags were not populated @@ -929,24 +944,3 @@ func (c *converter) convertTagsToAPITags(ctx context.Context, tags []*gtsmodel.T return apiTags, errs.Combine() } - -// multiError allows encapsulating multiple errors under a singular instance, -// which is useful when you only want to log on errors, not return early / bubble up. -// TODO: if this is useful elsewhere, move into a separate gts subpackage. -type multiError []string - -func (e *multiError) Append(err error) { - *e = append(*e, err.Error()) -} - -func (e *multiError) Appendf(format string, args ...any) { - *e = append(*e, fmt.Sprintf(format, args...)) -} - -// Combine converts this multiError to a singular error instance, returning nil if empty. -func (e multiError) Combine() error { - if len(e) == 0 { - return nil - } - return errors.New(`"` + strings.Join(e, `","`) + `"`) -} diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go @@ -110,6 +110,17 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":null,"uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b)) } +func (suite *InternalToFrontendTestSuite) TestVideoAttachmentToFrontend() { + testAttachment := suite.testAttachments["local_account_1_status_4_attachment_2"] + apiAttachment, err := suite.typeconverter.AttachmentToAPIAttachment(context.Background(), testAttachment) + suite.NoError(err) + + b, err := json.Marshal(apiAttachment) + suite.NoError(err) + + suite.Equal(`{"id":"01CDR64G398ADCHXK08WWTHEZ5","type":"video","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","text_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","preview_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":720,"height":404,"frame_rate":"30/1","duration":15.033334,"bitrate":1206522,"size":"720x404","aspect":1.7821782},"small":{"width":720,"height":404,"size":"720x404","aspect":1.7821782},"focus":{"x":0,"y":0}},"description":"A cow adorably licking another cow!"}`, string(b)) +} + func (suite *InternalToFrontendTestSuite) TestInstanceToFrontend() { testInstance := >smodel.Instance{ CreatedAt: testrig.TimeMustParse("2021-10-20T11:36:45Z"), diff --git a/testrig/media/cowlick-original.mp4 b/testrig/media/cowlick-original.mp4 Binary files differ. diff --git a/testrig/media/cowlick-small.jpeg b/testrig/media/cowlick-small.jpeg Binary files differ. diff --git a/testrig/testmodels.go b/testrig/testmodels.go @@ -60,6 +60,14 @@ func StringPtr(in string) *string { return &in } +func Float32Ptr(in float32) *float32 { + return &in +} + +func Uint64Ptr(in uint64) *uint64 { + return &in +} + // NewTestTokens returns a map of tokens keyed according to which account the token belongs to. func NewTestTokens() map[string]*gtsmodel.Token { tokens := map[string]*gtsmodel.Token{ @@ -772,6 +780,58 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Header: FalseBool(), Cached: TrueBool(), }, + "local_account_1_status_4_attachment_2": { + ID: "01CDR64G398ADCHXK08WWTHEZ5", + StatusID: "01F8MH82FYRXD2RC6108DAJ5HB", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4", + RemoteURL: "", + CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), + UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), + Type: gtsmodel.FileTypeVideo, + FileMeta: gtsmodel.FileMeta{ + Original: gtsmodel.Original{ + Width: 720, + Height: 404, + Size: 290880, + Aspect: 1.78217821782178, + Duration: Float32Ptr(15.033334), + Framerate: Float32Ptr(30.0), + Bitrate: Uint64Ptr(1206522), + }, + Small: gtsmodel.Small{ + Width: 720, + Height: 404, + Size: 290880, + Aspect: 1.78217821782178, + }, + Focus: gtsmodel.Focus{ + X: 0, + Y: 0, + }, + }, + AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", + Description: "A cow adorably licking another cow!", + ScheduledStatusID: "", + Blurhash: "", + Processing: 2, + File: gtsmodel.File{ + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.gif", + ContentType: "video/mp4", + FileSize: 2273532, + UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), + }, + Thumbnail: gtsmodel.Thumbnail{ + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg", + ContentType: "image/jpeg", + FileSize: 5272, + UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg", + RemoteURL: "", + }, + Avatar: FalseBool(), + Header: FalseBool(), + Cached: TrueBool(), + }, "local_account_1_unattached_1": { ID: "01F8MH8RMYQ6MSNY3JM2XT1CQ5", StatusID: "", // this attachment isn't connected to a status YET @@ -1209,6 +1269,10 @@ func newTestStoredAttachments() map[string]filenames { Original: "trent-original.gif", Small: "trent-small.jpeg", }, + "local_account_1_status_4_attachment_2": { + Original: "cowlick-original.mp4", + Small: "cowlick-small.jpeg", + }, "local_account_1_unattached_1": { Original: "ohyou-original.jpeg", Small: "ohyou-small.jpeg", @@ -1434,9 +1498,9 @@ func NewTestStatuses() map[string]*gtsmodel.Status { ID: "01F8MH82FYRXD2RC6108DAJ5HB", URI: "http://localhost:8080/users/the_mighty_zork/statuses/01F8MH82FYRXD2RC6108DAJ5HB", URL: "http://localhost:8080/@the_mighty_zork/statuses/01F8MH82FYRXD2RC6108DAJ5HB", - Content: "here's a little gif of trent", - Text: "here's a little gif of trent", - AttachmentIDs: []string{"01F8MH7TDVANYKWVE8VVKFPJTJ"}, + Content: "here's a little gif of trent.... and also a cow", + Text: "here's a little gif of trent.... and also a cow", + AttachmentIDs: []string{"01F8MH7TDVANYKWVE8VVKFPJTJ", "01CDR64G398ADCHXK08WWTHEZ5"}, CreatedAt: TimeMustParse("2021-10-20T12:40:37+02:00"), UpdatedAt: TimeMustParse("2021-10-20T12:40:37+02:00"), Local: TrueBool(), @@ -1444,7 +1508,7 @@ func NewTestStatuses() map[string]*gtsmodel.Status { AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF", InReplyToID: "", BoostOfID: "", - ContentWarning: "eye contact, trent reznor gif", + ContentWarning: "eye contact, trent reznor gif, cow", Visibility: gtsmodel.VisibilityMutualsOnly, Sensitive: FalseBool(), Language: "en",
