commit 53180548083c0a100db2f703d5f5da047a9e0031 parent 3512325e4647fc76e6ce82ea44e2a321a0809451 Author: kim <89579420+NyaaaWhatsUpDoc@users.noreply.github.com> Date: Wed, 11 Jan 2023 11:13:13 +0000 [performance] media processing improvements (#1288) * media processor consolidation and reformatting, reduce amount of required syscalls Signed-off-by: kim <grufwub@gmail.com> * update go-store library, stream jpeg/png encoding + use buffer pools, improved media processing AlreadyExists error handling Signed-off-by: kim <grufwub@gmail.com> * fix duration not being set, fix mp4 test expecting error Signed-off-by: kim <grufwub@gmail.com> * fix test expecting media files with different extension Signed-off-by: kim <grufwub@gmail.com> * remove unused code Signed-off-by: kim <grufwub@gmail.com> * fix expected storage paths in tests, update expected test thumbnails Signed-off-by: kim <grufwub@gmail.com> * remove dead code Signed-off-by: kim <grufwub@gmail.com> * fix cached presigned s3 url fetching Signed-off-by: kim <grufwub@gmail.com> * fix tests Signed-off-by: kim <grufwub@gmail.com> * fix test models Signed-off-by: kim <grufwub@gmail.com> * update media processing to use sync.Once{} for concurrency protection Signed-off-by: kim <grufwub@gmail.com> * shutup linter Signed-off-by: kim <grufwub@gmail.com> * fix passing in KVStore GetStream() as stream to PutStream() Signed-off-by: kim <grufwub@gmail.com> * fix unlocks of storage keys Signed-off-by: kim <grufwub@gmail.com> * whoops, return the error... Signed-off-by: kim <grufwub@gmail.com> * pour one out for tobi's code <3 Signed-off-by: kim <grufwub@gmail.com> * add back the byte slurping code Signed-off-by: kim <grufwub@gmail.com> * check for both ErrUnexpectedEOF and EOF Signed-off-by: kim <grufwub@gmail.com> * add back links to file format header information Signed-off-by: kim <grufwub@gmail.com> Signed-off-by: kim <grufwub@gmail.com> Diffstat:
64 files changed, 1244 insertions(+), 1370 deletions(-)
diff --git a/go.mod b/go.mod @@ -10,9 +10,9 @@ require ( codeberg.org/gruf/go-errors/v2 v2.0.2 codeberg.org/gruf/go-kv v1.5.2 codeberg.org/gruf/go-logger/v2 v2.2.1 - codeberg.org/gruf/go-mutexes v1.1.4 + codeberg.org/gruf/go-mutexes v1.1.5 codeberg.org/gruf/go-runners v1.4.0 - codeberg.org/gruf/go-store/v2 v2.0.10 + codeberg.org/gruf/go-store/v2 v2.2.1 github.com/abema/go-mp4 v0.9.0 github.com/buckket/go-blurhash v1.1.0 github.com/coreos/go-oidc/v3 v3.5.0 @@ -65,10 +65,11 @@ require ( codeberg.org/gruf/go-atomics v1.1.0 // indirect codeberg.org/gruf/go-bitutil v1.0.1 // indirect codeberg.org/gruf/go-bytes v1.0.2 // indirect - codeberg.org/gruf/go-fastcopy v1.1.1 // indirect + codeberg.org/gruf/go-fastcopy v1.1.2 // indirect codeberg.org/gruf/go-fastpath v1.0.3 // indirect codeberg.org/gruf/go-fastpath/v2 v2.0.0 // indirect codeberg.org/gruf/go-hashenc v1.0.2 // indirect + codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225 // indirect codeberg.org/gruf/go-mangler v1.2.2 // indirect codeberg.org/gruf/go-maps v1.0.3 // indirect codeberg.org/gruf/go-pools v1.1.0 // indirect diff --git a/go.sum b/go.sum @@ -56,8 +56,8 @@ codeberg.org/gruf/go-debug v1.2.0/go.mod h1:N+vSy9uJBQgpQcJUqjctvqFz7tBHJf+S/PIj codeberg.org/gruf/go-errors/v2 v2.0.0/go.mod h1:ZRhbdhvgoUA3Yw6e56kd9Ox984RrvbEFC2pOXyHDJP4= codeberg.org/gruf/go-errors/v2 v2.0.2 h1:T9CqfC+ntSIQL5mdQxwHlUMod1htpgNe3P1tugxKlT4= codeberg.org/gruf/go-errors/v2 v2.0.2/go.mod h1:6sI75OmvXE2AtRm4WUyGMEyqEOKTsfe+CA+aBXwbtJY= -codeberg.org/gruf/go-fastcopy v1.1.1 h1:HhPCeFdVR5pwiSVDnQEGJ+J2ny9b5QgfiESc0zrWQAY= -codeberg.org/gruf/go-fastcopy v1.1.1/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s= +codeberg.org/gruf/go-fastcopy v1.1.2 h1:YwmYXPsyOcRBxKEE2+w1bGAZfclHVaPijFsOVOcnNcw= +codeberg.org/gruf/go-fastcopy v1.1.2/go.mod h1:GDDYR0Cnb3U/AIfGM3983V/L+GN+vuwVMvrmVABo21s= codeberg.org/gruf/go-fastpath v1.0.1/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI= codeberg.org/gruf/go-fastpath v1.0.3 h1:3Iftz9Z2suCEgTLkQMucew+2+4Oe46JPbAM2JEhnjTU= codeberg.org/gruf/go-fastpath v1.0.3/go.mod h1:edveE/Kp3Eqi0JJm0lXYdkVrB28cNUkcb/bRGFTPqeI= @@ -65,6 +65,8 @@ codeberg.org/gruf/go-fastpath/v2 v2.0.0 h1:iAS9GZahFhyWEH0KLhFEJR+txx1ZhMXxYzu2q codeberg.org/gruf/go-fastpath/v2 v2.0.0/go.mod h1:3pPqu5nZjpbRrOqvLyAK7puS1OfEtQvjd6342Cwz56Q= codeberg.org/gruf/go-hashenc v1.0.2 h1:U3jH6zMXZiL96czD/qaJd8OR2h7LlBzGv/2WxnMHI/g= codeberg.org/gruf/go-hashenc v1.0.2/go.mod h1:eK+A8clLcEN/m1nftNsRId0kfYDQnETnuIfBGZ8Gvsg= +codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225 h1:tP9YvEBfADGG3mXkfrALLadlcbrZsFsWKZvFtUZtrt8= +codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225/go.mod h1:B8uq4yHtIcKXhBZT9C/SYisz25lldLHMVpwZPz4ADLQ= codeberg.org/gruf/go-kv v1.5.2 h1:B0RkAXLUXYn3Za1NzTXOcUvAc+JUC2ZadTMkCUDa0mc= codeberg.org/gruf/go-kv v1.5.2/go.mod h1:al6ASW/2CbGqz2YcM8B00tvWnVi1bU1CH3HYs5tZxo4= codeberg.org/gruf/go-logger/v2 v2.2.1 h1:RP2u059EQKTBFV3cN8X6xDxNk2RkzqdgXGKflKqB7Oc= @@ -73,16 +75,16 @@ codeberg.org/gruf/go-mangler v1.2.2 h1:fisdWXa6dW4p1uYdbz5Of3R4lDDFPuRqKavGI9O03 codeberg.org/gruf/go-mangler v1.2.2/go.mod h1:X/7URkFhLBAVKkTxmqF11Oxw3A6pSSxgPeHssQaiq28= codeberg.org/gruf/go-maps v1.0.3 h1:VDwhnnaVNUIy5O93CvkcE2IZXnMB1+IJjzfop9V12es= codeberg.org/gruf/go-maps v1.0.3/go.mod h1:D5LNDxlC9rsDuVQVM6JObaVGAdHB6g2dTdOdkh1aXWA= -codeberg.org/gruf/go-mutexes v1.1.4 h1:HWaIZavPL92SBJxNOlIXAmAT5CB2hAs72/lBN31jnzM= -codeberg.org/gruf/go-mutexes v1.1.4/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8= +codeberg.org/gruf/go-mutexes v1.1.5 h1:8Y8DwCGf24MyzOSaPvLrtk/B4ecVx4z+fppL6dY+PG8= +codeberg.org/gruf/go-mutexes v1.1.5/go.mod h1:1j/6/MBeBQUedAtAtysLLnBKogfOZAxdym0E3wlaBD8= codeberg.org/gruf/go-pools v1.1.0 h1:LbYP24eQLl/YI1fSU2pafiwhGol1Z1zPjRrMsXpF88s= codeberg.org/gruf/go-pools v1.1.0/go.mod h1:ZMYpt/DjQWYC3zFD3T97QWSFKs62zAUGJ/tzvgB9D68= codeberg.org/gruf/go-runners v1.4.0 h1:977nVjigAdH95+VAB/a6tyBJOKk99e60h+mfHzBs/n8= codeberg.org/gruf/go-runners v1.4.0/go.mod h1:kUM6GYL7dC+f9Sc/XuwdvB/mB4FuI4fJFb150ADMsmw= codeberg.org/gruf/go-sched v1.2.0 h1:utZl/7srVcbh30rFw42LC2/cMtak4UZRxtIOt/5riNA= codeberg.org/gruf/go-sched v1.2.0/go.mod h1:v4ueWq+fAtAw9JYt4aFXvadI1YoOqofgHQgszRYuslA= -codeberg.org/gruf/go-store/v2 v2.0.10 h1:/2iZ4j29A//EhM3XziJP6SxtdIcaAyPmJEv31+6XD8g= -codeberg.org/gruf/go-store/v2 v2.0.10/go.mod h1:KMRE173S6W2sGhuIa4jY/OPIO65F9++7rmWTfZ4xTeY= +codeberg.org/gruf/go-store/v2 v2.2.1 h1:lbvMjhMLebefiaPNLtWvPySKSYM5xN1aztSxxz+vCzU= +codeberg.org/gruf/go-store/v2 v2.2.1/go.mod h1:pxdyfSzau8fFs1TfZlyRzhDYvZWLaj1sXpcjXpzBB6k= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -818,6 +820,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/api/client/accounts/accountupdate_test.go b/internal/api/client/accounts/accountupdate_test.go @@ -300,8 +300,8 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerWit suite.NotEmpty(apimodelAccount.HeaderStatic) // should be different from the values set before - suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.Header) - suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.HeaderStatic) + suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.Header) + suite.NotEqual("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.HeaderStatic) } func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandlerEmptyForm() { diff --git a/internal/api/client/accounts/accountverify_test.go b/internal/api/client/accounts/accountverify_test.go @@ -74,10 +74,10 @@ func (suite *AccountVerifyTestSuite) TestAccountVerifyGet() { suite.Equal(*testAccount.Bot, apimodelAccount.Bot) suite.WithinDuration(testAccount.CreatedAt, createdAt, 30*time.Second) // we lose a bit of accuracy serializing so fuzz this a bit suite.Equal(testAccount.URL, apimodelAccount.URL) - suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg", apimodelAccount.Avatar) - suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg", apimodelAccount.AvatarStatic) - suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.Header) - suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", apimodelAccount.HeaderStatic) + suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg", apimodelAccount.Avatar) + suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg", apimodelAccount.AvatarStatic) + suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.Header) + suite.Equal("http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", apimodelAccount.HeaderStatic) suite.Equal(2, apimodelAccount.FollowersCount) suite.Equal(2, apimodelAccount.FollowingCount) suite.Equal(5, apimodelAccount.StatusesCount) diff --git a/internal/api/fileserver/servefile.go b/internal/api/fileserver/servefile.go @@ -117,14 +117,19 @@ func (m *Module) ServeFile(c *gin.Context) { return } - // try to slurp the first few bytes to make sure we have something - b := bytes.NewBuffer(make([]byte, 0, 64)) - if _, err := io.CopyN(b, content.Content, 64); err != nil { + // create a "slurp" buffer ;) + b := make([]byte, 64) + + // Try read the first 64 bytes into memory, to try return a more useful "not found" error. + if _, err := io.ReadFull(content.Content, b); err != nil && + (err != io.ErrUnexpectedEOF && err != io.EOF) { err = fmt.Errorf("ServeFile: error reading from content: %w", err) apiutil.ErrorHandler(c, gtserror.NewErrorNotFound(err, err.Error()), m.processor.InstanceGet) return } // we're good, return the slurped bytes + the rest of the content - c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader(b, content.Content), nil) + c.DataFromReader(http.StatusOK, content.ContentLength, format, io.MultiReader( + bytes.NewReader(b), content.Content, + ), nil) } diff --git a/internal/api/fileserver/servefile_test.go b/internal/api/fileserver/servefile_test.go @@ -99,7 +99,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalLocalFileOK() { targetAttachment.AccountID, media.TypeAttachment, media.SizeOriginal, - targetAttachment.ID+".jpeg", + targetAttachment.ID+".jpg", ) suite.Equal(http.StatusOK, code) @@ -119,7 +119,7 @@ func (suite *ServeFileTestSuite) TestServeSmallLocalFileOK() { targetAttachment.AccountID, media.TypeAttachment, media.SizeSmall, - targetAttachment.ID+".jpeg", + targetAttachment.ID+".jpg", ) suite.Equal(http.StatusOK, code) @@ -139,7 +139,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileOK() { targetAttachment.AccountID, media.TypeAttachment, media.SizeOriginal, - targetAttachment.ID+".jpeg", + targetAttachment.ID+".jpg", ) suite.Equal(http.StatusOK, code) @@ -159,7 +159,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileOK() { targetAttachment.AccountID, media.TypeAttachment, media.SizeSmall, - targetAttachment.ID+".jpeg", + targetAttachment.ID+".jpg", ) suite.Equal(http.StatusOK, code) @@ -182,7 +182,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecache() { targetAttachment.AccountID, media.TypeAttachment, media.SizeOriginal, - targetAttachment.ID+".jpeg", + targetAttachment.ID+".jpg", ) suite.Equal(http.StatusOK, code) @@ -205,7 +205,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecache() { targetAttachment.AccountID, media.TypeAttachment, media.SizeSmall, - targetAttachment.ID+".jpeg", + targetAttachment.ID+".jpg", ) suite.Equal(http.StatusOK, code) @@ -228,7 +228,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalRemoteFileRecacheNotFound() { targetAttachment.AccountID, media.TypeAttachment, media.SizeOriginal, - targetAttachment.ID+".jpeg", + targetAttachment.ID+".jpg", ) suite.Equal(http.StatusNotFound, code) @@ -249,7 +249,7 @@ func (suite *ServeFileTestSuite) TestServeSmallRemoteFileRecacheNotFound() { targetAttachment.AccountID, media.TypeAttachment, media.SizeSmall, - targetAttachment.ID+".jpeg", + targetAttachment.ID+".jpg", ) suite.Equal(http.StatusNotFound, code) @@ -261,7 +261,7 @@ func (suite *ServeFileTestSuite) TestServeFileNotFound() { "01GMMY4G9B0QEG0PQK5Q5JGJWZ", media.TypeAttachment, media.SizeOriginal, - "01GMMY68Y7E5DJ3CA3Y9SS8524.jpeg", + "01GMMY68Y7E5DJ3CA3Y9SS8524.jpg", ) suite.Equal(http.StatusNotFound, code) diff --git a/internal/federation/dereferencing/media_test.go b/internal/federation/dereferencing/media_test.go @@ -21,11 +21,11 @@ package dereferencing_test import ( "context" "testing" + "time" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" - "github.com/superseriousbusiness/gotosocial/testrig" ) type AttachmentTestSuite struct { @@ -42,7 +42,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentBlocking() { attachmentContentType := "image/jpeg" attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" attachmentDescription := "It's a cute plushie." - attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" + attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az" media, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{ StatusID: &attachmentStatus, @@ -116,7 +116,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() { attachmentContentType := "image/jpeg" attachmentURL := "https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg" attachmentDescription := "It's a cute plushie." - attachmentBlurhash := "LwP?p=aK_4%N%MRjWXt7%hozM_a}" + attachmentBlurhash := "LtQ9yKi__4%g%MRjWCt7%hozM_az" processingMedia, err := suite.dereferencer.GetRemoteMedia(ctx, fetchingAccount.Username, attachmentOwner, attachmentURL, &media.AdditionalMediaInfo{ StatusID: &attachmentStatus, @@ -127,11 +127,7 @@ func (suite *AttachmentTestSuite) TestDereferenceAttachmentAsync() { suite.NoError(err) attachmentID := processingMedia.AttachmentID() - if !testrig.WaitFor(func() bool { - return processingMedia.Finished() - }) { - suite.FailNow("timed out waiting for media to be processed") - } + time.Sleep(time.Second * 3) // now get the attachment from the database attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) diff --git a/internal/iotools/io.go b/internal/iotools/io.go @@ -119,3 +119,41 @@ func (w *SilentWriter) Write(b []byte) (int, error) { func (w *SilentWriter) Error() error { return w.err } + +func StreamReadFunc(read func(io.Reader) error) io.Writer { + // In-memory stream. + pr, pw := io.Pipe() + + go func() { + var err error + + defer func() { + // Always pass along error. + pr.CloseWithError(err) + }() + + // Start reading. + err = read(pr) + }() + + return pw +} + +func StreamWriteFunc(write func(io.Writer) error) io.Reader { + // In-memory stream. + pr, pw := io.Pipe() + + go func() { + var err error + + defer func() { + // Always pass along error. + pw.CloseWithError(err) + }() + + // Start writing. + err = write(pw) + }() + + return pr +} diff --git a/internal/media/image.go b/internal/media/image.go @@ -19,182 +19,167 @@ package media import ( - "bytes" - "errors" - "fmt" + "bufio" "image" - "image/gif" + "image/color" + "image/draw" "image/jpeg" "image/png" "io" + "sync" "github.com/buckket/go-blurhash" "github.com/disintegration/imaging" - _ "golang.org/x/image/webp" // blank import to support WebP decoding + "github.com/superseriousbusiness/gotosocial/internal/iotools" + + // import to init webp encode/decoding. + _ "golang.org/x/image/webp" ) -const ( - thumbnailMaxWidth = 512 - thumbnailMaxHeight = 512 +var ( + // pngEncoder provides our global PNG encoding with + // specified compression level, and memory pooled buffers. + pngEncoder = png.Encoder{ + CompressionLevel: png.DefaultCompression, + BufferPool: &pngEncoderBufferPool{}, + } + + // jpegBufferPool is a memory pool of byte buffers for JPEG encoding. + jpegBufferPool = sync.Pool{ + New: func() any { + return bufio.NewWriter(nil) + }, + } ) -func decodeGif(r io.Reader) (*mediaMeta, error) { - gif, err := gif.DecodeAll(r) +// gtsImage is a thin wrapper around the standard library image +// interface to provide our own useful helper functions for image +// size and aspect ratio calculations, streamed encoding to various +// types, and creating reduced size thumbnail images. +type gtsImage struct{ image image.Image } + +// blankImage generates a blank image of given dimensions. +func blankImage(width int, height int) *gtsImage { + // create a rectangle with the same dimensions as the video + img := image.NewRGBA(image.Rect(0, 0, width, height)) + + // fill the rectangle with our desired fill color. + draw.Draw(img, img.Bounds(), &image.Uniform{ + color.RGBA{42, 43, 47, 0}, + }, image.Point{}, draw.Src) + + return >sImage{image: img} +} + +// decodeImage will decode image from reader stream and return image wrapped in our own gtsImage{} type. +func decodeImage(r io.Reader, opts ...imaging.DecodeOption) (*gtsImage, error) { + img, err := imaging.Decode(r, opts...) if err != nil { return nil, err } + return >sImage{image: img}, nil +} - // use the first frame to get the static characteristics - width := gif.Config.Width - height := gif.Config.Height - size := width * height - aspect := float32(width) / float32(height) - - return &mediaMeta{ - width: width, - height: height, - size: size, - aspect: aspect, - }, nil +// Width returns the image width in pixels. +func (m *gtsImage) Width() uint32 { + return uint32(m.image.Bounds().Size().X) } -func decodeImage(r io.Reader, contentType string) (*mediaMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImageJpeg, mimeImageWebp: - i, err = imaging.Decode(r, imaging.AutoOrientation(true)) - case mimeImagePng: - strippedPngReader := io.Reader(&PNGAncillaryChunkStripper{ - Reader: r, - }) - i, err = imaging.Decode(strippedPngReader, imaging.AutoOrientation(true)) - default: - err = fmt.Errorf("content type %s not recognised", contentType) - } +// Height returns the image height in pixels. +func (m *gtsImage) Height() uint32 { + return uint32(m.image.Bounds().Size().Y) +} - if err != nil { - return nil, err - } +// Size returns the total number of image pixels. +func (m *gtsImage) Size() uint64 { + return uint64(m.image.Bounds().Size().X) * + uint64(m.image.Bounds().Size().Y) +} + +// AspectRatio returns the image ratio of width:height. +func (m *gtsImage) AspectRatio() float32 { + return float32(m.image.Bounds().Size().X) / + float32(m.image.Bounds().Size().Y) +} - if i == nil { - return nil, errors.New("processed image was nil") +// Thumbnail returns a small sized copy of gtsImage{}, limited to 512x512 if not small enough. +func (m *gtsImage) Thumbnail() *gtsImage { + const ( + // max thumb + // dimensions. + maxWidth = 512 + maxHeight = 512 + ) + + // Check the receiving image is within max thumnail bounds. + if m.Width() <= maxWidth && m.Height() <= maxHeight { + return >sImage{image: imaging.Clone(m.image)} } - width := i.Bounds().Size().X - height := i.Bounds().Size().Y - size := width * height - aspect := float32(width) / float32(height) - - return &mediaMeta{ - width: width, - height: height, - size: size, - aspect: aspect, - }, nil + // Image is too large, needs to be resized to thumbnail max. + img := imaging.Fit(m.image, maxWidth, maxHeight, imaging.Linear) + return >sImage{image: img} } -// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png. -func deriveStaticEmoji(r io.Reader, contentType string) (*mediaMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImagePng: - i, err = StrippedPngDecode(r) - if err != nil { - return nil, err - } - case mimeImageGif: - i, err = gif.Decode(r) - if err != nil { - return nil, err - } - default: - return nil, fmt.Errorf("content type %s not allowed for emoji", contentType) - } +// Blurhash calculates the blurhash for the receiving image data. +func (m *gtsImage) Blurhash() (string, error) { + // for generating blurhashes, it's more cost effective to + // lose detail since it's blurry, so make a tiny version. + tiny := imaging.Resize(m.image, 32, 0, imaging.NearestNeighbor) - out := &bytes.Buffer{} - if err := png.Encode(out, i); err != nil { - return nil, err - } - return &mediaMeta{ - small: out.Bytes(), - }, nil + // Encode blurhash from resized version + return blurhash.Encode(4, 3, tiny) } -// deriveThumbnailFromImage returns a byte slice and metadata for a thumbnail -// of a given piece of media, or an error if something goes wrong. -// -// If createBlurhash is true, then a blurhash will also be generated from a tiny -// version of the image. This costs precious CPU cycles, so only use it if you -// really need a blurhash and don't have one already. -// -// If createBlurhash is false, then the blurhash field on the returned ImageAndMeta -// will be an empty string. -func deriveThumbnailFromImage(r io.Reader, contentType string, createBlurhash bool) (*mediaMeta, error) { - var i image.Image - var err error - - switch contentType { - case mimeImageJpeg, mimeImageGif, mimeImageWebp: - i, err = imaging.Decode(r, imaging.AutoOrientation(true)) - case mimeImagePng: - strippedPngReader := io.Reader(&PNGAncillaryChunkStripper{ - Reader: r, - }) - i, err = imaging.Decode(strippedPngReader, imaging.AutoOrientation(true)) - default: - err = fmt.Errorf("content type %s can't be thumbnailed as an image", contentType) - } +// ToJPEG creates a new streaming JPEG encoder from receiving image, and a size ptr +// which stores the number of bytes written during the image encoding process. +func (m *gtsImage) ToJPEG(opts *jpeg.Options) io.Reader { + return iotools.StreamWriteFunc(func(w io.Writer) error { + // Get encoding buffer + bw := getJPEGBuffer(w) - if err != nil { - return nil, fmt.Errorf("error decoding %s: %s", contentType, err) - } + // Encode JPEG to buffered writer. + err := jpeg.Encode(bw, m.image, opts) - originalX := i.Bounds().Size().X - originalY := i.Bounds().Size().Y + // Replace buffer. + // + // NOTE: jpeg.Encode() already + // performs a bufio.Writer.Flush(). + putJPEGBuffer(bw) - var thumb image.Image - if originalX <= thumbnailMaxWidth && originalY <= thumbnailMaxHeight { - // it's already small, no need to resize - thumb = i - } else { - thumb = imaging.Fit(i, thumbnailMaxWidth, thumbnailMaxHeight, imaging.Linear) - } + return err + }) +} - thumbX := thumb.Bounds().Size().X - thumbY := thumb.Bounds().Size().Y - size := thumbX * thumbY - aspect := float32(thumbX) / float32(thumbY) +// ToPNG creates a new streaming PNG encoder from receiving image, and a size ptr +// which stores the number of bytes written during the image encoding process. +func (m *gtsImage) ToPNG() io.Reader { + return iotools.StreamWriteFunc(func(w io.Writer) error { + return pngEncoder.Encode(w, m.image) + }) +} - im := &mediaMeta{ - width: thumbX, - height: thumbY, - size: size, - aspect: aspect, - } +// getJPEGBuffer fetches a reset JPEG encoding buffer from global JPEG buffer pool. +func getJPEGBuffer(w io.Writer) *bufio.Writer { + buf, _ := jpegBufferPool.Get().(*bufio.Writer) + buf.Reset(w) + return buf +} - if createBlurhash { - // for generating blurhashes, it's more cost effective to lose detail rather than - // pass a big image into the blurhash algorithm, so make a teeny tiny version - tiny := imaging.Resize(thumb, 32, 0, imaging.NearestNeighbor) - bh, err := blurhash.Encode(4, 3, tiny) - if err != nil { - return nil, fmt.Errorf("error creating blurhash: %s", err) - } - im.blurhash = bh - } +// putJPEGBuffer resets the given bufio writer and places in global JPEG buffer pool. +func putJPEGBuffer(buf *bufio.Writer) { + buf.Reset(nil) + jpegBufferPool.Put(buf) +} - out := &bytes.Buffer{} - if err := jpeg.Encode(out, thumb, &jpeg.Options{ - // Quality isn't extremely important for thumbnails, so 75 is "good enough" - Quality: 75, - }); err != nil { - return nil, fmt.Errorf("error encoding thumbnail: %s", err) - } - im.small = out.Bytes() +// pngEncoderBufferPool implements png.EncoderBufferPool. +type pngEncoderBufferPool sync.Pool + +func (p *pngEncoderBufferPool) Get() *png.EncoderBuffer { + buf, _ := (*sync.Pool)(p).Get().(*png.EncoderBuffer) + return buf +} - return im, nil +func (p *pngEncoderBufferPool) Put(buf *png.EncoderBuffer) { + (*sync.Pool)(p).Put(buf) } diff --git a/internal/media/manager.go b/internal/media/manager.go @@ -148,9 +148,6 @@ func NewManager(database db.DB, storage *storage.Driver) (Manager, error) { // Prepare the media worker pool m.mediaWorker = concurrency.NewWorkerPool[*ProcessingMedia](-1, 10) m.mediaWorker.SetProcessor(func(ctx context.Context, media *ProcessingMedia) error { - if err := ctx.Err(); err != nil { - return err - } if _, err := media.LoadAttachment(ctx); err != nil { return fmt.Errorf("error loading media %s: %v", media.AttachmentID(), err) } @@ -160,9 +157,6 @@ func NewManager(database db.DB, storage *storage.Driver) (Manager, error) { // Prepare the emoji worker pool m.emojiWorker = concurrency.NewWorkerPool[*ProcessingEmoji](-1, 10) m.emojiWorker.SetProcessor(func(ctx context.Context, emoji *ProcessingEmoji) error { - if err := ctx.Err(); err != nil { - return err - } if _, err := emoji.LoadEmoji(ctx); err != nil { return fmt.Errorf("error loading emoji %s: %v", emoji.EmojiID(), err) } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go @@ -26,6 +26,7 @@ import ( "os" "path" "testing" + "time" "codeberg.org/gruf/go-store/v2/kv" "codeberg.org/gruf/go-store/v2/storage" @@ -33,7 +34,6 @@ import ( gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/media" gtsstorage "github.com/superseriousbusiness/gotosocial/internal/storage" - "github.com/superseriousbusiness/gotosocial/testrig" ) type ManagerTestSuite struct { @@ -214,7 +214,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() { // do a blocking call to fetch the emoji emoji, err := processingEmoji.LoadEmoji(ctx) - suite.EqualError(err, "store: given emoji fileSize (645688b) is larger than allowed size (51200b)") + suite.EqualError(err, "given emoji size 630kiB greater than max allowed 50.0kiB") suite.Nil(emoji) } @@ -227,7 +227,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() { if err != nil { panic(err) } - return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil + return io.NopCloser(bytes.NewBuffer(b)), -1, nil } emojiID := "01GDQ9G782X42BAMFASKP64343" @@ -238,7 +238,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() { // do a blocking call to fetch the emoji emoji, err := processingEmoji.LoadEmoji(ctx) - suite.EqualError(err, "store: given emoji fileSize (645688b) is larger than allowed size (51200b)") + suite.EqualError(err, "calculated emoji size 630kiB greater than max allowed 50.0kiB") suite.Nil(emoji) } @@ -396,6 +396,9 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() { // fetch the attachment id from the processing media attachmentID := processingMedia.AttachmentID() + // Give time for processing + time.Sleep(time.Second * 3) + // do a blocking call to fetch the attachment attachment, err := processingMedia.LoadAttachment(ctx) suite.NoError(err) @@ -420,7 +423,7 @@ func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() { suite.Equal("video/mp4", attachment.File.ContentType) suite.Equal("image/jpeg", attachment.Thumbnail.ContentType) suite.Equal(312413, attachment.File.FileSize) - suite.Equal("", attachment.Blurhash) + suite.Equal("L00000fQfQfQfQfQfQfQfQfQfQfQ", attachment.Blurhash) // now make sure the attachment is in the database dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) @@ -491,12 +494,12 @@ func (suite *ManagerTestSuite) TestLongerMp4ProcessBlocking() { suite.EqualValues(10, *attachment.FileMeta.Original.Framerate) suite.EqualValues(0xc8fb, *attachment.FileMeta.Original.Bitrate) suite.EqualValues(gtsmodel.Small{ - Width: 600, Height: 330, Size: 198000, Aspect: 1.8181819, + Width: 512, Height: 281, Size: 143872, Aspect: 1.822064, }, attachment.FileMeta.Small) suite.Equal("video/mp4", attachment.File.ContentType) suite.Equal("image/jpeg", attachment.Thumbnail.ContentType) suite.Equal(109549, attachment.File.FileSize) - suite.Equal("", attachment.Blurhash) + suite.Equal("L00000fQfQfQfQfQfQfQfQfQfQfQ", attachment.Blurhash) // now make sure the attachment is in the database dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) @@ -550,7 +553,7 @@ func (suite *ManagerTestSuite) TestNotAnMp4ProcessBlocking() { // we should get an error while loading attachment, err := processingMedia.LoadAttachment(ctx) - suite.EqualError(err, "\"video width could not be discovered\",\"video height could not be discovered\",\"video duration could not be discovered\",\"video framerate could not be discovered\",\"video bitrate could not be discovered\"") + suite.EqualError(err, "error decoding video: error determining video metadata: [width height duration framerate bitrate]") suite.Nil(attachment) } @@ -928,7 +931,8 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { } func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { - ctx := context.Background() + ctx, cncl := context.WithTimeout(context.Background(), time.Second*30) + defer cncl() data := func(_ context.Context) (io.ReadCloser, int64, error) { // load bytes from a test image @@ -944,15 +948,12 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { // process the media with no additional info provided processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil) suite.NoError(err) + // fetch the attachment id from the processing media attachmentID := processingMedia.AttachmentID() - // wait for the media to finish processing - if !testrig.WaitFor(func() bool { - return processingMedia.Finished() - }) { - suite.FailNow("timed out waiting for media to be processed") - } + // Give time for processing to happen. + time.Sleep(time.Second * 3) // fetch the attachment from the database attachment, err := suite.db.GetAttachmentByID(ctx, attachmentID) diff --git a/internal/media/png-stripper.go b/internal/media/png-stripper.go @@ -75,8 +75,6 @@ package media import ( "encoding/binary" - "image" - "image/png" "io" ) @@ -192,13 +190,3 @@ func (r *PNGAncillaryChunkStripper) Read(p []byte) (int, error) { } } } - -// StrippedPngDecode strips ancillary data from png to allow more lenient decoding of pngs -// see: https://github.com/golang/go/issues/43382 -// and: https://github.com/google/wuffs/blob/414a011491ff513b86d8694c5d71800f3cb5a715/script/strip-png-ancillary-chunks.go -func StrippedPngDecode(r io.Reader) (image.Image, error) { - strippedPngReader := io.Reader(&PNGAncillaryChunkStripper{ - Reader: r, - }) - return png.Decode(strippedPngReader) -} diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go @@ -24,84 +24,74 @@ import ( "errors" "fmt" "io" - "strings" "sync" - "sync/atomic" "time" + "codeberg.org/gruf/go-bytesize" gostore "codeberg.org/gruf/go-store/v2/storage" + "github.com/h2non/filetype" "github.com/superseriousbusiness/gotosocial/internal/config" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/uris" ) // ProcessingEmoji represents an emoji currently processing. It exposes // various functions for retrieving data from the process. type ProcessingEmoji struct { - mu sync.Mutex - - // id of this instance's account -- pinned for convenience here so we only need to fetch it once - instanceAccountID string - - /* - below fields should be set on newly created media; - emoji will be updated incrementally as media goes through processing - */ - - emoji *gtsmodel.Emoji - data DataFunc - postData PostDataCallbackFunc - read bool // bool indicating that data function has been triggered already - - /* - below fields represent the processing state of the static of the emoji - */ - staticState int32 - - /* - below pointers to database and storage are maintained so that - the media can store and update itself during processing steps - */ - - database db.DB - storage *storage.Driver - - err error // error created during processing, if any - - // track whether this emoji has already been put in the databse - insertedInDB bool - - // is this a refresh of an existing emoji? - refresh bool - // if it is a refresh, which alternate ID should we use in the storage and URL paths? - newPathID string + instAccID string // instance account ID + emoji *gtsmodel.Emoji // processing emoji details + refresh bool // whether this is an existing emoji being refreshed + newPathID string // new emoji path ID to use if refreshed + dataFn DataFunc // load-data function, returns media stream + postFn PostDataCallbackFunc // post data callback function + err error // error encountered during processing + manager *manager // manager instance (access to db / storage) + once sync.Once // once ensures processing only occurs once } // EmojiID returns the ID of the underlying emoji without blocking processing. func (p *ProcessingEmoji) EmojiID() string { - return p.emoji.ID + return p.emoji.ID // immutable, safe outside mutex. } // LoadEmoji blocks until the static and fullsize image // has been processed, and then returns the completed emoji. func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error) { - p.mu.Lock() - defer p.mu.Unlock() + // only process once. + p.once.Do(func() { + var err error + + defer func() { + if r := recover(); r != nil { + if err != nil { + rOld := r // wrap the panic so we don't lose existing returned error + r = fmt.Errorf("panic occured after error %q: %v", err.Error(), rOld) + } - if err := p.store(ctx); err != nil { - return nil, err - } + // Catch any panics and wrap as error. + err = fmt.Errorf("caught panic: %v", r) + } - if err := p.loadStatic(ctx); err != nil { - return nil, err - } + if err != nil { + // Store error. + p.err = err + } + }() + + // Attempt to store media and calculate + // full-size media attachment details. + if err = p.store(ctx); err != nil { + return + } + + // Finish processing by reloading media into + // memory to get dimension and generate a thumb. + if err = p.finish(ctx); err != nil { + return + } - // store the result in the database before returning it - if !p.insertedInDB { if p.refresh { columns := []string{ "updated_at", @@ -118,176 +108,195 @@ func (p *ProcessingEmoji) LoadEmoji(ctx context.Context) (*gtsmodel.Emoji, error "shortcode", "uri", } - if _, err := p.database.UpdateEmoji(ctx, p.emoji, columns...); err != nil { - return nil, err - } - } else { - if err := p.database.PutEmoji(ctx, p.emoji); err != nil { - return nil, err - } - } - p.insertedInDB = true - } - - return p.emoji, nil -} - -// Finished returns true if processing has finished for both the thumbnail -// and full fized version of this piece of media. -func (p *ProcessingEmoji) Finished() bool { - return atomic.LoadInt32(&p.staticState) == int32(complete) -} -func (p *ProcessingEmoji) loadStatic(ctx context.Context) error { - staticState := atomic.LoadInt32(&p.staticState) - switch processState(staticState) { - case received: - // stream the original file out of storage... - stored, err := p.storage.GetStream(ctx, p.emoji.ImagePath) - if err != nil { - p.err = fmt.Errorf("loadStatic: error fetching file from storage: %s", err) - atomic.StoreInt32(&p.staticState, int32(errored)) - return p.err + // Existing emoji we're refreshing, so only need to update. + _, err = p.manager.db.UpdateEmoji(ctx, p.emoji, columns...) + return } - defer stored.Close() - // we haven't processed a static version of this emoji yet so do it now - static, err := deriveStaticEmoji(stored, p.emoji.ImageContentType) - if err != nil { - p.err = fmt.Errorf("loadStatic: error deriving static: %s", err) - atomic.StoreInt32(&p.staticState, int32(errored)) - return p.err - } - - // Close stored emoji now we're done - if err := stored.Close(); err != nil { - log.Errorf("loadStatic: error closing stored full size: %s", err) - } - - // put the static image in storage - if err := p.storage.Put(ctx, p.emoji.ImageStaticPath, static.small); err != nil && err != storage.ErrAlreadyExists { - p.err = fmt.Errorf("loadStatic: error storing static: %s", err) - atomic.StoreInt32(&p.staticState, int32(errored)) - return p.err - } + // New emoji media, first time caching. + err = p.manager.db.PutEmoji(ctx, p.emoji) + return //nolint shutup linter i like this here + }) - p.emoji.ImageStaticFileSize = len(static.small) - - // we're done processing the static version of the emoji! - atomic.StoreInt32(&p.staticState, int32(complete)) - fallthrough - case complete: - return nil - case errored: - return p.err + if p.err != nil { + return nil, p.err } - return fmt.Errorf("static processing status %d unknown", p.staticState) + return p.emoji, nil } // store calls the data function attached to p if it hasn't been called yet, // and updates the underlying attachment fields as necessary. It will then stream // bytes from p's reader directly into storage so that it can be retrieved later. func (p *ProcessingEmoji) store(ctx context.Context) error { - // check if we've already done this and bail early if we have - if p.read { - return nil - } + defer func() { + if p.postFn == nil { + return + } - // execute the data function to get the readcloser out of it - rc, fileSize, err := p.data(ctx) + // Ensure post callback gets called. + if err := p.postFn(ctx); err != nil { + log.Errorf("error executing postdata function: %v", err) + } + }() + + // Load media from provided data fn. + rc, sz, err := p.dataFn(ctx) if err != nil { - return fmt.Errorf("store: error executing data function: %s", err) + return fmt.Errorf("error executing data function: %w", err) } - // defer closing the reader when we're done with it defer func() { + // Ensure data reader gets closed on return. if err := rc.Close(); err != nil { - log.Errorf("store: error closing readcloser: %s", err) + log.Errorf("error closing data reader: %v", err) } }() - // execute the postData function no matter what happens - defer func() { - if p.postData != nil { - if err := p.postData(ctx); err != nil { - log.Errorf("store: error executing postData: %s", err) - } - } - }() + // Byte buffer to read file header into. + // See: https://en.wikipedia.org/wiki/File_format#File_header + // and https://github.com/h2non/filetype + hdrBuf := make([]byte, 261) - // extract no more than 261 bytes from the beginning of the file -- this is the header - firstBytes := make([]byte, maxFileHeaderBytes) - if _, err := rc.Read(firstBytes); err != nil { - return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err) + // Read the first 261 header bytes into buffer. + if _, err := io.ReadFull(rc, hdrBuf); err != nil { + return fmt.Errorf("error reading incoming media: %w", err) } - // now we have the file header we can work out the content type from it - contentType, err := parseContentType(firstBytes) + // Parse file type info from header buffer. + info, err := filetype.Match(hdrBuf) if err != nil { - return fmt.Errorf("store: error parsing content type: %s", err) + return fmt.Errorf("error parsing file type: %w", err) } - // bail if this is a type we can't process - if !supportedEmoji(contentType) { - return fmt.Errorf("store: content type %s was not valid for an emoji", contentType) + switch info.Extension { + // only supported emoji types + case "gif", "png": + + // unhandled + default: + return fmt.Errorf("unsupported emoji filetype: %s", info.Extension) } - // extract the file extension - split := strings.Split(contentType, "/") - extension := split[1] // something like 'gif' + // Recombine header bytes with remaining stream + r := io.MultiReader(bytes.NewReader(hdrBuf), rc) + + var maxSize bytesize.Size + + if p.emoji.Domain == "" { + // this is a local emoji upload + maxSize = config.GetMediaEmojiLocalMaxSize() + } else { + // this is a remote incoming emoji + maxSize = config.GetMediaEmojiRemoteMaxSize() + } + + // Check that provided size isn't beyond max. We check beforehand + // so that we don't attempt to stream the emoji into storage if not needed. + if size := bytesize.Size(sz); sz > 0 && size > maxSize { + return fmt.Errorf("given emoji size %s greater than max allowed %s", size, maxSize) + } - // set some additional fields on the emoji now that - // we know more about what the underlying image actually is var pathID string + if p.refresh { + // This is a refreshed emoji with a new + // path ID that this will be stored under. pathID = p.newPathID } else { + // This is a new emoji, simply use provided ID. pathID = p.emoji.ID } - p.emoji.ImageURL = uris.GenerateURIForAttachment(p.instanceAccountID, string(TypeEmoji), string(SizeOriginal), pathID, extension) - p.emoji.ImagePath = fmt.Sprintf("%s/%s/%s/%s.%s", p.instanceAccountID, TypeEmoji, SizeOriginal, pathID, extension) - p.emoji.ImageContentType = contentType - // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) - readerToStore := io.MultiReader(bytes.NewBuffer(firstBytes), rc) + // Calculate emoji file path. + p.emoji.ImagePath = fmt.Sprintf( + "%s/%s/%s/%s.%s", + p.instAccID, + TypeEmoji, + SizeOriginal, + pathID, + info.Extension, + ) + + // This shouldn't already exist, but we do a check as it's worth logging. + if have, _ := p.manager.storage.Has(ctx, p.emoji.ImagePath); have { + log.Warnf("emoji already exists at storage path: %s", p.emoji.ImagePath) + + // Attempt to remove existing emoji at storage path (might be broken / out-of-date) + if err := p.manager.storage.Delete(ctx, p.emoji.ImagePath); err != nil { + return fmt.Errorf("error removing emoji from storage: %v", err) + } + } - var maxEmojiSize int64 - if p.emoji.Domain == "" { - maxEmojiSize = int64(config.GetMediaEmojiLocalMaxSize()) - } else { - maxEmojiSize = int64(config.GetMediaEmojiRemoteMaxSize()) + // Write the final image reader stream to our storage. + sz, err = p.manager.storage.PutStream(ctx, p.emoji.ImagePath, r) + if err != nil { + return fmt.Errorf("error writing emoji to storage: %w", err) } - // if we know the fileSize already, make sure it's not bigger than our limit - var checkedSize bool - if fileSize > 0 { - checkedSize = true - if fileSize > maxEmojiSize { - return fmt.Errorf("store: given emoji fileSize (%db) is larger than allowed size (%db)", fileSize, maxEmojiSize) + // Once again check size in case none was provided previously. + if size := bytesize.Size(sz); size > maxSize { + if err := p.manager.storage.Delete(ctx, p.emoji.ImagePath); err != nil { + log.Errorf("error removing too-large-emoji from storage: %v", err) } + return fmt.Errorf("calculated emoji size %s greater than max allowed %s", size, maxSize) } - // store this for now -- other processes can pull it out of storage as they please - if fileSize, err = putStream(ctx, p.storage, p.emoji.ImagePath, readerToStore, fileSize); err != nil { - if !errors.Is(err, storage.ErrAlreadyExists) { - return fmt.Errorf("store: error storing stream: %s", err) - } - log.Warnf("emoji %s already exists at storage path: %s", p.emoji.ID, p.emoji.ImagePath) + // Fill in remaining attachment data now it's stored. + p.emoji.ImageURL = uris.GenerateURIForAttachment( + p.instAccID, + string(TypeEmoji), + string(SizeOriginal), + pathID, + info.Extension, + ) + p.emoji.ImageContentType = info.MIME.Value + p.emoji.ImageFileSize = int(sz) + + return nil +} + +func (p *ProcessingEmoji) finish(ctx context.Context) error { + // Fetch a stream to the original file in storage. + rc, err := p.manager.storage.GetStream(ctx, p.emoji.ImagePath) + if err != nil { + return fmt.Errorf("error loading file from storage: %w", err) } + defer rc.Close() - // if we didn't know the fileSize yet, we do now, so check if we need to - if !checkedSize && fileSize > maxEmojiSize { - err = fmt.Errorf("store: discovered emoji fileSize (%db) is larger than allowed emojiRemoteMaxSize (%db), will delete from the store now", fileSize, maxEmojiSize) - log.Warn(err) - if deleteErr := p.storage.Delete(ctx, p.emoji.ImagePath); deleteErr != nil { - log.Errorf("store: error removing too-large emoji from the store: %s", deleteErr) + // Decode the image from storage. + staticImg, err := decodeImage(rc) + if err != nil { + return fmt.Errorf("error decoding image: %w", err) + } + + // The image should be in-memory by now. + if err := rc.Close(); err != nil { + return fmt.Errorf("error closing file: %w", err) + } + + // This shouldn't already exist, but we do a check as it's worth logging. + if have, _ := p.manager.storage.Has(ctx, p.emoji.ImageStaticPath); have { + log.Warnf("static emoji already exists at storage path: %s", p.emoji.ImagePath) + + // Attempt to remove static existing emoji at storage path (might be broken / out-of-date) + if err := p.manager.storage.Delete(ctx, p.emoji.ImageStaticPath); err != nil { + return fmt.Errorf("error removing static emoji from storage: %v", err) } - return err } - p.emoji.ImageFileSize = int(fileSize) - p.read = true + // Create an emoji PNG encoder stream. + enc := staticImg.ToPNG() + + // Stream-encode the PNG static image into storage. + sz, err := p.manager.storage.PutStream(ctx, p.emoji.ImageStaticPath, enc) + if err != nil { + return fmt.Errorf("error stream-encoding static emoji to storage: %w", err) + } + + // Set written image size. + p.emoji.ImageStaticFileSize = int(sz) return nil } @@ -406,15 +415,13 @@ func (m *manager) preProcessEmoji(ctx context.Context, data DataFunc, postData P } processingEmoji := &ProcessingEmoji{ - instanceAccountID: instanceAccount.ID, - emoji: emoji, - data: data, - postData: postData, - staticState: int32(received), - database: m.db, - storage: m.storage, - refresh: refresh, - newPathID: newPathID, + instAccID: instanceAccount.ID, + emoji: emoji, + refresh: refresh, + newPathID: newPathID, + dataFn: data, + postFn: postData, + manager: m, } return processingEmoji, nil diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go @@ -21,387 +21,329 @@ package media import ( "bytes" "context" - "errors" "fmt" + "image/jpeg" "io" - "strings" "sync" - "sync/atomic" "time" + "github.com/disintegration/imaging" + "github.com/h2non/filetype" terminator "github.com/superseriousbusiness/exif-terminator" - "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/storage" "github.com/superseriousbusiness/gotosocial/internal/uris" ) // ProcessingMedia represents a piece of media that is currently being processed. It exposes // various functions for retrieving data from the process. type ProcessingMedia struct { - mu sync.Mutex - - /* - below fields should be set on newly created media; - attachment will be updated incrementally as media goes through processing - */ - - attachment *gtsmodel.MediaAttachment - data DataFunc - postData PostDataCallbackFunc - read bool // bool indicating that data function has been triggered already - - thumbState int32 // the processing state of the media thumbnail - fullSizeState int32 // the processing state of the full-sized media - - /* - below pointers to database and storage are maintained so that - the media can store and update itself during processing steps - */ - - database db.DB - storage *storage.Driver - - err error // error created during processing, if any - - // track whether this media has already been put in the databse - insertedInDB bool - - // true if this is a recache, false if it's brand new media - recache bool + media *gtsmodel.MediaAttachment // processing media attachment details + recache bool // recaching existing (uncached) media + dataFn DataFunc // load-data function, returns media stream + postFn PostDataCallbackFunc // post data callback function + err error // error encountered during processing + manager *manager // manager instance (access to db / storage) + once sync.Once // once ensures processing only occurs once } // AttachmentID returns the ID of the underlying media attachment without blocking processing. func (p *ProcessingMedia) AttachmentID() string { - return p.attachment.ID + return p.media.ID // immutable, safe outside mutex. } // LoadAttachment blocks until the thumbnail and fullsize content // has been processed, and then returns the completed attachment. func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) { - p.mu.Lock() - defer p.mu.Unlock() + // only process once. + p.once.Do(func() { + var err error - if err := p.store(ctx); err != nil { - return nil, err - } + defer func() { + if r := recover(); r != nil { + if err != nil { + rOld := r // wrap the panic so we don't lose existing returned error + r = fmt.Errorf("panic occured after error %q: %v", err.Error(), rOld) + } - if err := p.loadFullSize(ctx); err != nil { - return nil, err - } + // Catch any panics and wrap as error. + err = fmt.Errorf("caught panic: %v", r) + } - if err := p.loadThumb(ctx); err != nil { - return nil, err - } + if err != nil { + // Store error. + p.err = err + } + }() + + // Attempt to store media and calculate + // full-size media attachment details. + if err = p.store(ctx); err != nil { + return + } + + // Finish processing by reloading media into + // memory to get dimension and generate a thumb. + if err = p.finish(ctx); err != nil { + return + } - if !p.insertedInDB { if p.recache { - // This is an existing media attachment we're recaching, so only need to update it - if err := p.database.UpdateByID(ctx, p.attachment, p.attachment.ID); err != nil { - return nil, err - } - } else { - // This is a new media attachment we're caching for first time - if err := p.database.Put(ctx, p.attachment); err != nil { - return nil, err - } + // Existing attachment we're recaching, so only need to update. + err = p.manager.db.UpdateByID(ctx, p.media, p.media.ID) + return } - // Mark this as stored in DB - p.insertedInDB = true + // New attachment, first time caching. + err = p.manager.db.Put(ctx, p.media) + return //nolint shutup linter i like this here + }) + + if p.err != nil { + return nil, p.err } - log.Tracef("finished loading attachment %s", p.attachment.URL) - return p.attachment, nil + return p.media, nil } -// Finished returns true if processing has finished for both the thumbnail -// and full fized version of this piece of media. -func (p *ProcessingMedia) Finished() bool { - return atomic.LoadInt32(&p.thumbState) == int32(complete) && atomic.LoadInt32(&p.fullSizeState) == int32(complete) -} +// store calls the data function attached to p if it hasn't been called yet, +// and updates the underlying attachment fields as necessary. It will then stream +// bytes from p's reader directly into storage so that it can be retrieved later. +func (p *ProcessingMedia) store(ctx context.Context) error { + defer func() { + if p.postFn == nil { + return + } -func (p *ProcessingMedia) loadThumb(ctx context.Context) error { - thumbState := atomic.LoadInt32(&p.thumbState) - switch processState(thumbState) { - case received: - // we haven't processed a thumbnail for this media yet so do it now - // check if we need to create a blurhash or if there's already one set - var createBlurhash bool - if p.attachment.Blurhash == "" { - // no blurhash created yet - createBlurhash = true + // ensure post callback gets called. + if err := p.postFn(ctx); err != nil { + log.Errorf("error executing postdata function: %v", err) } + }() - var ( - thumb *mediaMeta - err error - ) - switch ct := p.attachment.File.ContentType; ct { - case mimeImageJpeg, mimeImagePng, mimeImageWebp, mimeImageGif: - // thumbnail the image from the original stored full size version - stored, err := p.storage.GetStream(ctx, p.attachment.File.Path) - if err != nil { - p.err = fmt.Errorf("loadThumb: error fetching file from storage: %s", err) - atomic.StoreInt32(&p.thumbState, int32(errored)) - return p.err - } + // Load media from provided data fun + rc, sz, err := p.dataFn(ctx) + if err != nil { + return fmt.Errorf("error executing data function: %w", err) + } + + defer func() { + // Ensure data reader gets closed on return. + if err := rc.Close(); err != nil { + log.Errorf("error closing data reader: %v", err) + } + }() - thumb, err = deriveThumbnailFromImage(stored, ct, createBlurhash) + // Byte buffer to read file header into. + // See: https://en.wikipedia.org/wiki/File_format#File_header + // and https://github.com/h2non/filetype + hdrBuf := make([]byte, 261) - // try to close the stored stream we had open, no matter what - if closeErr := stored.Close(); closeErr != nil { - log.Errorf("error closing stream: %s", closeErr) - } + // Read the first 261 header bytes into buffer. + if _, err := io.ReadFull(rc, hdrBuf); err != nil { + return fmt.Errorf("error reading incoming media: %w", err) + } - // now check if we managed to get a thumbnail - if err != nil { - p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err) - atomic.StoreInt32(&p.thumbState, int32(errored)) - return p.err - } - case mimeVideoMp4: - // create a generic thumbnail based on video height + width - thumb, err = deriveThumbnailFromVideo(p.attachment.FileMeta.Original.Height, p.attachment.FileMeta.Original.Width) + // Parse file type info from header buffer. + info, err := filetype.Match(hdrBuf) + if err != nil { + return fmt.Errorf("error parsing file type: %w", err) + } + + // Recombine header bytes with remaining stream + r := io.MultiReader(bytes.NewReader(hdrBuf), rc) + + switch info.Extension { + case "mp4": + p.media.Type = gtsmodel.FileTypeVideo + + case "gif": + p.media.Type = gtsmodel.FileTypeImage + + case "jpg", "jpeg", "png", "webp": + p.media.Type = gtsmodel.FileTypeImage + if sz > 0 { + // A file size was provided so we can clean exif data from image. + r, err = terminator.Terminate(r, int(sz), info.Extension) if err != nil { - p.err = fmt.Errorf("loadThumb: error deriving thumbnail: %s", err) - atomic.StoreInt32(&p.thumbState, int32(errored)) - return p.err + return fmt.Errorf("error cleaning exif data: %w", err) } - default: - p.err = fmt.Errorf("loadThumb: content type %s not a processible image type", ct) - atomic.StoreInt32(&p.thumbState, int32(errored)) - return p.err } - // put the thumbnail in storage - if err := p.storage.Put(ctx, p.attachment.Thumbnail.Path, thumb.small); err != nil && err != storage.ErrAlreadyExists { - p.err = fmt.Errorf("loadThumb: error storing thumbnail: %s", err) - atomic.StoreInt32(&p.thumbState, int32(errored)) - return p.err - } + default: + return fmt.Errorf("unsupported file type: %s", info.Extension) + } - // set appropriate fields on the attachment based on the thumbnail we derived - if createBlurhash { - p.attachment.Blurhash = thumb.blurhash + // Calculate attachment file path. + p.media.File.Path = fmt.Sprintf( + "%s/%s/%s/%s.%s", + p.media.AccountID, + TypeAttachment, + SizeOriginal, + p.media.ID, + info.Extension, + ) + + // This shouldn't already exist, but we do a check as it's worth logging. + if have, _ := p.manager.storage.Has(ctx, p.media.File.Path); have { + log.Warnf("media already exists at storage path: %s", p.media.File.Path) + + // Attempt to remove existing media at storage path (might be broken / out-of-date) + if err := p.manager.storage.Delete(ctx, p.media.File.Path); err != nil { + return fmt.Errorf("error removing media from storage: %v", err) } - p.attachment.FileMeta.Small = gtsmodel.Small{ - Width: thumb.width, - Height: thumb.height, - Size: thumb.size, - Aspect: thumb.aspect, - } - p.attachment.Thumbnail.FileSize = len(thumb.small) - - // we're done processing the thumbnail! - atomic.StoreInt32(&p.thumbState, int32(complete)) - log.Tracef("finished processing thumbnail for attachment %s", p.attachment.URL) - fallthrough - case complete: - return nil - case errored: - return p.err } - return fmt.Errorf("loadThumb: thumbnail processing status %d unknown", p.thumbState) -} + // Write the final image reader stream to our storage. + sz, err = p.manager.storage.PutStream(ctx, p.media.File.Path, r) + if err != nil { + return fmt.Errorf("error writing media to storage: %w", err) + } -func (p *ProcessingMedia) loadFullSize(ctx context.Context) error { - fullSizeState := atomic.LoadInt32(&p.fullSizeState) - switch processState(fullSizeState) { - case received: - var err error - var decoded *mediaMeta + // Set written image size. + p.media.File.FileSize = int(sz) + + // Fill in remaining attachment data now it's stored. + p.media.URL = uris.GenerateURIForAttachment( + p.media.AccountID, + string(TypeAttachment), + string(SizeOriginal), + p.media.ID, + info.Extension, + ) + p.media.File.ContentType = info.MIME.Value + cached := true + p.media.Cached = &cached - // stream the original file out of storage... - stored, err := p.storage.GetStream(ctx, p.attachment.File.Path) - if err != nil { - p.err = fmt.Errorf("loadFullSize: error fetching file from storage: %s", err) - atomic.StoreInt32(&p.fullSizeState, int32(errored)) - return p.err - } + return nil +} - defer func() { - if err := stored.Close(); err != nil { - log.Errorf("loadFullSize: error closing stored full size: %s", err) - } - }() +func (p *ProcessingMedia) finish(ctx context.Context) error { + // Fetch a stream to the original file in storage. + rc, err := p.manager.storage.GetStream(ctx, p.media.File.Path) + if err != nil { + return fmt.Errorf("error loading file from storage: %w", err) + } + defer rc.Close() - // decode the image - ct := p.attachment.File.ContentType - switch ct { - case mimeImageJpeg, mimeImagePng, mimeImageWebp: - decoded, err = decodeImage(stored, ct) - case mimeImageGif: - decoded, err = decodeGif(stored) - case mimeVideoMp4: - decoded, err = decodeVideo(stored, ct) - default: - err = fmt.Errorf("loadFullSize: content type %s not a processible image type", ct) - } + var fullImg *gtsImage + switch p.media.File.ContentType { + // .jpeg, .gif, .webp image type + case mimeImageJpeg, mimeImageGif, mimeImageWebp: + fullImg, err = decodeImage(rc, imaging.AutoOrientation(true)) if err != nil { - p.err = err - atomic.StoreInt32(&p.fullSizeState, int32(errored)) - return p.err + return fmt.Errorf("error decoding image: %w", err) } - // set appropriate fields on the attachment based on the image we derived - - // generic fields - p.attachment.File.UpdatedAt = time.Now() - p.attachment.FileMeta.Original = gtsmodel.Original{ - Width: decoded.width, - Height: decoded.height, - Size: decoded.size, - Aspect: decoded.aspect, + // .png image (requires ancillary chunk stripping) + case mimeImagePng: + fullImg, err = decodeImage(&PNGAncillaryChunkStripper{ + Reader: rc, + }, imaging.AutoOrientation(true)) + if err != nil { + return fmt.Errorf("error decoding image: %w", err) } - // nullable fields - if decoded.duration != 0 { - i := decoded.duration - p.attachment.FileMeta.Original.Duration = &i - } - if decoded.framerate != 0 { - i := decoded.framerate - p.attachment.FileMeta.Original.Framerate = &i - } - if decoded.bitrate != 0 { - i := decoded.bitrate - p.attachment.FileMeta.Original.Bitrate = &i + // .mp4 video type + case mimeVideoMp4: + video, err := decodeVideoFrame(rc) + if err != nil { + return fmt.Errorf("error decoding video: %w", err) } - // we're done processing the full-size image - p.attachment.Processing = gtsmodel.ProcessingStatusProcessed - atomic.StoreInt32(&p.fullSizeState, int32(complete)) - log.Tracef("finished processing full size image for attachment %s", p.attachment.URL) - fallthrough - case complete: - return nil - case errored: - return p.err - } + // Set video frame as image. + fullImg = video.frame - return fmt.Errorf("loadFullSize: full size processing status %d unknown", p.fullSizeState) -} + // Set video metadata in attachment info. + p.media.FileMeta.Original.Duration = &video.duration + p.media.FileMeta.Original.Framerate = &video.framerate + p.media.FileMeta.Original.Bitrate = &video.bitrate + } -// store calls the data function attached to p if it hasn't been called yet, -// and updates the underlying attachment fields as necessary. It will then stream -// bytes from p's reader directly into storage so that it can be retrieved later. -func (p *ProcessingMedia) store(ctx context.Context) error { - // check if we've already done this and bail early if we have - if p.read { - return nil + // The image should be in-memory by now. + if err := rc.Close(); err != nil { + return fmt.Errorf("error closing file: %w", err) } - // execute the data function to get the readcloser out of it - rc, fileSize, err := p.data(ctx) + // Set full-size dimensions in attachment info. + p.media.FileMeta.Original.Width = int(fullImg.Width()) + p.media.FileMeta.Original.Height = int(fullImg.Height()) + p.media.FileMeta.Original.Size = int(fullImg.Size()) + p.media.FileMeta.Original.Aspect = fullImg.AspectRatio() + + // Calculate attachment thumbnail file path + p.media.Thumbnail.Path = fmt.Sprintf( + "%s/%s/%s/%s.jpg", + p.media.AccountID, + TypeAttachment, + SizeSmall, + p.media.ID, + ) + + // Get smaller thumbnail image + thumbImg := fullImg.Thumbnail() + + // Garbage collector, you may + // now take our large son. + fullImg = nil + + // Blurhash needs generating from thumb. + hash, err := thumbImg.Blurhash() if err != nil { - return fmt.Errorf("store: error executing data function: %s", err) + return fmt.Errorf("error generating blurhash: %w", err) } - // defer closing the reader when we're done with it - defer func() { - if err := rc.Close(); err != nil { - log.Errorf("store: error closing readcloser: %s", err) - } - }() + // Set the attachment blurhash. + p.media.Blurhash = hash - // execute the postData function no matter what happens - defer func() { - if p.postData != nil { - if err := p.postData(ctx); err != nil { - log.Errorf("store: error executing postData: %s", err) - } - } - }() + // This shouldn't already exist, but we do a check as it's worth logging. + if have, _ := p.manager.storage.Has(ctx, p.media.Thumbnail.Path); have { + log.Warnf("thumbnail already exists at storage path: %s", p.media.Thumbnail.Path) - // extract no more than 261 bytes from the beginning of the file -- this is the header - firstBytes := make([]byte, maxFileHeaderBytes) - if _, err := rc.Read(firstBytes); err != nil { - return fmt.Errorf("store: error reading initial %d bytes: %s", maxFileHeaderBytes, err) + // Attempt to remove existing thumbnail at storage path (might be broken / out-of-date) + if err := p.manager.storage.Delete(ctx, p.media.Thumbnail.Path); err != nil { + return fmt.Errorf("error removing thumbnail from storage: %v", err) + } } - // now we have the file header we can work out the content type from it - contentType, err := parseContentType(firstBytes) - if err != nil { - return fmt.Errorf("store: error parsing content type: %s", err) - } + // Create a thumbnail JPEG encoder stream. + enc := thumbImg.ToJPEG(&jpeg.Options{ + Quality: 70, // enough for a thumbnail. + }) - // bail if this is a type we can't process - if !supportedAttachment(contentType) { - return fmt.Errorf("store: media type %s not (yet) supported", contentType) + // Stream-encode the JPEG thumbnail image into storage. + sz, err := p.manager.storage.PutStream(ctx, p.media.Thumbnail.Path, enc) + if err != nil { + return fmt.Errorf("error stream-encoding thumbnail to storage: %w", err) } - // extract the file extension - split := strings.Split(contentType, "/") - if len(split) != 2 { - return fmt.Errorf("store: content type %s was not valid", contentType) + // Fill in remaining thumbnail now it's stored + p.media.Thumbnail.ContentType = mimeImageJpeg + p.media.Thumbnail.URL = uris.GenerateURIForAttachment( + p.media.AccountID, + string(TypeAttachment), + string(SizeSmall), + p.media.ID, + "jpg", // always jpeg + ) + + // Set thumbnail dimensions in attachment info. + p.media.FileMeta.Small = gtsmodel.Small{ + Width: int(thumbImg.Width()), + Height: int(thumbImg.Height()), + Size: int(thumbImg.Size()), + Aspect: thumbImg.AspectRatio(), } - extension := split[1] // something like 'jpeg' - - // concatenate the cleaned up first bytes with the existing bytes still in the reader (thanks Mara) - multiReader := io.MultiReader(bytes.NewBuffer(firstBytes), rc) - - // use the extension to derive the attachment type - // and, while we're in here, clean up exif data from - // the image if we already know the fileSize - var readerToStore io.Reader - switch extension { - case mimeGif: - p.attachment.Type = gtsmodel.FileTypeImage - // nothing to terminate, we can just store the multireader - readerToStore = multiReader - case mimeJpeg, mimePng, mimeWebp: - p.attachment.Type = gtsmodel.FileTypeImage - if fileSize > 0 { - terminated, err := terminator.Terminate(multiReader, int(fileSize), extension) - if err != nil { - return fmt.Errorf("store: exif error: %s", err) - } - defer func() { - if closer, ok := terminated.(io.Closer); ok { - if err := closer.Close(); err != nil { - log.Errorf("store: error closing terminator reader: %s", err) - } - } - }() - // store the exif-terminated version of what was in the multireader - readerToStore = terminated - } else { - // can't terminate if we don't know the file size, so just store the multiReader - readerToStore = multiReader - } - case mimeMp4: - p.attachment.Type = gtsmodel.FileTypeVideo - // nothing to terminate, we can just store the multireader - readerToStore = multiReader - default: - return fmt.Errorf("store: couldn't process %s", extension) - } - - // now set some additional fields on the attachment since - // we know more about what the underlying media actually is - p.attachment.URL = uris.GenerateURIForAttachment(p.attachment.AccountID, string(TypeAttachment), string(SizeOriginal), p.attachment.ID, extension) - p.attachment.File.ContentType = contentType - p.attachment.File.Path = fmt.Sprintf("%s/%s/%s/%s.%s", p.attachment.AccountID, TypeAttachment, SizeOriginal, p.attachment.ID, extension) - // store this for now -- other processes can pull it out of storage as they please - if fileSize, err = putStream(ctx, p.storage, p.attachment.File.Path, readerToStore, fileSize); err != nil { - if !errors.Is(err, storage.ErrAlreadyExists) { - return fmt.Errorf("store: error storing stream: %s", err) - } - log.Warnf("attachment %s already exists at storage path: %s", p.attachment.ID, p.attachment.File.Path) - } + // Set written image size. + p.media.Thumbnail.FileSize = int(sz) - cached := true - p.attachment.Cached = &cached - p.attachment.File.FileSize = int(fileSize) - p.read = true + // Finally set the attachment as processed and update time. + p.media.Processing = gtsmodel.ProcessingStatusProcessed + p.media.File.UpdatedAt = time.Now() - log.Tracef("finished storing initial data for attachment %s", p.attachment.URL) return nil } @@ -411,19 +353,6 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P return nil, err } - file := gtsmodel.File{ - Path: "", // we don't know yet because it depends on the uncalled DataFunc - ContentType: "", // we don't know yet because it depends on the uncalled DataFunc - UpdatedAt: time.Now(), - } - - thumbnail := gtsmodel.Thumbnail{ - URL: uris.GenerateURIForAttachment(accountID, string(TypeAttachment), string(SizeSmall), id, mimeJpeg), // all thumbnails are encoded as jpeg, - Path: fmt.Sprintf("%s/%s/%s/%s.%s", accountID, TypeAttachment, SizeSmall, id, mimeJpeg), // all thumbnails are encoded as jpeg, - ContentType: mimeImageJpeg, - UpdatedAt: time.Now(), - } - avatar := false header := false cached := false @@ -443,8 +372,8 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P ScheduledStatusID: "", Blurhash: "", Processing: gtsmodel.ProcessingStatusReceived, - File: file, - Thumbnail: thumbnail, + File: gtsmodel.File{UpdatedAt: time.Now()}, + Thumbnail: gtsmodel.Thumbnail{UpdatedAt: time.Now()}, Avatar: &avatar, Header: &header, Cached: &cached, @@ -495,34 +424,28 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P } processingMedia := &ProcessingMedia{ - attachment: attachment, - data: data, - postData: postData, - thumbState: int32(received), - fullSizeState: int32(received), - database: m.db, - storage: m.storage, + media: attachment, + dataFn: data, + postFn: postData, + manager: m, } return processingMedia, nil } -func (m *manager) preProcessRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) { - // get the existing attachment - attachment, err := m.db.GetAttachmentByID(ctx, attachmentID) +func (m *manager) preProcessRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, id string) (*ProcessingMedia, error) { + // get the existing attachment from database. + attachment, err := m.db.GetAttachmentByID(ctx, id) if err != nil { return nil, err } processingMedia := &ProcessingMedia{ - attachment: attachment, - data: data, - postData: postData, - thumbState: int32(received), - fullSizeState: int32(received), - database: m.db, - storage: m.storage, - recache: true, // indicate it's a recache + media: attachment, + dataFn: data, + postFn: postData, + manager: m, + recache: true, // indicate it's a recache } return processingMedia, nil diff --git a/internal/media/pruneorphaned_test.go b/internal/media/pruneorphaned_test.go @@ -39,7 +39,7 @@ func (suite *PruneOrphanedTestSuite) TestPruneOrphanedDry() { } pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachments/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" - if err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil { + if _, err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil { panic(err) } @@ -62,7 +62,7 @@ func (suite *PruneOrphanedTestSuite) TestPruneOrphanedMoist() { } pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachments/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" - if err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil { + if _, err := suite.storage.PutStream(context.Background(), pandaPath, bytes.NewBuffer(b)); err != nil { panic(err) } diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go @@ -87,7 +87,7 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { // now recache the image.... data := func(_ context.Context) (io.ReadCloser, int64, error) { // load bytes from a test image - b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpeg") + b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg") if err != nil { panic(err) } diff --git a/internal/media/test/longer-mp4-thumbnail.jpg b/internal/media/test/longer-mp4-thumbnail.jpg Binary files differ. diff --git a/internal/media/test/test-jpeg-thumbnail.jpg b/internal/media/test/test-jpeg-thumbnail.jpg Binary files differ. diff --git a/internal/media/test/test-mp4-thumbnail.jpg b/internal/media/test/test-mp4-thumbnail.jpg Binary files differ. diff --git a/internal/media/test/test-png-alphachannel-thumbnail.jpg b/internal/media/test/test-png-alphachannel-thumbnail.jpg Binary files differ. diff --git a/internal/media/test/test-png-noalphachannel-thumbnail.jpg b/internal/media/test/test-png-noalphachannel-thumbnail.jpg Binary files differ. diff --git a/internal/media/types.go b/internal/media/types.go @@ -24,13 +24,6 @@ import ( "time" ) -// maxFileHeaderBytes represents the maximum amount of bytes we want -// to examine from the beginning of a file to determine its type. -// -// See: https://en.wikipedia.org/wiki/File_format#File_header -// and https://github.com/h2non/filetype -const maxFileHeaderBytes = 261 - // mime consts const ( mimeImage = "image" @@ -52,14 +45,6 @@ const ( mimeVideoMp4 = mimeVideo + "/" + mimeMp4 ) -type processState int32 - -const ( - received processState = iota // processing order has been received but not done yet - complete // processing order has been completed successfully - errored // processing order has been completed with an error -) - // EmojiMaxBytes is the maximum permitted bytes of an emoji upload (50kb) // const EmojiMaxBytes = 51200 @@ -132,17 +117,3 @@ type DataFunc func(ctx context.Context) (reader io.ReadCloser, fileSize int64, e // // This can be set to nil, and will then not be executed. type PostDataCallbackFunc func(ctx context.Context) error - -type mediaMeta struct { - width int - height int - size int - aspect float32 - blurhash string - small []byte - - // video-specific properties - duration float32 - framerate float32 - bitrate uint64 -} diff --git a/internal/media/util.go b/internal/media/util.go @@ -19,72 +19,22 @@ package media import ( - "context" - "errors" "fmt" - "io" - "github.com/h2non/filetype" "github.com/superseriousbusiness/gotosocial/internal/log" - "github.com/superseriousbusiness/gotosocial/internal/storage" ) -// AllSupportedMIMETypes just returns all media -// MIME types supported by this instance. -func AllSupportedMIMETypes() []string { - return []string{ - mimeImageJpeg, - mimeImageGif, - mimeImagePng, - mimeImageWebp, - mimeVideoMp4, - } -} - -// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg"). -// Returns an error if the content type is not something we can process. -// -// Fileheader should be no longer than 262 bytes; anything more than this is inefficient. -func parseContentType(fileHeader []byte) (string, error) { - if fhLength := len(fileHeader); fhLength > maxFileHeaderBytes { - return "", fmt.Errorf("parseContentType requires %d bytes max, we got %d", maxFileHeaderBytes, fhLength) - } - - kind, err := filetype.Match(fileHeader) - if err != nil { - return "", err - } - - if kind == filetype.Unknown { - return "", errors.New("filetype unknown") - } - - return kind.MIME.Value, nil -} - -// supportedAttachment checks mime type of an attachment against a -// slice of accepted types, and returns True if the mime type is accepted. -func supportedAttachment(mimeType string) bool { - for _, accepted := range AllSupportedMIMETypes() { - if mimeType == accepted { - return true - } - } - return false +var SupportedMIMETypes = []string{ + mimeImageJpeg, + mimeImageGif, + mimeImagePng, + mimeImageWebp, + mimeVideoMp4, } -// supportedEmoji checks that the content type is image/png or image/gif -- the only types supported for emoji. -func supportedEmoji(mimeType string) bool { - acceptedEmojiTypes := []string{ - mimeImageGif, - mimeImagePng, - } - for _, accepted := range acceptedEmojiTypes { - if mimeType == accepted { - return true - } - } - return false +var SupportedEmojiMIMETypes = []string{ + mimeImageGif, + mimeImagePng, } // ParseMediaType converts s to a recognized MediaType, or returns an error if unrecognized @@ -127,31 +77,3 @@ func (l *logrusWrapper) Info(msg string, keysAndValues ...interface{}) { func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{}) { log.Error("media manager cron logger: ", err, msg, keysAndValues) } - -// lengthReader wraps a reader and reads the length of total bytes written as it goes. -type lengthReader struct { - source io.Reader - length int64 -} - -func (r *lengthReader) Read(b []byte) (int, error) { - n, err := r.source.Read(b) - r.length += int64(n) - return n, err -} - -// putStream either puts a file with a known fileSize into storage directly, and returns the -// fileSize unchanged, or it wraps the reader with a lengthReader and returns the discovered -// fileSize. -func putStream(ctx context.Context, storage *storage.Driver, key string, r io.Reader, fileSize int64) (int64, error) { - if fileSize > 0 { - return fileSize, storage.PutStream(ctx, key, r) - } - - lr := &lengthReader{ - source: r, - } - - err := storage.PutStream(ctx, key, lr) - return lr.length, err -} diff --git a/internal/media/video.go b/internal/media/video.go @@ -19,63 +19,55 @@ package media import ( - "bytes" "fmt" - "image" - "image/color" - "image/draw" - "image/jpeg" "io" "os" "github.com/abema/go-mp4" - "github.com/superseriousbusiness/gotosocial/internal/gtserror" - "github.com/superseriousbusiness/gotosocial/internal/log" ) -var thumbFill = color.RGBA{42, 43, 47, 0} // the color to fill video thumbnails with +type gtsVideo struct { + frame *gtsImage + duration float32 // in seconds + bitrate uint64 + framerate float32 +} -func decodeVideo(r io.Reader, contentType string) (*mediaMeta, error) { +// decodeVideoFrame decodes and returns an image from a single frame in the given video stream. +// (note: currently this only returns a blank image resized to fit video dimensions). +func decodeVideoFrame(r io.Reader) (*gtsVideo, error) { // We'll need a readseeker to decode the video. We can get a readseeker // without burning too much mem by first copying the reader into a temp file. // First create the file in the temporary directory... - tempFile, err := os.CreateTemp(os.TempDir(), "gotosocial-") + tmp, err := os.CreateTemp(os.TempDir(), "gotosocial-") if err != nil { - return nil, fmt.Errorf("could not create temporary file while decoding video: %w", err) + return nil, err } - tempFileName := tempFile.Name() - // Make sure to clean up the temporary file when we're done with it defer func() { - if err := tempFile.Close(); err != nil { - log.Errorf("could not close file %s: %s", tempFileName, err) - } - if err := os.Remove(tempFileName); err != nil { - log.Errorf("could not remove file %s: %s", tempFileName, err) - } + tmp.Close() + os.Remove(tmp.Name()) }() // Now copy the entire reader we've been provided into the // temporary file; we won't use the reader again after this. - if _, err := io.Copy(tempFile, r); err != nil { - return nil, fmt.Errorf("could not copy video reader into temporary file %s: %w", tempFileName, err) + if _, err := io.Copy(tmp, r); err != nil { + return nil, err } - var ( - width int - height int - duration float32 - framerate float32 - bitrate uint64 - ) - // probe the video file to extract useful metadata from it; for methodology, see: // https://github.com/abema/go-mp4/blob/7d8e5a7c5e644e0394261b0cf72fef79ce246d31/mp4tool/probe/probe.go#L85-L154 - info, err := mp4.Probe(tempFile) + info, err := mp4.Probe(tmp) if err != nil { - return nil, fmt.Errorf("could not probe temporary video file %s: %w", tempFileName, err) + return nil, fmt.Errorf("error probing tmp file %s: %w", tmp.Name(), err) } + var ( + width int + height int + video gtsVideo + ) + for _, tr := range info.Tracks { if tr.AVC == nil { continue @@ -89,72 +81,42 @@ func decodeVideo(r io.Reader, contentType string) (*mediaMeta, error) { height = h } - if br := tr.Samples.GetBitrate(tr.Timescale); br > bitrate { - bitrate = br - } else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > bitrate { - bitrate = br + if br := tr.Samples.GetBitrate(tr.Timescale); br > video.bitrate { + video.bitrate = br + } else if br := info.Segments.GetBitrate(tr.TrackID, tr.Timescale); br > video.bitrate { + video.bitrate = br } - if d := float32(tr.Duration) / float32(tr.Timescale); d > duration { - duration = d - framerate = float32(len(tr.Samples)) / duration + if d := float64(tr.Duration) / float64(tr.Timescale); d > float64(video.duration) { + video.framerate = float32(len(tr.Samples)) / float32(d) + video.duration = float32(d) } } - var errs gtserror.MultiError + // Check for empty video metadata. + var empty []string if width == 0 { - errs = append(errs, "video width could not be discovered") + empty = append(empty, "width") } - if height == 0 { - errs = append(errs, "video height could not be discovered") + empty = append(empty, "height") } - - if duration == 0 { - errs = append(errs, "video duration could not be discovered") + if video.duration == 0 { + empty = append(empty, "duration") } - - if framerate == 0 { - errs = append(errs, "video framerate could not be discovered") + if video.framerate == 0 { + empty = append(empty, "framerate") } - - if bitrate == 0 { - errs = append(errs, "video bitrate could not be discovered") + if video.bitrate == 0 { + empty = append(empty, "bitrate") } - - if errs != nil { - return nil, errs.Combine() + if len(empty) > 0 { + return nil, fmt.Errorf("error determining video metadata: %v", empty) } - return &mediaMeta{ - width: width, - height: height, - duration: duration, - framerate: framerate, - bitrate: bitrate, - size: height * width, - aspect: float32(width) / float32(height), - }, nil -} - -func deriveThumbnailFromVideo(height int, width int) (*mediaMeta, error) { - // create a rectangle with the same dimensions as the video - img := image.NewRGBA(image.Rect(0, 0, width, height)) - - // fill the rectangle with our desired fill color - draw.Draw(img, img.Bounds(), &image.Uniform{thumbFill}, image.Point{}, draw.Src) - - // we can get away with using extremely poor quality for this monocolor thumbnail - out := &bytes.Buffer{} - if err := jpeg.Encode(out, img, &jpeg.Options{Quality: 1}); err != nil { - return nil, fmt.Errorf("error encoding video thumbnail: %w", err) - } + // Create new empty "frame" image. + // TODO: decode frame from video file. + video.frame = blankImage(width, height) - return &mediaMeta{ - width: width, - height: height, - size: width * height, - aspect: float32(width) / float32(height), - small: out.Bytes(), - }, nil + return &video, nil } diff --git a/internal/processing/account/getrss_test.go b/internal/processing/account/getrss_test.go @@ -40,7 +40,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSAdmin() { fmt.Println(feed) - suite.Equal("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n <channel>\n <title>Posts from @admin@localhost:8080</title>\n <link>http://localhost:8080/@admin</link>\n <description>Posts from @admin@localhost:8080</description>\n <pubDate>Wed, 20 Oct 2021 12:36:45 +0000</pubDate>\n <lastBuildDate>Wed, 20 Oct 2021 12:36:45 +0000</lastBuildDate>\n <item>\n <title>open to see some puppies</title>\n <link>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</link>\n <description>@admin@localhost:8080 made a new post: "🐕🐕🐕🐕🐕"</description>\n <content:encoded><![CDATA[🐕🐕🐕🐕🐕]]></content:encoded>\n <author>@admin@localhost:8080</author>\n <guid>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</guid>\n <pubDate>Wed, 20 Oct 2021 12:36:45 +0000</pubDate>\n <source>http://localhost:8080/@admin/feed.rss</source>\n </item>\n <item>\n <title>hello world! #welcome ! first post on the instance :rainbow: !</title>\n <link>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</link>\n <description>@admin@localhost:8080 posted 1 attachment: "hello world! #welcome ! first post on the instance :rainbow: !"</description>\n <content:encoded><![CDATA[hello world! #welcome ! first post on the instance <img src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" class=\"emoji\"/> !]]></content:encoded>\n <author>@admin@localhost:8080</author>\n <enclosure url=\"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg\" length=\"62529\" type=\"image/jpeg\"></enclosure>\n <guid>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</guid>\n <pubDate>Wed, 20 Oct 2021 11:36:45 +0000</pubDate>\n <source>http://localhost:8080/@admin/feed.rss</source>\n </item>\n </channel>\n</rss>", feed) + suite.Equal("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n <channel>\n <title>Posts from @admin@localhost:8080</title>\n <link>http://localhost:8080/@admin</link>\n <description>Posts from @admin@localhost:8080</description>\n <pubDate>Wed, 20 Oct 2021 12:36:45 +0000</pubDate>\n <lastBuildDate>Wed, 20 Oct 2021 12:36:45 +0000</lastBuildDate>\n <item>\n <title>open to see some puppies</title>\n <link>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</link>\n <description>@admin@localhost:8080 made a new post: "🐕🐕🐕🐕🐕"</description>\n <content:encoded><![CDATA[🐕🐕🐕🐕🐕]]></content:encoded>\n <author>@admin@localhost:8080</author>\n <guid>http://localhost:8080/@admin/statuses/01F8MHAAY43M6RJ473VQFCVH37</guid>\n <pubDate>Wed, 20 Oct 2021 12:36:45 +0000</pubDate>\n <source>http://localhost:8080/@admin/feed.rss</source>\n </item>\n <item>\n <title>hello world! #welcome ! first post on the instance :rainbow: !</title>\n <link>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</link>\n <description>@admin@localhost:8080 posted 1 attachment: "hello world! #welcome ! first post on the instance :rainbow: !"</description>\n <content:encoded><![CDATA[hello world! #welcome ! first post on the instance <img src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" class=\"emoji\"/> !]]></content:encoded>\n <author>@admin@localhost:8080</author>\n <enclosure url=\"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg\" length=\"62529\" type=\"image/jpeg\"></enclosure>\n <guid>http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R</guid>\n <pubDate>Wed, 20 Oct 2021 11:36:45 +0000</pubDate>\n <source>http://localhost:8080/@admin/feed.rss</source>\n </item>\n </channel>\n</rss>", feed) } func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { @@ -53,7 +53,7 @@ func (suite *GetRSSTestSuite) TestGetAccountRSSZork() { fmt.Println(feed) - suite.Equal("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n <channel>\n <title>Posts from @the_mighty_zork@localhost:8080</title>\n <link>http://localhost:8080/@the_mighty_zork</link>\n <description>Posts from @the_mighty_zork@localhost:8080</description>\n <pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>\n <lastBuildDate>Wed, 20 Oct 2021 10:40:37 +0000</lastBuildDate>\n <image>\n <url>http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg</url>\n <title>Avatar for @the_mighty_zork@localhost:8080</title>\n <link>http://localhost:8080/@the_mighty_zork</link>\n </image>\n <item>\n <title>introduction post</title>\n <link>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</link>\n <description>@the_mighty_zork@localhost:8080 made a new post: "hello everyone!"</description>\n <content:encoded><![CDATA[hello everyone!]]></content:encoded>\n <author>@the_mighty_zork@localhost:8080</author>\n <guid>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</guid>\n <pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>\n <source>http://localhost:8080/@the_mighty_zork/feed.rss</source>\n </item>\n </channel>\n</rss>", feed) + suite.Equal("<?xml version=\"1.0\" encoding=\"UTF-8\"?><rss version=\"2.0\" xmlns:content=\"http://purl.org/rss/1.0/modules/content/\">\n <channel>\n <title>Posts from @the_mighty_zork@localhost:8080</title>\n <link>http://localhost:8080/@the_mighty_zork</link>\n <description>Posts from @the_mighty_zork@localhost:8080</description>\n <pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>\n <lastBuildDate>Wed, 20 Oct 2021 10:40:37 +0000</lastBuildDate>\n <image>\n <url>http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg</url>\n <title>Avatar for @the_mighty_zork@localhost:8080</title>\n <link>http://localhost:8080/@the_mighty_zork</link>\n </image>\n <item>\n <title>introduction post</title>\n <link>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</link>\n <description>@the_mighty_zork@localhost:8080 made a new post: "hello everyone!"</description>\n <content:encoded><![CDATA[hello everyone!]]></content:encoded>\n <author>@the_mighty_zork@localhost:8080</author>\n <guid>http://localhost:8080/@the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY</guid>\n <pubDate>Wed, 20 Oct 2021 10:40:37 +0000</pubDate>\n <source>http://localhost:8080/@the_mighty_zork/feed.rss</source>\n </item>\n </channel>\n</rss>", feed) } func TestGetRSSTestSuite(t *testing.T) { diff --git a/internal/processing/admin/updateemoji.go b/internal/processing/admin/updateemoji.go @@ -90,9 +90,8 @@ func (p *processor) emojiUpdateCopy(ctx context.Context, emoji *gtsmodel.Emoji, newEmojiURI := uris.GenerateURIForEmoji(newEmojiID) data := func(ctx context.Context) (reader io.ReadCloser, fileSize int64, err error) { - // 'copy' the emoji by pulling the existing one out of storage - i, err := p.storage.GetStream(ctx, emoji.ImagePath) - return i, int64(emoji.ImageFileSize), err + rc, err := p.storage.GetStream(ctx, emoji.ImagePath) + return rc, int64(emoji.ImageFileSize), err } var ai *media.AdditionalEmojiInfo diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go @@ -28,7 +28,6 @@ import ( apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" - "github.com/superseriousbusiness/gotosocial/internal/iotools" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/transport" "github.com/superseriousbusiness/gotosocial/internal/uris" @@ -99,135 +98,70 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, owningAccountID)) } - // get file information from the attachment depending on the requested media size - switch mediaSize { - case media.SizeOriginal: - attachmentContent.ContentType = a.File.ContentType - attachmentContent.ContentLength = int64(a.File.FileSize) - storagePath = a.File.Path - case media.SizeSmall: - attachmentContent.ContentType = a.Thumbnail.ContentType - attachmentContent.ContentLength = int64(a.Thumbnail.FileSize) - storagePath = a.Thumbnail.Path - default: - return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize)) - } - - // if we have the media cached on our server already, we can now simply return it from storage - if *a.Cached { - return p.retrieveFromStorage(ctx, storagePath, attachmentContent) - } - - // if we don't have it cached, then we can assume two things: - // 1. this is remote media, since local media should never be uncached - // 2. we need to fetch it again using a transport and the media manager - remoteMediaIRI, err := url.Parse(a.RemoteURL) - if err != nil { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %s", a.RemoteURL, err)) - } - - // use an empty string as requestingUsername to use the instance account, unless the request for this - // media has been http signed, then use the requesting account to make the request to remote server - var requestingUsername string - if requestingAccount != nil { - requestingUsername = requestingAccount.Username - } + if !*a.Cached { + // if we don't have it cached, then we can assume two things: + // 1. this is remote media, since local media should never be uncached + // 2. we need to fetch it again using a transport and the media manager + remoteMediaIRI, err := url.Parse(a.RemoteURL) + if err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %s", a.RemoteURL, err)) + } - var data media.DataFunc + // use an empty string as requestingUsername to use the instance account, unless the request for this + // media has been http signed, then use the requesting account to make the request to remote server + var requestingUsername string + if requestingAccount != nil { + requestingUsername = requestingAccount.Username + } - if mediaSize == media.SizeSmall { - // if it's the thumbnail that's requested then the user will have to wait a bit while we process the - // large version and derive a thumbnail from it, so use the normal recaching procedure: fetch the media, - // process it, then return the thumbnail data - data = func(innerCtx context.Context) (io.ReadCloser, int64, error) { + // Pour one out for tobi's original streamed recache + // (streaming data both to the client and storage). + // Gone and forever missed <3 + // + // [ + // the reason it was removed was because a slow + // client connection could hold open a storage + // recache operation, and so holding open a media + // worker worker. + // ] + + dataFn := func(innerCtx context.Context) (io.ReadCloser, int64, error) { t, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername) if err != nil { return nil, 0, err } return t.DereferenceMedia(transport.WithFastfail(innerCtx), remoteMediaIRI) } - } else { - // if it's the full-sized version being requested, we can cheat a bit by streaming data to the user as - // it's retrieved from the remote server, using tee; this saves the user from having to wait while - // we process the media on our side - // - // this looks a bit like this: - // - // http fetch pipe - // remote server ------------> data function ----------------> api caller - // | - // | tee - // | - // ▼ - // instance storage - - // This pipe will connect the caller to the in-process media retrieval... - pipeReader, pipeWriter := io.Pipe() - // Wrap the output pipe to silence any errors during the actual media - // streaming process. We catch the error later but they must be silenced - // during stream to prevent interruptions to storage of the actual media. - silencedWriter := iotools.SilenceWriter(pipeWriter) - - // Pass the reader side of the pipe to the caller to slurp from. - attachmentContent.Content = pipeReader - - // Create a data function which injects the writer end of the pipe - // into the data retrieval process. If something goes wrong while - // doing the data retrieval, we hang up the underlying pipeReader - // to indicate to the caller that no data is available. It's up to - // the caller of this processor function to handle that gracefully. - data = func(innerCtx context.Context) (io.ReadCloser, int64, error) { - t, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername) - if err != nil { - // propagate the transport error to read end of pipe. - _ = pipeWriter.CloseWithError(fmt.Errorf("error getting transport for user: %w", err)) - return nil, 0, err - } - - readCloser, fileSize, err := t.DereferenceMedia(transport.WithFastfail(innerCtx), remoteMediaIRI) - if err != nil { - // propagate the dereference error to read end of pipe. - _ = pipeWriter.CloseWithError(fmt.Errorf("error dereferencing media: %w", err)) - return nil, 0, err - } - - // Make a TeeReader so that everything read from the readCloser, - // aka the remote instance, will also be written into the pipe. - teeReader := io.TeeReader(readCloser, silencedWriter) - - // Wrap teereader to implement original readcloser's close, - // and also ensuring that we close the pipe from write end. - return iotools.ReadFnCloser(teeReader, func() error { - defer func() { - // We use the error (if any) encountered by the - // silenced writer to close connection to make sure it - // gets propagated to the attachment.Content reader. - _ = pipeWriter.CloseWithError(silencedWriter.Error()) - }() - - return readCloser.Close() - }), fileSize, nil + // Start recaching this media with the prepared data function. + processingMedia, err := p.mediaManager.RecacheMedia(ctx, dataFn, nil, wantedMediaID) + if err != nil { + return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err)) } - } - - // put the media recached in the queue - processingMedia, err := p.mediaManager.RecacheMedia(ctx, data, nil, wantedMediaID) - if err != nil { - return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err)) - } - // if it's the thumbnail, stream the processed thumbnail from storage, after waiting for processing to finish - if mediaSize == media.SizeSmall { - // below function call blocks until all processing on the attachment has finished... - if _, err := processingMedia.LoadAttachment(ctx); err != nil { + // Load attachment and block until complete + a, err = processingMedia.LoadAttachment(ctx) + if err != nil { return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err)) } - // ... so now we can safely return it - return p.retrieveFromStorage(ctx, storagePath, attachmentContent) } - return attachmentContent, nil + // get file information from the attachment depending on the requested media size + switch mediaSize { + case media.SizeOriginal: + attachmentContent.ContentType = a.File.ContentType + attachmentContent.ContentLength = int64(a.File.FileSize) + storagePath = a.File.Path + case media.SizeSmall: + attachmentContent.ContentType = a.Thumbnail.ContentType + attachmentContent.ContentLength = int64(a.Thumbnail.FileSize) + storagePath = a.Thumbnail.Path + default: + return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize)) + } + + // ... so now we can safely return it + return p.retrieveFromStorage(ctx, storagePath, attachmentContent) } func (p *processor) getEmojiContent(ctx context.Context, fileName string, owningAccountID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) { diff --git a/internal/storage/storage.go b/internal/storage/storage.go @@ -26,12 +26,14 @@ import ( "path" "time" + "codeberg.org/gruf/go-bytesize" "codeberg.org/gruf/go-cache/v3/ttl" "codeberg.org/gruf/go-store/v2/kv" "codeberg.org/gruf/go-store/v2/storage" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/log" ) const ( @@ -63,9 +65,14 @@ func (d *Driver) URL(ctx context.Context, key string) *url.URL { return nil } - // access the cache member directly to avoid extending the TTL - if u, ok := d.PresignedCache.Cache.Get(key); ok { - return u.Value + // Check cache underlying cache map directly to + // avoid extending the TTL (which cache.Get() does). + d.PresignedCache.Lock() + e, ok := d.PresignedCache.Cache.Get(key) + d.PresignedCache.Unlock() + + if ok { + return e.Value } u, err := s3.Client().PresignedGetObject(ctx, d.Bucket, key, urlCacheTTL, url.Values{ @@ -88,7 +95,6 @@ func AutoConfig() (*Driver, error) { default: return nil, fmt.Errorf("invalid storage backend: %s", backend) } - } func NewFileStorage() (*Driver, error) { @@ -102,12 +108,17 @@ func NewFileStorage() (*Driver, error) { // overwriting the lockfile if we store a file called 'store.lock'. // However, in this case it's OK because the keys are set by // GtS and not the user, so we know we're never going to overwrite it. - LockFile: path.Join(basePath, "store.lock"), + LockFile: path.Join(basePath, "store.lock"), + WriteBufSize: int(16 * bytesize.KiB), }) if err != nil { return nil, fmt.Errorf("error opening disk storage: %w", err) } + if err := disk.Clean(context.Background()); err != nil { + log.Errorf("error performing storage cleanup: %v", err) + } + return &Driver{ KVStore: kv.New(disk), Storage: disk, diff --git a/internal/typeutils/internaltoas_test.go b/internal/typeutils/internaltoas_test.go @@ -51,7 +51,7 @@ func (suite *InternalToASTestSuite) TestAccountToAS() { // this is necessary because the order of multiple 'context' entries is not determinate trimmed := strings.Split(string(bytes), "\"discoverable\"")[1] - suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":[],"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) + suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":[],"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) } func (suite *InternalToASTestSuite) TestAccountToASWithEmoji() { @@ -72,7 +72,7 @@ func (suite *InternalToASTestSuite) TestAccountToASWithEmoji() { // this is necessary because the order of multiple 'context' entries is not determinate trimmed := strings.Split(string(bytes), "\"discoverable\"")[1] - suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T12:40:37+02:00"},"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) + suite.Equal(`:true,"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T12:40:37+02:00"},"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) } func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() { @@ -94,7 +94,7 @@ func (suite *InternalToASTestSuite) TestAccountToASWithSharedInbox() { // this is necessary because the order of multiple 'context' entries is not determinate trimmed := strings.Split(string(bytes), "\"discoverable\"")[1] - suite.Equal(`:true,"endpoints":{"sharedInbox":"http://localhost:8080/sharedInbox"},"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":[],"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) + suite.Equal(`:true,"endpoints":{"sharedInbox":"http://localhost:8080/sharedInbox"},"featured":"http://localhost:8080/users/the_mighty_zork/collections/featured","followers":"http://localhost:8080/users/the_mighty_zork/followers","following":"http://localhost:8080/users/the_mighty_zork/following","icon":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg"},"id":"http://localhost:8080/users/the_mighty_zork","image":{"mediaType":"image/jpeg","type":"Image","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg"},"inbox":"http://localhost:8080/users/the_mighty_zork/inbox","manuallyApprovesFollowers":false,"name":"original zork (he/they)","outbox":"http://localhost:8080/users/the_mighty_zork/outbox","preferredUsername":"the_mighty_zork","publicKey":{"id":"http://localhost:8080/users/the_mighty_zork/main-key","owner":"http://localhost:8080/users/the_mighty_zork","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwXTcOAvM1Jiw5Ffpk0qn\nr0cwbNvFe/5zQ+Tp7tumK/ZnT37o7X0FUEXrxNi+dkhmeJ0gsaiN+JQGNUewvpSk\nPIAXKvi908aSfCGjs7bGlJCJCuDuL5d6m7hZnP9rt9fJc70GElPpG0jc9fXwlz7T\nlsPb2ecatmG05Y4jPwdC+oN4MNCv9yQzEvCVMzl76EJaM602kIHC1CISn0rDFmYd\n9rSN7XPlNJw1F6PbpJ/BWQ+pXHKw3OEwNTETAUNYiVGnZU+B7a7bZC9f6/aPbJuV\nt8Qmg+UnDvW1Y8gmfHnxaWG2f5TDBvCHmcYtucIZPLQD4trAozC4ryqlmCWQNKbt\n0wIDAQAB\n-----END PUBLIC KEY-----\n"},"summary":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","tag":[],"type":"Person","url":"http://localhost:8080/@the_mighty_zork"}`, trimmed) } func (suite *InternalToASTestSuite) TestOutboxToASCollection() { @@ -157,7 +157,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASWithIDs() { // http://joinmastodon.org/ns, https://www.w3.org/ns/activitystreams -- // will appear, so trim them out of the string for consistency trimmed := strings.SplitAfter(string(bytes), `"attachment":`)[1] - suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) + suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) } func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() { @@ -179,7 +179,7 @@ func (suite *InternalToASTestSuite) TestStatusWithTagsToASFromDB() { // http://joinmastodon.org/ns, https://www.w3.org/ns/activitystreams -- // will appear, so trim them out of the string for consistency trimmed := strings.SplitAfter(string(bytes), `"attachment":`)[1] - suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) + suite.Equal(`{"blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj","mediaType":"image/jpeg","name":"Black and white image of some 50's style text saying: Welcome On Board","type":"Document","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg"},"attributedTo":"http://localhost:8080/users/admin","cc":"http://localhost:8080/users/admin/followers","content":"hello world! #welcome ! first post on the instance :rainbow: !","id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","published":"2021-10-20T11:36:45Z","replies":{"first":{"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?page=true","next":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R/replies","type":"Collection"},"sensitive":false,"summary":"","tag":{"icon":{"mediaType":"image/png","type":"Image","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png"},"id":"http://localhost:8080/emoji/01F8MH9H8E4VG3KDYJR9EGPXCQ","name":":rainbow:","type":"Emoji","updated":"2021-09-20T10:40:37Z"},"to":"https://www.w3.org/ns/activitystreams#Public","type":"Note","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R"}`, trimmed) } func (suite *InternalToASTestSuite) TestStatusToASWithMentions() { diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go @@ -663,7 +663,7 @@ func (c *converter) InstanceToAPIInstance(ctx context.Context, i *gtsmodel.Insta CharactersReservedPerURL: instanceStatusesCharactersReservedPerURL, }, MediaAttachments: &apimodel.InstanceConfigurationMediaAttachments{ - SupportedMimeTypes: media.AllSupportedMIMETypes(), + SupportedMimeTypes: media.SupportedMIMETypes, ImageSizeLimit: int(config.GetMediaImageMaxSize()), // bytes ImageMatrixLimit: instanceMediaAttachmentsImageMatrixLimit, // height*width VideoSizeLimit: int(config.GetMediaVideoMaxSize()), // bytes diff --git a/internal/typeutils/internaltofrontend_test.go b/internal/typeutils/internaltofrontend_test.go @@ -40,7 +40,7 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontend() { b, err := json.Marshal(apiAccount) suite.NoError(err) - suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"user"}`, string(b)) + suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"user"}`, string(b)) } func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiStruct() { @@ -55,7 +55,7 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiStruct() b, err := json.Marshal(apiAccount) suite.NoError(err) - suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"fields":[],"enable_rss":true,"role":"user"}`, string(b)) + suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"fields":[],"enable_rss":true,"role":"user"}`, string(b)) } func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiIDs() { @@ -70,7 +70,7 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendWithEmojiIDs() { b, err := json.Marshal(apiAccount) suite.NoError(err) - suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"fields":[],"enable_rss":true,"role":"user"}`, string(b)) + suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"fields":[],"enable_rss":true,"role":"user"}`, string(b)) } func (suite *InternalToFrontendTestSuite) TestAccountToFrontendSensitive() { @@ -81,7 +81,7 @@ func (suite *InternalToFrontendTestSuite) TestAccountToFrontendSensitive() { b, err := json.Marshal(apiAccount) suite.NoError(err) - suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"source":{"privacy":"public","language":"en","status_format":"plain","note":"hey yo this is my profile!","fields":[]},"enable_rss":true,"role":"user"}`, string(b)) + suite.Equal(`{"id":"01F8MH1H7YV1Z7D2C8K2730QBF","username":"the_mighty_zork","acct":"the_mighty_zork","display_name":"original zork (he/they)","locked":false,"bot":false,"created_at":"2022-05-20T11:09:18.000Z","note":"\u003cp\u003ehey yo this is my profile!\u003c/p\u003e","url":"http://localhost:8080/@the_mighty_zork","avatar":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg","avatar_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg","header":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","header_static":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg","followers_count":2,"following_count":2,"statuses_count":5,"last_status_at":"2022-05-20T11:37:55.000Z","emojis":[],"fields":[],"source":{"privacy":"public","language":"en","status_format":"plain","note":"hey yo this is my profile!","fields":[]},"enable_rss":true,"role":"user"}`, string(b)) } func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() { @@ -93,7 +93,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontend() { b, err := json.Marshal(apiStatus) suite.NoError(err) - suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":"en","uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b)) + suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":"en","uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b)) } func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() { @@ -107,7 +107,7 @@ func (suite *InternalToFrontendTestSuite) TestStatusToFrontendUnknownLanguage() b, err := json.Marshal(apiStatus) suite.NoError(err) - suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":null,"uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b)) + suite.Equal(`{"id":"01F8MH75CBF9JFX4ZAD54N0W0R","created_at":"2021-10-20T11:36:45.000Z","in_reply_to_id":null,"in_reply_to_account_id":null,"sensitive":false,"spoiler_text":"","visibility":"public","language":null,"uri":"http://localhost:8080/users/admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","url":"http://localhost:8080/@admin/statuses/01F8MH75CBF9JFX4ZAD54N0W0R","replies_count":0,"reblogs_count":0,"favourites_count":1,"favourited":true,"reblogged":false,"muted":false,"bookmarked":true,"pinned":false,"content":"hello world! #welcome ! first post on the instance :rainbow: !","reblog":null,"application":{"name":"superseriousbusiness","website":"https://superserious.business"},"account":{"id":"01F8MH17FWEB39HZJ76B6VXSKF","username":"admin","acct":"admin","display_name":"","locked":false,"bot":false,"created_at":"2022-05-17T13:10:59.000Z","note":"","url":"http://localhost:8080/@admin","avatar":"","avatar_static":"","header":"http://localhost:8080/assets/default_header.png","header_static":"http://localhost:8080/assets/default_header.png","followers_count":1,"following_count":1,"statuses_count":4,"last_status_at":"2021-10-20T10:41:37.000Z","emojis":[],"fields":[],"enable_rss":true,"role":"admin"},"media_attachments":[{"id":"01F8MH6NEM8D7527KZAECTCR76","type":"image","url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg","text_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg","preview_url":"http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":1200,"height":630,"size":"1200x630","aspect":1.9047619},"small":{"width":256,"height":134,"size":"256x134","aspect":1.9104477},"focus":{"x":0,"y":0}},"description":"Black and white image of some 50's style text saying: Welcome On Board","blurhash":"LNJRdVM{00Rj%Mayt7j[4nWBofRj"}],"mentions":[],"tags":[{"name":"welcome","url":"http://localhost:8080/tags/welcome"}],"emojis":[{"shortcode":"rainbow","url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","static_url":"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/static/01F8MH9H8E4VG3KDYJR9EGPXCQ.png","visible_in_picker":true,"category":"reactions"}],"card":null,"poll":null,"text":"hello world! #welcome ! first post on the instance :rainbow: !"}`, string(b)) } func (suite *InternalToFrontendTestSuite) TestVideoAttachmentToFrontend() { @@ -118,7 +118,7 @@ func (suite *InternalToFrontendTestSuite) TestVideoAttachmentToFrontend() { b, err := json.Marshal(apiAttachment) suite.NoError(err) - suite.Equal(`{"id":"01CDR64G398ADCHXK08WWTHEZ5","type":"video","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","text_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","preview_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":720,"height":404,"frame_rate":"30/1","duration":15.033334,"bitrate":1206522,"size":"720x404","aspect":1.7821782},"small":{"width":720,"height":404,"size":"720x404","aspect":1.7821782},"focus":{"x":0,"y":0}},"description":"A cow adorably licking another cow!"}`, string(b)) + suite.Equal(`{"id":"01CDR64G398ADCHXK08WWTHEZ5","type":"video","url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","text_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01CDR64G398ADCHXK08WWTHEZ5.mp4","preview_url":"http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpg","remote_url":null,"preview_remote_url":null,"meta":{"original":{"width":720,"height":404,"frame_rate":"30/1","duration":15.033334,"bitrate":1206522,"size":"720x404","aspect":1.7821782},"small":{"width":720,"height":404,"size":"720x404","aspect":1.7821782},"focus":{"x":0,"y":0}},"description":"A cow adorably licking another cow!"}`, string(b)) } func (suite *InternalToFrontendTestSuite) TestInstanceToFrontend() { diff --git a/internal/typeutils/internaltorss_test.go b/internal/typeutils/internaltorss_test.go @@ -77,7 +77,7 @@ func (suite *InternalToRSSTestSuite) TestStatusToRSSItem2() { suite.EqualValues(1634729805, item.Created.Unix()) suite.Equal("62529", item.Enclosure.Length) suite.Equal("image/jpeg", item.Enclosure.Type) - suite.Equal("http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg", item.Enclosure.Url) + suite.Equal("http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", item.Enclosure.Url) suite.Equal("hello world! #welcome ! first post on the instance <img src=\"http://localhost:8080/fileserver/01AY6P665V14JJR0AFVRT7311Y/emoji/original/01F8MH9H8E4VG3KDYJR9EGPXCQ.png\" title=\":rainbow:\" alt=\":rainbow:\" class=\"emoji\"/> !", item.Content) } diff --git a/internal/validate/mediaattachment_test.go b/internal/validate/mediaattachment_test.go @@ -62,7 +62,7 @@ func happyMediaAttachment() *gtsmodel.MediaAttachment { CreatedAt: time.Now().Add(-71 * time.Hour), UpdatedAt: time.Now().Add(-71 * time.Hour), StatusID: "01F8MH75CBF9JFX4ZAD54N0W0R", - URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", RemoteURL: "", Type: gtsmodel.FileTypeImage, FileMeta: gtsmodel.FileMeta{ @@ -95,7 +95,7 @@ func happyMediaAttachment() *gtsmodel.MediaAttachment { ContentType: "image/jpeg", FileSize: 6872, UpdatedAt: time.Now().Add(-71 * time.Hour), - URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg", RemoteURL: "", }, Avatar: testrig.FalseBool(), diff --git a/testrig/media/ohyou-original.jpeg b/testrig/media/ohyou-original.jpg Binary files differ. diff --git a/testrig/media/ohyou-small.jpeg b/testrig/media/ohyou-small.jpg Binary files differ. diff --git a/testrig/media/team-fortress-original.jpeg b/testrig/media/team-fortress-original.jpg Binary files differ. diff --git a/testrig/media/team-fortress-small.jpeg b/testrig/media/team-fortress-small.jpg Binary files differ. diff --git a/testrig/media/thoughtsofdog-original.jpeg b/testrig/media/thoughtsofdog-original.jpg Binary files differ. diff --git a/testrig/media/thoughtsofdog-small.jpeg b/testrig/media/thoughtsofdog-small.jpeg Binary files differ. diff --git a/testrig/media/thoughtsofdog-small.jpg b/testrig/media/thoughtsofdog-small.jpg Binary files differ. diff --git a/testrig/media/trent-small.jpeg b/testrig/media/trent-small.jpg Binary files differ. diff --git a/testrig/media/welcome-original.jpeg b/testrig/media/welcome-original.jpg Binary files differ. diff --git a/testrig/media/welcome-small.jpeg b/testrig/media/welcome-small.jpg Binary files differ. diff --git a/testrig/media/zork-original.jpeg b/testrig/media/zork-original.jpg Binary files differ. diff --git a/testrig/media/zork-small.jpeg b/testrig/media/zork-small.jpg Binary files differ. diff --git a/testrig/storage.go b/testrig/storage.go @@ -55,14 +55,14 @@ func StandardStorageSetup(storage *gtsstorage.Driver, relativePath string) { if err != nil { panic(err) } - if err := storage.Put(context.TODO(), pathOriginal, bOriginal); err != nil { + if _, err := storage.Put(context.TODO(), pathOriginal, bOriginal); err != nil { panic(err) } bSmall, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameSmall)) if err != nil { panic(err) } - if err := storage.Put(context.TODO(), pathSmall, bSmall); err != nil { + if _, err := storage.Put(context.TODO(), pathSmall, bSmall); err != nil { panic(err) } } @@ -82,14 +82,14 @@ func StandardStorageSetup(storage *gtsstorage.Driver, relativePath string) { if err != nil { panic(err) } - if err := storage.Put(context.TODO(), pathOriginal, bOriginal); err != nil { + if _, err := storage.Put(context.TODO(), pathOriginal, bOriginal); err != nil { panic(err) } bStatic, err := os.ReadFile(fmt.Sprintf("%s/%s", relativePath, filenameStatic)) if err != nil { panic(err) } - if err := storage.Put(context.TODO(), pathStatic, bStatic); err != nil { + if _, err := storage.Put(context.TODO(), pathStatic, bStatic); err != nil { panic(err) } } diff --git a/testrig/testmodels.go b/testrig/testmodels.go @@ -689,7 +689,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { "admin_account_status_1_attachment_1": { ID: "01F8MH6NEM8D7527KZAECTCR76", StatusID: "01F8MH75CBF9JFX4ZAD54N0W0R", - URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-04T13:12:00Z"), UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"), @@ -714,17 +714,17 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Blurhash: "LNJRdVM{00Rj%Mayt7j[4nWBofRj", Processing: 2, File: gtsmodel.File{ - Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg", + Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpg", ContentType: "image/jpeg", FileSize: 62529, UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg", + Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg", ContentType: "image/jpeg", FileSize: 6872, UpdatedAt: TimeMustParse("2022-06-04T13:12:00Z"), - URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpg", RemoteURL: "", }, Avatar: FalseBool(), @@ -769,11 +769,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpeg", + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpg", ContentType: "image/jpeg", FileSize: 8803, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpg", RemoteURL: "", }, Avatar: FalseBool(), @@ -821,11 +821,11 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg", + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpg", ContentType: "image/jpeg", FileSize: 5272, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01CDR64G398ADCHXK08WWTHEZ5.jpg", RemoteURL: "", }, Avatar: FalseBool(), @@ -835,7 +835,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { "local_account_1_unattached_1": { ID: "01F8MH8RMYQ6MSNY3JM2XT1CQ5", StatusID: "", // this attachment isn't connected to a status YET - URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), @@ -864,17 +864,17 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Blurhash: "LSAd]9ogDge-R:M|j=xWIto0xXWX", Processing: 2, File: gtsmodel.File{ - Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg", + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg", ContentType: "image/jpeg", FileSize: 27759, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg", + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg", ContentType: "image/jpeg", FileSize: 6177, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpg", RemoteURL: "", }, Avatar: FalseBool(), @@ -884,7 +884,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { "local_account_1_avatar": { ID: "01F8MH58A357CV5K7R7TJMSH6S", StatusID: "", // this attachment isn't connected to a status - URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), @@ -913,17 +913,17 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Blurhash: "LKK9MT,p|YSNDkJ-5rsmvnwcOoe:", Processing: 2, File: gtsmodel.File{ - Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg", + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpg", ContentType: "image/jpeg", FileSize: 457680, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg", + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg", ContentType: "image/jpeg", FileSize: 15374, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpg", RemoteURL: "", }, Avatar: TrueBool(), @@ -933,7 +933,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { "local_account_1_header": { ID: "01PFPMWK2FF0D9WMHEJHR07C3Q", StatusID: "", - URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", RemoteURL: "", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), @@ -962,17 +962,17 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Blurhash: "L26j{^WCs+R-N}jsxWj@4;WWxDoK", Processing: 2, File: gtsmodel.File{ - Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", ContentType: "image/jpeg", FileSize: 517226, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", + Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", ContentType: "image/jpeg", FileSize: 42308, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpg", RemoteURL: "", }, Avatar: FalseBool(), @@ -982,8 +982,8 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { "remote_account_1_status_1_attachment_1": { ID: "01FVW7RXPQ8YJHTEXYPE7Q8ZY0", StatusID: "01FVW7JHQFSFK166WWKR8CBA6M", - URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", - RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", + RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpg", CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), Type: gtsmodel.FileTypeImage, @@ -1011,18 +1011,18 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6", Processing: 2, File: gtsmodel.File{ - Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", + Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", ContentType: "image/jpeg", FileSize: 19310, UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", + Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", ContentType: "image/jpeg", - FileSize: 20395, + FileSize: 19312, UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), - URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", - RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", + RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg", }, Avatar: FalseBool(), Header: FalseBool(), @@ -1031,8 +1031,8 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { "remote_account_1_status_1_attachment_2": { ID: "01FVW7RXPQ8YJHTEXYPE7Q8ZY1", StatusID: "01FVW7JHQFSFK166WWKR8CBA6M", - URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", - RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", + RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpg", CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), Type: gtsmodel.FileTypeImage, @@ -1060,18 +1060,18 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6", Processing: 2, File: gtsmodel.File{ - Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", + Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", ContentType: "image/jpeg", FileSize: 19310, UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", + Path: "01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", ContentType: "image/jpeg", FileSize: 20395, UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"), - URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg", - RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg", + URL: "http://localhost:8080/fileserver/01F8MH5ZK5VRH73AKHQM6Y9VNX/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpg", + RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg", }, Avatar: FalseBool(), Header: FalseBool(), @@ -1080,8 +1080,8 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { "remote_account_3_header": { ID: "01PFPMWK2FF0D9WMHEJHR07C3R", StatusID: "", - URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg", - RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg", + URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg", + RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg", CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"), UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), Type: gtsmodel.FileTypeImage, @@ -1109,18 +1109,18 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment { Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6", Processing: 2, File: gtsmodel.File{ - Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg", + Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpg", ContentType: "image/jpeg", FileSize: 19310, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), }, Thumbnail: gtsmodel.Thumbnail{ - Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg", + Path: "062G5WYKY35KKD12EMSM3F8PJ8/attachment/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpg", ContentType: "image/jpeg", FileSize: 20395, UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"), - URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg", - RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg", + URL: "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpg", + RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpg", }, Avatar: FalseBool(), Header: TrueBool(), @@ -1262,32 +1262,32 @@ type filenames struct { func newTestStoredAttachments() map[string]filenames { return map[string]filenames{ "admin_account_status_1_attachment_1": { - Original: "welcome-original.jpeg", - Small: "welcome-small.jpeg", + Original: "welcome-original.jpg", + Small: "welcome-small.jpg", }, "local_account_1_status_4_attachment_1": { Original: "trent-original.gif", - Small: "trent-small.jpeg", + Small: "trent-small.jpg", }, "local_account_1_status_4_attachment_2": { Original: "cowlick-original.mp4", Small: "cowlick-small.jpeg", }, "local_account_1_unattached_1": { - Original: "ohyou-original.jpeg", - Small: "ohyou-small.jpeg", + Original: "ohyou-original.jpg", + Small: "ohyou-small.jpg", }, "local_account_1_avatar": { - Original: "zork-original.jpeg", - Small: "zork-small.jpeg", + Original: "zork-original.jpg", + Small: "zork-small.jpg", }, "local_account_1_header": { - Original: "team-fortress-original.jpeg", - Small: "team-fortress-small.jpeg", + Original: "team-fortress-original.jpg", + Small: "team-fortress-small.jpg", }, "remote_account_1_status_1_attachment_1": { - Original: "thoughtsofdog-original.jpeg", - Small: "thoughtsofdog-small.jpeg", + Original: "thoughtsofdog-original.jpg", + Small: "thoughtsofdog-small.jpg", }, } } @@ -2070,7 +2070,7 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit []vocab.ActivityStreamsMention{}, []vocab.ActivityStreamsImage{ newAPImage( - URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpeg"), + URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpg"), "image/jpeg", "trent reznor looking handsome as balls", "LEDara58O=t5EMSOENEN9]}?aK%0"), @@ -2322,7 +2322,7 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile panic(err) } - thoughtsOfDogBytes, err := os.ReadFile(fmt.Sprintf("%s/thoughtsofdog-original.jpeg", relativePath)) + thoughtsOfDogBytes, err := os.ReadFile(fmt.Sprintf("%s/thoughtsofdog-original.jpg", relativePath)) if err != nil { panic(err) } @@ -2352,7 +2352,7 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile Data: beeBytes, ContentType: "image/jpeg", }, - "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg": { + "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpg": { Data: thoughtsOfDogBytes, ContentType: "image/jpeg", }, @@ -2390,7 +2390,7 @@ func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote { []vocab.ActivityStreamsMention{}, []vocab.ActivityStreamsImage{ newAPImage( - URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpeg"), + URLMustParse("http://example.org/users/Some_User/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpg"), "image/jpeg", "trent reznor looking handsome as balls", "LEDara58O=t5EMSOENEN9]}?aK%0"), diff --git a/vendor/codeberg.org/gruf/go-fastcopy/copy.go b/vendor/codeberg.org/gruf/go-fastcopy/copy.go @@ -78,16 +78,16 @@ func (cp *CopyPool) Copy(dst io.Writer, src io.Reader) (int64, error) { var buf []byte - if b, ok := cp.pool.Get().([]byte); ok { + if b, ok := cp.pool.Get().(*[]byte); ok { // Acquired buf from pool - buf = b + buf = *b } else { // Allocate new buffer of size buf = make([]byte, cp.Buffer(0)) } // Defer release to pool - defer cp.pool.Put(buf) + defer cp.pool.Put(&buf) var n int64 for { diff --git a/vendor/codeberg.org/gruf/go-iotools/LICENSE b/vendor/codeberg.org/gruf/go-iotools/LICENSE @@ -0,0 +1,9 @@ +MIT License + +Copyright (c) 2022 gruf + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/codeberg.org/gruf/go-iotools/close.go b/vendor/codeberg.org/gruf/go-iotools/close.go @@ -0,0 +1,35 @@ +package iotools + +import "io" + +// CloserFunc is a function signature which allows +// a function to implement the io.Closer type. +type CloserFunc func() error + +func (c CloserFunc) Close() error { + return c() +} + +func CloserCallback(c io.Closer, cb func()) io.Closer { + return CloserFunc(func() error { + defer cb() + return c.Close() + }) +} + +// CloseOnce wraps an io.Closer to ensure it only performs the close logic once. +func CloseOnce(c io.Closer) io.Closer { + return CloserFunc(func() error { + if c == nil { + // already run. + return nil + } + + // Acquire. + cptr := c + c = nil + + // Call the closer. + return cptr.Close() + }) +} diff --git a/vendor/codeberg.org/gruf/go-iotools/read.go b/vendor/codeberg.org/gruf/go-iotools/read.go @@ -0,0 +1,28 @@ +package iotools + +import ( + "io" +) + +// ReaderFunc is a function signature which allows +// a function to implement the io.Reader type. +type ReaderFunc func([]byte) (int, error) + +func (r ReaderFunc) Read(b []byte) (int, error) { + return r(b) +} + +// ReadCloser wraps an io.Reader and io.Closer in order to implement io.ReadCloser. +func ReadCloser(r io.Reader, c io.Closer) io.ReadCloser { + return &struct { + io.Reader + io.Closer + }{r, c} +} + +// NopReadCloser wraps an io.Reader to implement io.ReadCloser with empty io.Closer implementation. +func NopReadCloser(r io.Reader) io.ReadCloser { + return ReadCloser(r, CloserFunc(func() error { + return nil + })) +} diff --git a/vendor/codeberg.org/gruf/go-iotools/write.go b/vendor/codeberg.org/gruf/go-iotools/write.go @@ -0,0 +1,26 @@ +package iotools + +import "io" + +// WriterFunc is a function signature which allows +// a function to implement the io.Writer type. +type WriterFunc func([]byte) (int, error) + +func (w WriterFunc) Write(b []byte) (int, error) { + return w(b) +} + +// WriteCloser wraps an io.Writer and io.Closer in order to implement io.WriteCloser. +func WriteCloser(w io.Writer, c io.Closer) io.WriteCloser { + return &struct { + io.Writer + io.Closer + }{w, c} +} + +// NopWriteCloser wraps an io.Writer to implement io.WriteCloser with empty io.Closer implementation. +func NopWriteCloser(w io.Writer) io.WriteCloser { + return WriteCloser(w, CloserFunc(func() error { + return nil + })) +} diff --git a/vendor/codeberg.org/gruf/go-mutexes/map.go b/vendor/codeberg.org/gruf/go-mutexes/map.go @@ -454,7 +454,9 @@ func (mu *rwmutex) Unlock() { if mu.rcnt > 0 { // RUnlock mu.rcnt-- - } else { + } + + if mu.rcnt == 0 { // Total unlock mu.lock = 0 } diff --git a/vendor/codeberg.org/gruf/go-store/v2/kv/state.go b/vendor/codeberg.org/gruf/go-store/v2/kv/state.go @@ -77,17 +77,17 @@ func (st *StateRW) GetStream(ctx context.Context, key string) (io.ReadCloser, er } // Put: see KVStore.Put(). Returns error if state already closed. -func (st *StateRW) Put(ctx context.Context, key string, value []byte) error { +func (st *StateRW) Put(ctx context.Context, key string, value []byte) (int, error) { if st.store == nil { - return ErrStateClosed + return 0, ErrStateClosed } return st.store.put(st.state.Lock, ctx, key, value) } // PutStream: see KVStore.PutStream(). Returns error if state already closed. -func (st *StateRW) PutStream(ctx context.Context, key string, r io.Reader) error { +func (st *StateRW) PutStream(ctx context.Context, key string, r io.Reader) (int64, error) { if st.store == nil { - return ErrStateClosed + return 0, ErrStateClosed } return st.store.putStream(st.state.Lock, ctx, key, r) } diff --git a/vendor/codeberg.org/gruf/go-store/v2/kv/store.go b/vendor/codeberg.org/gruf/go-store/v2/kv/store.go @@ -4,9 +4,9 @@ import ( "context" "io" + "codeberg.org/gruf/go-iotools" "codeberg.org/gruf/go-mutexes" "codeberg.org/gruf/go-store/v2/storage" - "codeberg.org/gruf/go-store/v2/util" ) // KVStore is a very simple, yet performant key-value store @@ -117,17 +117,25 @@ func (st *KVStore) getStream(rlock func(string) func(), ctx context.Context, key return nil, err } - // Wrap readcloser in our own callback closer - return util.ReadCloserWithCallback(rd, runlock), nil + var unlocked bool + + // Wrap readcloser to call our own callback + return iotools.ReadCloser(rd, iotools.CloserFunc(func() error { + if !unlocked { + unlocked = true + defer runlock() + } + return rd.Close() + })), nil } // Put places the bytes at the supplied key in the store. -func (st *KVStore) Put(ctx context.Context, key string, value []byte) error { +func (st *KVStore) Put(ctx context.Context, key string, value []byte) (int, error) { return st.put(st.Lock, ctx, key, value) } // put performs the underlying logic for KVStore.Put(), using supplied lock func to allow use with states. -func (st *KVStore) put(lock func(string) func(), ctx context.Context, key string, value []byte) error { +func (st *KVStore) put(lock func(string) func(), ctx context.Context, key string, value []byte) (int, error) { // Acquire write lock for key unlock := lock(key) defer unlock() @@ -137,12 +145,12 @@ func (st *KVStore) put(lock func(string) func(), ctx context.Context, key string } // PutStream writes the bytes from the supplied Reader at the supplied key in the store. -func (st *KVStore) PutStream(ctx context.Context, key string, r io.Reader) error { +func (st *KVStore) PutStream(ctx context.Context, key string, r io.Reader) (int64, error) { return st.putStream(st.Lock, ctx, key, r) } // putStream performs the underlying logic for KVStore.PutStream(), using supplied lock func to allow use with states. -func (st *KVStore) putStream(lock func(string) func(), ctx context.Context, key string, r io.Reader) error { +func (st *KVStore) putStream(lock func(string) func(), ctx context.Context, key string, r io.Reader) (int64, error) { // Acquire write lock for key unlock := lock(key) defer unlock() diff --git a/vendor/codeberg.org/gruf/go-store/v2/storage/block.go b/vendor/codeberg.org/gruf/go-store/v2/storage/block.go @@ -10,12 +10,14 @@ import ( "os" "strings" "sync" + "sync/atomic" "syscall" "codeberg.org/gruf/go-byteutil" "codeberg.org/gruf/go-errors/v2" "codeberg.org/gruf/go-fastcopy" "codeberg.org/gruf/go-hashenc" + "codeberg.org/gruf/go-iotools" "codeberg.org/gruf/go-pools" "codeberg.org/gruf/go-store/v2/util" ) @@ -354,7 +356,7 @@ func (st *BlockStorage) ReadStream(ctx context.Context, key string) (io.ReadClos } // Prepare block reader and return - return util.NopReadCloser(&blockReader{ + return iotools.NopReadCloser(&blockReader{ storage: st, node: &node, }), nil @@ -384,52 +386,54 @@ func (st *BlockStorage) readBlock(key string) ([]byte, error) { } // WriteBytes implements Storage.WriteBytes(). -func (st *BlockStorage) WriteBytes(ctx context.Context, key string, value []byte) error { - return st.WriteStream(ctx, key, bytes.NewReader(value)) +func (st *BlockStorage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) { + n, err := st.WriteStream(ctx, key, bytes.NewReader(value)) + return int(n), err } // WriteStream implements Storage.WriteStream(). -func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader) error { +func (st *BlockStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) { // Get node file path for key npath, err := st.nodePathForKey(key) if err != nil { - return err + return 0, err } // Check if open if st.lock.Closed() { - return ErrClosed + return 0, ErrClosed } // Check context still valid if err := ctx.Err(); err != nil { - return err + return 0, err } // Check if this exists ok, err := stat(key) if err != nil { - return err + return 0, err } // Check if we allow overwrites if ok && !st.config.Overwrite { - return ErrAlreadyExists + return 0, ErrAlreadyExists } // Ensure nodes dir (and any leading up to) exists err = os.MkdirAll(st.nodePath, defaultDirPerms) if err != nil { - return err + return 0, err } // Ensure blocks dir (and any leading up to) exists err = os.MkdirAll(st.blockPath, defaultDirPerms) if err != nil { - return err + return 0, err } var node node + var total atomic.Int64 // Acquire HashEncoder hc := st.hashPool.Get().(*hashEncoder) @@ -456,7 +460,7 @@ loop: break loop default: st.bufpool.Put(buf) - return err + return 0, err } // Hash the encoded data @@ -469,7 +473,7 @@ loop: has, err := st.statBlock(sum) if err != nil { st.bufpool.Put(buf) - return err + return 0, err } else if has { st.bufpool.Put(buf) continue loop @@ -490,11 +494,14 @@ loop: }() // Write block to store at hash - err = st.writeBlock(sum, buf.B[:n]) + n, err := st.writeBlock(sum, buf.B[:n]) if err != nil { onceErr.Store(err) return } + + // Increment total. + total.Add(int64(n)) }() // Break at end @@ -506,12 +513,12 @@ loop: // Wait, check errors wg.Wait() if onceErr.IsSet() { - return onceErr.Load() + return 0, onceErr.Load() } // If no hashes created, return if len(node.hashes) < 1 { - return new_error("no hashes written") + return 0, new_error("no hashes written") } // Prepare to swap error if need-be @@ -535,7 +542,7 @@ loop: // Attempt to open RW file file, err := open(npath, flags) if err != nil { - return errSwap(err) + return 0, errSwap(err) } defer file.Close() @@ -546,11 +553,11 @@ loop: // Finally, write data to file _, err = io.CopyBuffer(file, &nodeReader{node: node}, buf.B) - return err + return total.Load(), err } // writeBlock writes the block with hash and supplied value to the filesystem. -func (st *BlockStorage) writeBlock(hash string, value []byte) error { +func (st *BlockStorage) writeBlock(hash string, value []byte) (int, error) { // Get block file path for key bpath := st.blockPathForKey(hash) @@ -560,20 +567,19 @@ func (st *BlockStorage) writeBlock(hash string, value []byte) error { if err == syscall.EEXIST { err = nil /* race issue describe in struct NOTE */ } - return err + return 0, err } defer file.Close() // Wrap the file in a compressor cFile, err := st.config.Compression.Writer(file) if err != nil { - return err + return 0, err } defer cFile.Close() // Write value to file - _, err = cFile.Write(value) - return err + return cFile.Write(value) } // statBlock checks for existence of supplied block hash. diff --git a/vendor/codeberg.org/gruf/go-store/v2/storage/compressor.go b/vendor/codeberg.org/gruf/go-store/v2/storage/compressor.go @@ -5,7 +5,7 @@ import ( "io" "sync" - "codeberg.org/gruf/go-store/v2/util" + "codeberg.org/gruf/go-iotools" "github.com/klauspost/compress/gzip" "github.com/klauspost/compress/snappy" @@ -15,10 +15,10 @@ import ( // Compressor defines a means of compressing/decompressing values going into a key-value store type Compressor interface { // Reader returns a new decompressing io.ReadCloser based on supplied (compressed) io.Reader - Reader(io.Reader) (io.ReadCloser, error) + Reader(io.ReadCloser) (io.ReadCloser, error) // Writer returns a new compressing io.WriteCloser based on supplied (uncompressed) io.Writer - Writer(io.Writer) (io.WriteCloser, error) + Writer(io.WriteCloser) (io.WriteCloser, error) } type gzipCompressor struct { @@ -47,8 +47,8 @@ func GZipCompressorLevel(level int) Compressor { // Write empty data to ensure gzip // header data is in byte buffer. - gw.Write([]byte{}) - gw.Close() + _, _ = gw.Write([]byte{}) + _ = gw.Close() return &gzipCompressor{ rpool: sync.Pool{ @@ -67,23 +67,61 @@ func GZipCompressorLevel(level int) Compressor { } } -func (c *gzipCompressor) Reader(r io.Reader) (io.ReadCloser, error) { +func (c *gzipCompressor) Reader(rc io.ReadCloser) (io.ReadCloser, error) { + var released bool + + // Acquire from pool. gr := c.rpool.Get().(*gzip.Reader) - if err := gr.Reset(r); err != nil { + if err := gr.Reset(rc); err != nil { c.rpool.Put(gr) return nil, err } - return util.ReadCloserWithCallback(gr, func() { - c.rpool.Put(gr) - }), nil + + return iotools.ReadCloser(gr, iotools.CloserFunc(func() error { + if !released { + released = true + defer c.rpool.Put(gr) + } + + // Close compressor + err1 := gr.Close() + + // Close original stream. + err2 := rc.Close() + + // Return err1 or 2 + if err1 != nil { + return err1 + } + return err2 + })), nil } -func (c *gzipCompressor) Writer(w io.Writer) (io.WriteCloser, error) { +func (c *gzipCompressor) Writer(wc io.WriteCloser) (io.WriteCloser, error) { + var released bool + + // Acquire from pool. gw := c.wpool.Get().(*gzip.Writer) - gw.Reset(w) - return util.WriteCloserWithCallback(gw, func() { - c.wpool.Put(gw) - }), nil + gw.Reset(wc) + + return iotools.WriteCloser(gw, iotools.CloserFunc(func() error { + if !released { + released = true + c.wpool.Put(gw) + } + + // Close compressor + err1 := gw.Close() + + // Close original stream. + err2 := wc.Close() + + // Return err1 or 2 + if err1 != nil { + return err1 + } + return err2 + })), nil } type zlibCompressor struct { @@ -139,26 +177,61 @@ func ZLibCompressorLevelDict(level int, dict []byte) Compressor { } } -func (c *zlibCompressor) Reader(r io.Reader) (io.ReadCloser, error) { +func (c *zlibCompressor) Reader(rc io.ReadCloser) (io.ReadCloser, error) { + var released bool zr := c.rpool.Get().(interface { io.ReadCloser zlib.Resetter }) - if err := zr.Reset(r, c.dict); err != nil { + if err := zr.Reset(rc, c.dict); err != nil { c.rpool.Put(zr) return nil, err } - return util.ReadCloserWithCallback(zr, func() { - c.rpool.Put(zr) - }), nil + return iotools.ReadCloser(zr, iotools.CloserFunc(func() error { + if !released { + released = true + defer c.rpool.Put(zr) + } + + // Close compressor + err1 := zr.Close() + + // Close original stream. + err2 := rc.Close() + + // Return err1 or 2 + if err1 != nil { + return err1 + } + return err2 + })), nil } -func (c *zlibCompressor) Writer(w io.Writer) (io.WriteCloser, error) { +func (c *zlibCompressor) Writer(wc io.WriteCloser) (io.WriteCloser, error) { + var released bool + + // Acquire from pool. zw := c.wpool.Get().(*zlib.Writer) - zw.Reset(w) - return util.WriteCloserWithCallback(zw, func() { - c.wpool.Put(zw) - }), nil + zw.Reset(wc) + + return iotools.WriteCloser(zw, iotools.CloserFunc(func() error { + if !released { + released = true + c.wpool.Put(zw) + } + + // Close compressor + err1 := zw.Close() + + // Close original stream. + err2 := wc.Close() + + // Return err1 or 2 + if err1 != nil { + return err1 + } + return err2 + })), nil } type snappyCompressor struct { @@ -178,22 +251,40 @@ func SnappyCompressor() Compressor { } } -func (c *snappyCompressor) Reader(r io.Reader) (io.ReadCloser, error) { +func (c *snappyCompressor) Reader(rc io.ReadCloser) (io.ReadCloser, error) { + var released bool + + // Acquire from pool. sr := c.rpool.Get().(*snappy.Reader) - sr.Reset(r) - return util.ReadCloserWithCallback( - util.NopReadCloser(sr), - func() { c.rpool.Put(sr) }, - ), nil + sr.Reset(rc) + + return iotools.ReadCloser(sr, iotools.CloserFunc(func() error { + if !released { + released = true + defer c.rpool.Put(sr) + } + + // Close original stream. + return rc.Close() + })), nil } -func (c *snappyCompressor) Writer(w io.Writer) (io.WriteCloser, error) { +func (c *snappyCompressor) Writer(wc io.WriteCloser) (io.WriteCloser, error) { + var released bool + + // Acquire from pool. sw := c.wpool.Get().(*snappy.Writer) - sw.Reset(w) - return util.WriteCloserWithCallback( - util.NopWriteCloser(sw), - func() { c.wpool.Put(sw) }, - ), nil + sw.Reset(wc) + + return iotools.WriteCloser(sw, iotools.CloserFunc(func() error { + if !released { + released = true + c.wpool.Put(sw) + } + + // Close original stream. + return wc.Close() + })), nil } type nopCompressor struct{} @@ -203,10 +294,10 @@ func NoCompression() Compressor { return &nopCompressor{} } -func (c *nopCompressor) Reader(r io.Reader) (io.ReadCloser, error) { - return util.NopReadCloser(r), nil +func (c *nopCompressor) Reader(rc io.ReadCloser) (io.ReadCloser, error) { + return rc, nil } -func (c *nopCompressor) Writer(w io.Writer) (io.WriteCloser, error) { - return util.NopWriteCloser(w), nil +func (c *nopCompressor) Writer(wc io.WriteCloser) (io.WriteCloser, error) { + return wc, nil } diff --git a/vendor/codeberg.org/gruf/go-store/v2/storage/disk.go b/vendor/codeberg.org/gruf/go-store/v2/storage/disk.go @@ -219,43 +219,41 @@ func (st *DiskStorage) ReadStream(ctx context.Context, key string) (io.ReadClose // Wrap the file in a compressor cFile, err := st.config.Compression.Reader(file) if err != nil { - file.Close() // close this here, ignore error + _ = file.Close() return nil, err } - // Wrap compressor to ensure file close - return util.ReadCloserWithCallback(cFile, func() { - file.Close() - }), nil + return cFile, nil } // WriteBytes implements Storage.WriteBytes(). -func (st *DiskStorage) WriteBytes(ctx context.Context, key string, value []byte) error { - return st.WriteStream(ctx, key, bytes.NewReader(value)) +func (st *DiskStorage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) { + n, err := st.WriteStream(ctx, key, bytes.NewReader(value)) + return int(n), err } // WriteStream implements Storage.WriteStream(). -func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader) error { +func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) { // Get file path for key kpath, err := st.filepath(key) if err != nil { - return err + return 0, err } // Check if open if st.lock.Closed() { - return ErrClosed + return 0, ErrClosed } // Check context still valid if err := ctx.Err(); err != nil { - return err + return 0, err } // Ensure dirs leading up to file exist err = os.MkdirAll(path.Dir(kpath), defaultDirPerms) if err != nil { - return err + return 0, err } // Prepare to swap error if need-be @@ -273,20 +271,21 @@ func (st *DiskStorage) WriteStream(ctx context.Context, key string, r io.Reader) // Attempt to open file file, err := open(kpath, flags) if err != nil { - return errSwap(err) + return 0, errSwap(err) } - defer file.Close() // Wrap the file in a compressor cFile, err := st.config.Compression.Writer(file) if err != nil { - return err + _ = file.Close() + return 0, err } + + // Wraps file.Close(). defer cFile.Close() // Copy provided reader to file - _, err = st.cppool.Copy(cFile, r) - return err + return st.cppool.Copy(cFile, r) } // Stat implements Storage.Stat(). diff --git a/vendor/codeberg.org/gruf/go-store/v2/storage/fs.go b/vendor/codeberg.org/gruf/go-store/v2/storage/fs.go @@ -1,6 +1,7 @@ package storage import ( + "fmt" "io/fs" "os" "syscall" @@ -102,46 +103,32 @@ outer: // cleanDirs traverses the dir tree of the supplied path, removing any folders with zero children func cleanDirs(path string) error { - // Acquire path builder pb := util.GetPathBuilder() defer util.PutPathBuilder(pb) - - // Get top-level dir entries - entries, err := readDir(path) - if err != nil { - return err - } - - for _, entry := range entries { - if entry.IsDir() { - // Recursively clean sub-directory entries - if err := cleanDir(pb, pb.Join(path, entry.Name())); err != nil { - return err - } - } - } - - return nil + return cleanDir(pb, path, true) } // cleanDir performs the actual dir cleaning logic for the above top-level version. -func cleanDir(pb *fastpath.Builder, path string) error { - // Get dir entries +func cleanDir(pb *fastpath.Builder, path string, top bool) error { + // Get dir entries at path. entries, err := readDir(path) if err != nil { return err } - // If no entries, delete - if len(entries) < 1 { + // If no entries, delete dir. + if !top && len(entries) == 0 { return rmdir(path) } for _, entry := range entries { if entry.IsDir() { - // Recursively clean sub-directory entries - if err := cleanDir(pb, pb.Join(path, entry.Name())); err != nil { - return err + // Calculate directory path. + dirPath := pb.Join(path, entry.Name()) + + // Recursively clean sub-directory entries. + if err := cleanDir(pb, dirPath, false); err != nil { + fmt.Fprintf(os.Stderr, "[go-store/storage] error cleaning %s: %v", dirPath, err) } } } diff --git a/vendor/codeberg.org/gruf/go-store/v2/storage/memory.go b/vendor/codeberg.org/gruf/go-store/v2/storage/memory.go @@ -6,7 +6,7 @@ import ( "sync/atomic" "codeberg.org/gruf/go-bytes" - "codeberg.org/gruf/go-store/v2/util" + "codeberg.org/gruf/go-iotools" "github.com/cornelk/hashmap" ) @@ -86,57 +86,57 @@ func (st *MemoryStorage) ReadStream(ctx context.Context, key string) (io.ReadClo // Create io.ReadCloser from 'b' copy r := bytes.NewReader(copyb(b)) - return util.NopReadCloser(r), nil + return iotools.NopReadCloser(r), nil } // WriteBytes implements Storage.WriteBytes(). -func (st *MemoryStorage) WriteBytes(ctx context.Context, key string, b []byte) error { +func (st *MemoryStorage) WriteBytes(ctx context.Context, key string, b []byte) (int, error) { // Check store open if st.closed() { - return ErrClosed + return 0, ErrClosed } // Check context still valid if err := ctx.Err(); err != nil { - return err + return 0, err } // Check for key that already exists if _, ok := st.fs.Get(key); ok && !st.ow { - return ErrAlreadyExists + return 0, ErrAlreadyExists } // Write key copy to store st.fs.Set(key, copyb(b)) - return nil + return len(b), nil } // WriteStream implements Storage.WriteStream(). -func (st *MemoryStorage) WriteStream(ctx context.Context, key string, r io.Reader) error { +func (st *MemoryStorage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) { // Check store open if st.closed() { - return ErrClosed + return 0, ErrClosed } // Check context still valid if err := ctx.Err(); err != nil { - return err + return 0, err } // Check for key that already exists if _, ok := st.fs.Get(key); ok && !st.ow { - return ErrAlreadyExists + return 0, ErrAlreadyExists } // Read all from reader b, err := io.ReadAll(r) if err != nil { - return err + return 0, err } // Write key to store st.fs.Set(key, b) - return nil + return int64(len(b)), nil } // Stat implements Storage.Stat(). diff --git a/vendor/codeberg.org/gruf/go-store/v2/storage/s3.go b/vendor/codeberg.org/gruf/go-store/v2/storage/s3.go @@ -160,22 +160,23 @@ func (st *S3Storage) ReadStream(ctx context.Context, key string) (io.ReadCloser, } // WriteBytes implements Storage.WriteBytes(). -func (st *S3Storage) WriteBytes(ctx context.Context, key string, value []byte) error { - return st.WriteStream(ctx, key, util.NewByteReaderSize(value)) +func (st *S3Storage) WriteBytes(ctx context.Context, key string, value []byte) (int, error) { + n, err := st.WriteStream(ctx, key, util.NewByteReaderSize(value)) + return int(n), err } // WriteStream implements Storage.WriteStream(). -func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) error { +func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) { // Check storage open if st.closed() { - return ErrClosed + return 0, ErrClosed } if rs, ok := r.(util.ReaderSize); ok { // This reader supports providing us the size of // the encompassed data, allowing us to perform // a singular .PutObject() call with length. - _, err := st.client.PutObject( + info, err := st.client.PutObject( ctx, st.bucket, key, @@ -186,9 +187,9 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e st.config.PutOpts, ) if err != nil { - return transformS3Error(err) + err = transformS3Error(err) } - return nil + return info.Size, err } // Start a new multipart upload to get ID @@ -199,14 +200,15 @@ func (st *S3Storage) WriteStream(ctx context.Context, key string, r io.Reader) e st.config.PutOpts, ) if err != nil { - return transformS3Error(err) + return 0, transformS3Error(err) } var ( - count = 1 + index = int(1) // parts index + total = int64(0) parts []minio.CompletePart chunk = make([]byte, st.config.PutChunkSize) - rdr = bytes.NewReader(nil) + rbuf = bytes.NewReader(nil) ) // Note that we do not perform any kind of @@ -234,11 +236,11 @@ loop: // All other errors default: - return err + return 0, err } // Reset byte reader - rdr.Reset(chunk[:n]) + rbuf.Reset(chunk[:n]) // Put this object chunk in S3 store pt, err := st.client.PutObjectPart( @@ -246,15 +248,15 @@ loop: st.bucket, key, uploadID, - count, - rdr, + index, + rbuf, int64(n), "", "", nil, ) if err != nil { - return err + return 0, err } // Append completed part to slice @@ -267,8 +269,11 @@ loop: ChecksumSHA256: pt.ChecksumSHA256, }) - // Iterate part count - count++ + // Iterate idx + index++ + + // Update total size + total += pt.Size } // Complete this multi-part upload operation @@ -281,10 +286,10 @@ loop: st.config.PutOpts, ) if err != nil { - return err + return 0, err } - return nil + return total, nil } // Stat implements Storage.Stat(). diff --git a/vendor/codeberg.org/gruf/go-store/v2/storage/storage.go b/vendor/codeberg.org/gruf/go-store/v2/storage/storage.go @@ -14,10 +14,10 @@ type Storage interface { ReadStream(ctx context.Context, key string) (io.ReadCloser, error) // WriteBytes writes the supplied value bytes at key in the storage - WriteBytes(ctx context.Context, key string, value []byte) error + WriteBytes(ctx context.Context, key string, value []byte) (int, error) // WriteStream writes the bytes from supplied reader at key in the storage - WriteStream(ctx context.Context, key string, r io.Reader) error + WriteStream(ctx context.Context, key string, r io.Reader) (int64, error) // Stat checks if the supplied key is in the storage Stat(ctx context.Context, key string) (bool, error) diff --git a/vendor/codeberg.org/gruf/go-store/v2/util/io.go b/vendor/codeberg.org/gruf/go-store/v2/util/io.go @@ -5,102 +5,37 @@ import ( "io" ) -// ReaderSize ... +// ReaderSize defines a reader of known size in bytes. type ReaderSize interface { io.Reader - - // Size ... Size() int64 } -// ByteReaderSize ... +// ByteReaderSize implements ReaderSize for an in-memory byte-slice. type ByteReaderSize struct { - bytes.Reader + br bytes.Reader sz int64 } -// NewByteReaderSize ... +// NewByteReaderSize returns a new ByteReaderSize instance reset to slice b. func NewByteReaderSize(b []byte) *ByteReaderSize { - rs := ByteReaderSize{} + rs := new(ByteReaderSize) rs.Reset(b) - return &rs + return rs +} + +// Read implements io.Reader. +func (rs *ByteReaderSize) Read(b []byte) (int, error) { + return rs.br.Read(b) } -// Size implements ReaderSize.Size(). -func (rs ByteReaderSize) Size() int64 { +// Size implements ReaderSize. +func (rs *ByteReaderSize) Size() int64 { return rs.sz } // Reset resets the ReaderSize to be reading from b. func (rs *ByteReaderSize) Reset(b []byte) { - rs.Reader.Reset(b) + rs.br.Reset(b) rs.sz = int64(len(b)) } - -// NopReadCloser turns a supplied io.Reader into io.ReadCloser with a nop Close() implementation. -func NopReadCloser(r io.Reader) io.ReadCloser { - return &nopReadCloser{r} -} - -// NopWriteCloser turns a supplied io.Writer into io.WriteCloser with a nop Close() implementation. -func NopWriteCloser(w io.Writer) io.WriteCloser { - return &nopWriteCloser{w} -} - -// ReadCloserWithCallback adds a customizable callback to be called upon Close() of a supplied io.ReadCloser. -// Note that the callback will never be called more than once, after execution this will remove the func reference. -func ReadCloserWithCallback(rc io.ReadCloser, cb func()) io.ReadCloser { - return &callbackReadCloser{ - ReadCloser: rc, - callback: cb, - } -} - -// WriteCloserWithCallback adds a customizable callback to be called upon Close() of a supplied io.WriteCloser. -// Note that the callback will never be called more than once, after execution this will remove the func reference. -func WriteCloserWithCallback(wc io.WriteCloser, cb func()) io.WriteCloser { - return &callbackWriteCloser{ - WriteCloser: wc, - callback: cb, - } -} - -// nopReadCloser turns an io.Reader -> io.ReadCloser with a nop Close(). -type nopReadCloser struct{ io.Reader } - -func (r *nopReadCloser) Close() error { return nil } - -// nopWriteCloser turns an io.Writer -> io.WriteCloser with a nop Close(). -type nopWriteCloser struct{ io.Writer } - -func (w nopWriteCloser) Close() error { return nil } - -// callbackReadCloser allows adding our own custom callback to an io.ReadCloser. -type callbackReadCloser struct { - io.ReadCloser - callback func() -} - -func (c *callbackReadCloser) Close() error { - if c.callback != nil { - cb := c.callback - c.callback = nil - defer cb() - } - return c.ReadCloser.Close() -} - -// callbackWriteCloser allows adding our own custom callback to an io.WriteCloser. -type callbackWriteCloser struct { - io.WriteCloser - callback func() -} - -func (c *callbackWriteCloser) Close() error { - if c.callback != nil { - cb := c.callback - c.callback = nil - defer cb() - } - return c.WriteCloser.Close() -} diff --git a/vendor/modules.txt b/vendor/modules.txt @@ -24,7 +24,7 @@ codeberg.org/gruf/go-debug # codeberg.org/gruf/go-errors/v2 v2.0.2 ## explicit; go 1.16 codeberg.org/gruf/go-errors/v2 -# codeberg.org/gruf/go-fastcopy v1.1.1 +# codeberg.org/gruf/go-fastcopy v1.1.2 ## explicit; go 1.17 codeberg.org/gruf/go-fastcopy # codeberg.org/gruf/go-fastpath v1.0.3 @@ -36,6 +36,9 @@ codeberg.org/gruf/go-fastpath/v2 # codeberg.org/gruf/go-hashenc v1.0.2 ## explicit; go 1.16 codeberg.org/gruf/go-hashenc +# codeberg.org/gruf/go-iotools v0.0.0-20221224124424-3386841cb225 +## explicit; go 1.19 +codeberg.org/gruf/go-iotools # codeberg.org/gruf/go-kv v1.5.2 ## explicit; go 1.19 codeberg.org/gruf/go-kv @@ -49,7 +52,7 @@ codeberg.org/gruf/go-mangler # codeberg.org/gruf/go-maps v1.0.3 ## explicit; go 1.19 codeberg.org/gruf/go-maps -# codeberg.org/gruf/go-mutexes v1.1.4 +# codeberg.org/gruf/go-mutexes v1.1.5 ## explicit; go 1.14 codeberg.org/gruf/go-mutexes # codeberg.org/gruf/go-pools v1.1.0 @@ -61,7 +64,7 @@ codeberg.org/gruf/go-runners # codeberg.org/gruf/go-sched v1.2.0 ## explicit; go 1.19 codeberg.org/gruf/go-sched -# codeberg.org/gruf/go-store/v2 v2.0.10 +# codeberg.org/gruf/go-store/v2 v2.2.1 ## explicit; go 1.19 codeberg.org/gruf/go-store/v2/kv codeberg.org/gruf/go-store/v2/storage
