commit 6e947ff2662d0b9fc475c5b21e9f1cfb0bb308a0
parent 545b16ad35bf67ff79b9b32eb18475212affbeb3
Author: Sashanoraa <sasha@noraa.gay>
Date: Sun, 15 May 2022 08:52:46 -0400
[feature] Media cleanup endpoint (#560)
Adds an admin endpoint to trigger a remote media cleanup.
Fixed #348
Signed-off-by: Sashanoraa <sasha@noraa.gay>
Diffstat:
8 files changed, 321 insertions(+), 5 deletions(-)
diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml
@@ -2574,6 +2574,38 @@ paths:
summary: View domain block with the given ID.
tags:
- admin
+ /api/v1/admin/media_cleanup:
+ post:
+ consumes:
+ - application/json
+ - application/xml
+ - application/x-www-form-urlencoded
+ operationId: mediaCleanup
+ parameters:
+ - description: |-
+ Number of days of remote media to keep. Native values will be treated as 0.
+ If value is not specified, the value of media-remote-cache-days in the server config will be used.
+ format: int64
+ in: query
+ name: remote_cache_days
+ type: integer
+ x-go-name: RemoteCacheDays
+ produces:
+ - application/json
+ responses:
+ "200":
+ description: Echos the number of days requested. The cleanup is performed
+ asynchronously after the request completes.
+ "400":
+ description: bad request
+ "403":
+ description: forbidden
+ security:
+ - OAuth2 Bearer:
+ - admin
+ summary: Clean up remote media older than the specified number of days.
+ tags:
+ - admin
/api/v1/apps:
post:
consumes:
diff --git a/internal/api/client/admin/admin.go b/internal/api/client/admin/admin.go
@@ -41,6 +41,7 @@ const (
AccountsPathWithID = AccountsPath + "/:" + IDKey
// AccountsActionPath is used for taking action on a single account.
AccountsActionPath = AccountsPathWithID + "/action"
+ MediaCleanupPath = BasePath + "/media_cleanup"
// ExportQueryKey is for requesting a public export of some data.
ExportQueryKey = "export"
@@ -70,5 +71,6 @@ func (m *Module) Route(r router.Router) error {
r.AttachHandler(http.MethodGet, DomainBlocksPathWithID, m.DomainBlockGETHandler)
r.AttachHandler(http.MethodDelete, DomainBlocksPathWithID, m.DomainBlockDELETEHandler)
r.AttachHandler(http.MethodPost, AccountsActionPath, m.AccountActionPOSTHandler)
+ r.AttachHandler(http.MethodPost, MediaCleanupPath, m.MediaCleanupPOSTHandler)
return nil
}
diff --git a/internal/api/client/admin/mediacleanup.go b/internal/api/client/admin/mediacleanup.go
@@ -0,0 +1,110 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin
+
+import (
+ "fmt"
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+ "github.com/sirupsen/logrus"
+ "github.com/spf13/viper"
+ "github.com/superseriousbusiness/gotosocial/internal/api/model"
+ "github.com/superseriousbusiness/gotosocial/internal/config"
+ "github.com/superseriousbusiness/gotosocial/internal/oauth"
+)
+
+// MediaCleanupPOSTHandler swagger:operation POST /api/v1/admin/media_cleanup mediaCleanup
+//
+// Clean up remote media older than the specified number of days.
+//
+// ---
+// tags:
+// - admin
+//
+// consumes:
+// - application/json
+// - application/xml
+// - application/x-www-form-urlencoded
+//
+// produces:
+// - application/json
+//
+// security:
+// - OAuth2 Bearer:
+// - admin
+//
+// responses:
+// '200':
+// description: |-
+// Echos the number of days requested. The cleanup is performed asynchronously after the request completes.
+// '403':
+// description: forbidden
+// '400':
+// description: bad request
+func (m *Module) MediaCleanupPOSTHandler(c *gin.Context) {
+ l := logrus.WithFields(logrus.Fields{
+ "func": "MediaCleanupPOSTHandler",
+ "request_uri": c.Request.RequestURI,
+ "user_agent": c.Request.UserAgent(),
+ "origin_ip": c.ClientIP(),
+ })
+
+ // make sure we're authed...
+ authed, err := oauth.Authed(c, true, true, true, true)
+ if err != nil {
+ l.Debugf("couldn't auth: %s", err)
+ c.JSON(http.StatusUnauthorized, gin.H{"error": err.Error()})
+ return
+ }
+
+ // with an admin account
+ if !authed.User.Admin {
+ l.Debugf("user %s not an admin", authed.User.ID)
+ c.JSON(http.StatusForbidden, gin.H{"error": "not an admin"})
+ return
+ }
+
+ // extract the form from the request context
+ l.Tracef("parsing request form: %+v", c.Request.Form)
+ form := &model.MediaCleanupRequest{}
+ if err := c.ShouldBind(form); err != nil {
+ l.Debugf("error parsing form %+v: %s", c.Request.Form, err)
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("could not parse form: %s", err)})
+ return
+ }
+
+ var remoteCacheDays int
+ if form.RemoteCacheDays == nil {
+ remoteCacheDays = viper.GetInt(config.Keys.MediaRemoteCacheDays)
+ } else {
+ remoteCacheDays = *form.RemoteCacheDays
+ }
+ if remoteCacheDays < 0 {
+ remoteCacheDays = 0
+ }
+
+ if errWithCode := m.processor.AdminMediaRemotePrune(c.Request.Context(), remoteCacheDays); errWithCode != nil {
+ l.Debugf("error starting prune of remote media: %s", errWithCode.Error())
+ c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
+ return
+ }
+
+ c.JSON(http.StatusOK, remoteCacheDays)
+}
diff --git a/internal/api/client/admin/mediacleanup_test.go b/internal/api/client/admin/mediacleanup_test.go
@@ -0,0 +1,114 @@
+/*
+ GoToSocial
+ Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package admin_test
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/api/client/admin"
+)
+
+type MediaCleanupTestSuite struct {
+ AdminStandardTestSuite
+}
+
+func (suite *MediaCleanupTestSuite) TestMediaCleanup() {
+ testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"]
+ suite.True(testAttachment.Cached)
+
+ // set up the request
+ recorder := httptest.NewRecorder()
+ ctx := suite.newContext(recorder, http.MethodPost, []byte("{\"remote_cache_days\": 1}"), admin.EmojiPath, "application/json")
+
+ // call the handler
+ suite.adminModule.MediaCleanupPOSTHandler(ctx)
+
+ // we should have OK because our request was valid
+ suite.Equal(http.StatusOK, recorder.Code)
+
+ // Wait for async task to finish
+ time.Sleep(1 * time.Second)
+
+ // Get media we prunes
+ prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+ suite.NoError(err)
+
+ // the media should no longer be cached
+ suite.False(prunedAttachment.Cached)
+}
+
+func (suite *MediaCleanupTestSuite) TestMediaCleanupNoArg() {
+ testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"]
+ suite.True(testAttachment.Cached)
+ println("TIME: ", testAttachment.CreatedAt.String())
+
+ // set up the request
+ recorder := httptest.NewRecorder()
+ ctx := suite.newContext(recorder, http.MethodPost, []byte("{}"), admin.EmojiPath, "application/json")
+
+ // call the handler
+ suite.adminModule.MediaCleanupPOSTHandler(ctx)
+
+ // we should have OK because our request was valid
+ suite.Equal(http.StatusOK, recorder.Code)
+
+ // Wait for async task to finish
+ time.Sleep(1 * time.Second)
+
+ // Get media we prunes
+ prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+ suite.NoError(err)
+
+ // the media should no longer be cached
+ suite.True(prunedAttachment.Cached)
+}
+
+func (suite *MediaCleanupTestSuite) TestMediaCleanupNotOldEnough() {
+ testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_2"]
+ suite.True(testAttachment.Cached)
+
+ // set up the request
+ recorder := httptest.NewRecorder()
+ ctx := suite.newContext(recorder, http.MethodPost, []byte("{\"remote_cache_days\": 3}"), admin.EmojiPath, "application/json")
+
+ // call the handler
+ suite.adminModule.MediaCleanupPOSTHandler(ctx)
+
+ // we should have OK because our request was valid
+ suite.Equal(http.StatusOK, recorder.Code)
+
+ // Wait for async task to finish
+ time.Sleep(1 * time.Second)
+
+ // Get media we prunes
+ prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+ suite.NoError(err)
+
+ // the media should still be cached
+ suite.True(prunedAttachment.Cached)
+}
+
+func TestMediaCleanupTestSuite(t *testing.T) {
+ suite.Run(t, &MediaCleanupTestSuite{})
+}
diff --git a/internal/api/model/admin.go b/internal/api/model/admin.go
@@ -91,3 +91,12 @@ type AdminAccountActionRequest struct {
// ID of the account to be acted on.
TargetAccountID string `form:"-" json:"-" xml:"-"`
}
+
+// MediaCleanupRequest models admin media cleanup parameters
+//
+// swagger:parameters mediaCleanup
+type MediaCleanupRequest struct {
+ // Number of days of remote media to keep. Native values will be treated as 0.
+ // If value is not specified, the value of media-remote-cache-days in the server config will be used.
+ RemoteCacheDays *int `form:"remote_cache_days" json:"remote_cache_days" xml:"remote_cache_days"`
+}
diff --git a/internal/db/bundb/media_test.go b/internal/db/bundb/media_test.go
@@ -40,7 +40,7 @@ func (suite *MediaTestSuite) TestGetAttachmentByID() {
func (suite *MediaTestSuite) TestGetOlder() {
attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20)
suite.NoError(err)
- suite.Len(attachments, 1)
+ suite.Len(attachments, 2)
}
func TestMediaTestSuite(t *testing.T) {
diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go
@@ -39,7 +39,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() {
totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
suite.NoError(err)
- suite.Equal(1, totalPruned)
+ suite.Equal(2, totalPruned)
prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
suite.NoError(err)
@@ -51,7 +51,7 @@ func (suite *PruneRemoteTestSuite) TestPruneRemote() {
func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
suite.NoError(err)
- suite.Equal(1, totalPruned)
+ suite.Equal(2, totalPruned)
// final prune should prune nothing, since the first prune already happened
totalPrunedAgain, err := suite.manager.PruneRemote(context.Background(), 1)
@@ -65,7 +65,7 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
totalPruned, err := suite.manager.PruneRemote(ctx, 1)
suite.NoError(err)
- suite.Equal(1, totalPruned)
+ suite.Equal(2, totalPruned)
// media should no longer be stored
_, err = suite.storage.Get(testAttachment.File.Path)
@@ -118,7 +118,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
// Now attempt to prune remote for item with db entry no file
totalPruned, err := suite.manager.PruneRemote(ctx, 1)
suite.NoError(err)
- suite.Equal(1, totalPruned)
+ suite.Equal(2, totalPruned)
}
func TestPruneRemoteTestSuite(t *testing.T) {
diff --git a/testrig/testmodels.go b/testrig/testmodels.go
@@ -831,6 +831,55 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Header: false,
Cached: true,
},
+ "remote_account_1_status_1_attachment_2": {
+ ID: "01FVW7RXPQ8YJHTEXYPE7Q8ZY1",
+ StatusID: "01FVW7JHQFSFK166WWKR8CBA6M",
+ URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+ RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg",
+ CreatedAt: time.Now().Add(-48 * time.Hour),
+ UpdatedAt: time.Now().Add(-48 * time.Hour),
+ Type: gtsmodel.FileTypeImage,
+ FileMeta: gtsmodel.FileMeta{
+ Original: gtsmodel.Original{
+ Width: 472,
+ Height: 291,
+ Size: 137352,
+ Aspect: 1.6219931271477663,
+ },
+ Small: gtsmodel.Small{
+ Width: 472,
+ Height: 291,
+ Size: 137352,
+ Aspect: 1.6219931271477663,
+ },
+ Focus: gtsmodel.Focus{
+ X: 0,
+ Y: 0,
+ },
+ },
+ AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF",
+ Description: "tweet from thoughts of dog: i drank. all the water. in my bowl. earlier. but just now. i returned. to the same bowl. and it was. full again.. the bowl. is haunted",
+ ScheduledStatusID: "",
+ Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6",
+ Processing: 2,
+ File: gtsmodel.File{
+ Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+ ContentType: "image/jpeg",
+ FileSize: 19310,
+ UpdatedAt: time.Now().Add(-48 * time.Hour),
+ },
+ Thumbnail: gtsmodel.Thumbnail{
+ Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+ ContentType: "image/jpeg",
+ FileSize: 20395,
+ UpdatedAt: time.Now().Add(-48 * time.Hour),
+ URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
+ RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
+ },
+ Avatar: false,
+ Header: false,
+ Cached: true,
+ },
}
}
