prune_test.go (12733B)
1 // GoToSocial 2 // Copyright (C) GoToSocial Authors admin@gotosocial.org 3 // SPDX-License-Identifier: AGPL-3.0-or-later 4 // 5 // This program is free software: you can redistribute it and/or modify 6 // it under the terms of the GNU Affero General Public License as published by 7 // the Free Software Foundation, either version 3 of the License, or 8 // (at your option) any later version. 9 // 10 // This program is distributed in the hope that it will be useful, 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU Affero General Public License for more details. 14 // 15 // You should have received a copy of the GNU Affero General Public License 16 // along with this program. If not, see <http://www.gnu.org/licenses/>. 17 18 package media_test 19 20 import ( 21 "bytes" 22 "context" 23 "io" 24 "os" 25 "testing" 26 27 "codeberg.org/gruf/go-store/v2/storage" 28 "github.com/stretchr/testify/suite" 29 "github.com/superseriousbusiness/gotosocial/internal/db" 30 gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" 31 ) 32 33 type PruneTestSuite struct { 34 MediaStandardTestSuite 35 } 36 37 func (suite *PruneTestSuite) TestPruneOrphanedDry() { 38 // add a big orphan panda to store 39 b, err := os.ReadFile("./test/big-panda.gif") 40 if err != nil { 41 suite.FailNow(err.Error()) 42 } 43 44 pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" 45 if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil { 46 suite.FailNow(err.Error()) 47 } 48 49 // dry run should show up 1 orphaned panda 50 totalPruned, err := suite.manager.PruneOrphaned(context.Background(), true) 51 suite.NoError(err) 52 suite.Equal(1, totalPruned) 53 54 // panda should still be in storage 55 hasKey, err := suite.storage.Has(context.Background(), pandaPath) 56 suite.NoError(err) 57 suite.True(hasKey) 58 } 59 60 func (suite *PruneTestSuite) TestPruneOrphanedMoist() { 61 // add a big orphan panda to store 62 b, err := os.ReadFile("./test/big-panda.gif") 63 if err != nil { 64 suite.FailNow(err.Error()) 65 } 66 67 pandaPath := "01GJQJ1YD9QCHCE12GG0EYHVNW/attachment/original/01GJQJ2AYM1VKSRW96YVAJ3NK3.gif" 68 if _, err := suite.storage.Put(context.Background(), pandaPath, b); err != nil { 69 suite.FailNow(err.Error()) 70 } 71 72 // should show up 1 orphaned panda 73 totalPruned, err := suite.manager.PruneOrphaned(context.Background(), false) 74 suite.NoError(err) 75 suite.Equal(1, totalPruned) 76 77 // panda should no longer be in storage 78 hasKey, err := suite.storage.Has(context.Background(), pandaPath) 79 suite.NoError(err) 80 suite.False(hasKey) 81 } 82 83 func (suite *PruneTestSuite) TestPruneUnusedLocal() { 84 testAttachment := suite.testAttachments["local_account_1_unattached_1"] 85 suite.True(*testAttachment.Cached) 86 87 totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false) 88 suite.NoError(err) 89 suite.Equal(1, totalPruned) 90 91 _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) 92 suite.ErrorIs(err, db.ErrNoEntries) 93 } 94 95 func (suite *PruneTestSuite) TestPruneUnusedLocalDry() { 96 testAttachment := suite.testAttachments["local_account_1_unattached_1"] 97 suite.True(*testAttachment.Cached) 98 99 totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), true) 100 suite.NoError(err) 101 suite.Equal(1, totalPruned) 102 103 _, err = suite.db.GetAttachmentByID(context.Background(), testAttachment.ID) 104 suite.NoError(err) 105 } 106 107 func (suite *PruneTestSuite) TestPruneRemoteTwice() { 108 totalPruned, err := suite.manager.PruneUnusedLocal(context.Background(), false) 109 suite.NoError(err) 110 suite.Equal(1, totalPruned) 111 112 // final prune should prune nothing, since the first prune already happened 113 totalPrunedAgain, err := suite.manager.PruneUnusedLocal(context.Background(), false) 114 suite.NoError(err) 115 suite.Equal(0, totalPrunedAgain) 116 } 117 118 func (suite *PruneTestSuite) TestPruneOneNonExistent() { 119 ctx := context.Background() 120 testAttachment := suite.testAttachments["local_account_1_unattached_1"] 121 122 // Delete this attachment cached on disk 123 media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID) 124 suite.NoError(err) 125 suite.True(*media.Cached) 126 err = suite.storage.Delete(ctx, media.File.Path) 127 suite.NoError(err) 128 129 // Now attempt to prune for item with db entry no file 130 totalPruned, err := suite.manager.PruneUnusedLocal(ctx, false) 131 suite.NoError(err) 132 suite.Equal(1, totalPruned) 133 } 134 135 func (suite *PruneTestSuite) TestPruneUnusedRemote() { 136 ctx := context.Background() 137 138 // start by clearing zork's avatar + header 139 zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] 140 zorkOldHeader := suite.testAttachments["local_account_1_avatar"] 141 zork := suite.testAccounts["local_account_1"] 142 zork.AvatarMediaAttachmentID = "" 143 zork.HeaderMediaAttachmentID = "" 144 if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { 145 panic(err) 146 } 147 148 totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) 149 suite.NoError(err) 150 suite.Equal(2, totalPruned) 151 152 // media should no longer be stored 153 _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) 154 suite.ErrorIs(err, storage.ErrNotFound) 155 _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) 156 suite.ErrorIs(err, storage.ErrNotFound) 157 _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) 158 suite.ErrorIs(err, storage.ErrNotFound) 159 _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) 160 suite.ErrorIs(err, storage.ErrNotFound) 161 162 // attachments should no longer be in the db 163 _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) 164 suite.ErrorIs(err, db.ErrNoEntries) 165 _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) 166 suite.ErrorIs(err, db.ErrNoEntries) 167 } 168 169 func (suite *PruneTestSuite) TestPruneUnusedRemoteTwice() { 170 ctx := context.Background() 171 172 // start by clearing zork's avatar + header 173 zork := suite.testAccounts["local_account_1"] 174 zork.AvatarMediaAttachmentID = "" 175 zork.HeaderMediaAttachmentID = "" 176 if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { 177 panic(err) 178 } 179 180 totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) 181 suite.NoError(err) 182 suite.Equal(2, totalPruned) 183 184 // final prune should prune nothing, since the first prune already happened 185 totalPruned, err = suite.manager.PruneUnusedRemote(ctx, false) 186 suite.NoError(err) 187 suite.Equal(0, totalPruned) 188 } 189 190 func (suite *PruneTestSuite) TestPruneUnusedRemoteMultipleAccounts() { 191 ctx := context.Background() 192 193 // start by clearing zork's avatar + header 194 zorkOldAvatar := suite.testAttachments["local_account_1_avatar"] 195 zorkOldHeader := suite.testAttachments["local_account_1_avatar"] 196 zork := suite.testAccounts["local_account_1"] 197 zork.AvatarMediaAttachmentID = "" 198 zork.HeaderMediaAttachmentID = "" 199 if err := suite.db.UpdateByID(ctx, zork, zork.ID, "avatar_media_attachment_id", "header_media_attachment_id"); err != nil { 200 panic(err) 201 } 202 203 // set zork's unused header as belonging to turtle 204 turtle := suite.testAccounts["local_account_1"] 205 zorkOldHeader.AccountID = turtle.ID 206 if err := suite.db.UpdateByID(ctx, zorkOldHeader, zorkOldHeader.ID, "account_id"); err != nil { 207 panic(err) 208 } 209 210 totalPruned, err := suite.manager.PruneUnusedRemote(ctx, false) 211 suite.NoError(err) 212 suite.Equal(2, totalPruned) 213 214 // media should no longer be stored 215 _, err = suite.storage.Get(ctx, zorkOldAvatar.File.Path) 216 suite.ErrorIs(err, storage.ErrNotFound) 217 _, err = suite.storage.Get(ctx, zorkOldAvatar.Thumbnail.Path) 218 suite.ErrorIs(err, storage.ErrNotFound) 219 _, err = suite.storage.Get(ctx, zorkOldHeader.File.Path) 220 suite.ErrorIs(err, storage.ErrNotFound) 221 _, err = suite.storage.Get(ctx, zorkOldHeader.Thumbnail.Path) 222 suite.ErrorIs(err, storage.ErrNotFound) 223 224 // attachments should no longer be in the db 225 _, err = suite.db.GetAttachmentByID(ctx, zorkOldAvatar.ID) 226 suite.ErrorIs(err, db.ErrNoEntries) 227 _, err = suite.db.GetAttachmentByID(ctx, zorkOldHeader.ID) 228 suite.ErrorIs(err, db.ErrNoEntries) 229 } 230 231 func (suite *PruneTestSuite) TestUncacheRemote() { 232 testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] 233 suite.True(*testStatusAttachment.Cached) 234 235 testHeader := suite.testAttachments["remote_account_3_header"] 236 suite.True(*testHeader.Cached) 237 238 totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, false) 239 suite.NoError(err) 240 suite.Equal(2, totalUncached) 241 242 uncachedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID) 243 suite.NoError(err) 244 suite.False(*uncachedAttachment.Cached) 245 246 uncachedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID) 247 suite.NoError(err) 248 suite.False(*uncachedAttachment.Cached) 249 } 250 251 func (suite *PruneTestSuite) TestUncacheRemoteDry() { 252 testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] 253 suite.True(*testStatusAttachment.Cached) 254 255 testHeader := suite.testAttachments["remote_account_3_header"] 256 suite.True(*testHeader.Cached) 257 258 totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, true) 259 suite.NoError(err) 260 suite.Equal(2, totalUncached) 261 262 uncachedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID) 263 suite.NoError(err) 264 suite.True(*uncachedAttachment.Cached) 265 266 uncachedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID) 267 suite.NoError(err) 268 suite.True(*uncachedAttachment.Cached) 269 } 270 271 func (suite *PruneTestSuite) TestUncacheRemoteTwice() { 272 totalUncached, err := suite.manager.UncacheRemote(context.Background(), 1, false) 273 suite.NoError(err) 274 suite.Equal(2, totalUncached) 275 276 // final uncache should uncache nothing, since the first uncache already happened 277 totalUncachedAgain, err := suite.manager.UncacheRemote(context.Background(), 1, false) 278 suite.NoError(err) 279 suite.Equal(0, totalUncachedAgain) 280 } 281 282 func (suite *PruneTestSuite) TestUncacheAndRecache() { 283 ctx := context.Background() 284 testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] 285 testHeader := suite.testAttachments["remote_account_3_header"] 286 287 totalUncached, err := suite.manager.UncacheRemote(ctx, 1, false) 288 suite.NoError(err) 289 suite.Equal(2, totalUncached) 290 291 // media should no longer be stored 292 _, err = suite.storage.Get(ctx, testStatusAttachment.File.Path) 293 suite.ErrorIs(err, storage.ErrNotFound) 294 _, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path) 295 suite.ErrorIs(err, storage.ErrNotFound) 296 _, err = suite.storage.Get(ctx, testHeader.File.Path) 297 suite.ErrorIs(err, storage.ErrNotFound) 298 _, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path) 299 suite.ErrorIs(err, storage.ErrNotFound) 300 301 // now recache the image.... 302 data := func(_ context.Context) (io.ReadCloser, int64, error) { 303 // load bytes from a test image 304 b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpg") 305 if err != nil { 306 panic(err) 307 } 308 return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil 309 } 310 311 for _, original := range []*gtsmodel.MediaAttachment{ 312 testStatusAttachment, 313 testHeader, 314 } { 315 processingRecache, err := suite.manager.PreProcessMediaRecache(ctx, data, original.ID) 316 suite.NoError(err) 317 318 // synchronously load the recached attachment 319 recachedAttachment, err := processingRecache.LoadAttachment(ctx) 320 suite.NoError(err) 321 suite.NotNil(recachedAttachment) 322 323 // recachedAttachment should be basically the same as the old attachment 324 suite.True(*recachedAttachment.Cached) 325 suite.Equal(original.ID, recachedAttachment.ID) 326 suite.Equal(original.File.Path, recachedAttachment.File.Path) // file should be stored in the same place 327 suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail 328 suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same 329 330 // recached files should be back in storage 331 _, err = suite.storage.Get(ctx, recachedAttachment.File.Path) 332 suite.NoError(err) 333 _, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path) 334 suite.NoError(err) 335 } 336 } 337 338 func (suite *PruneTestSuite) TestUncacheOneNonExistent() { 339 ctx := context.Background() 340 testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"] 341 342 // Delete this attachment cached on disk 343 media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID) 344 suite.NoError(err) 345 suite.True(*media.Cached) 346 err = suite.storage.Delete(ctx, media.File.Path) 347 suite.NoError(err) 348 349 // Now attempt to uncache remote for item with db entry no file 350 totalUncached, err := suite.manager.UncacheRemote(ctx, 1, false) 351 suite.NoError(err) 352 suite.Equal(2, totalUncached) 353 } 354 355 func TestPruneOrphanedTestSuite(t *testing.T) { 356 suite.Run(t, &PruneTestSuite{}) 357 }