gtsocial-umbx

prune.go (12994B)

---

 // GoToSocial
 // Copyright (C) GoToSocial Authors admin@gotosocial.org
 // SPDX-License-Identifier: AGPL-3.0-or-later
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful,
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package media
 
 import (
 	"context"
 	"errors"
 	"fmt"
 	"time"
 
 	"codeberg.org/gruf/go-store/v2/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
 	"github.com/superseriousbusiness/gotosocial/internal/regexes"
 	"github.com/superseriousbusiness/gotosocial/internal/uris"
 )
 
 const (
 	selectPruneLimit          = 50 // Amount of media entries to select at a time from the db when pruning.
 	unusedLocalAttachmentDays = 3  // Number of days to keep local media in storage if not attached to a status.
 )
 
 // PruneAll runs all of the below pruning/uncacheing functions, and then cleans up any resulting
 // empty directories from the storage driver. It can be called as a shortcut for calling the below
 // pruning functions one by one.
 //
 // If blocking is true, then any errors encountered during the prune will be combined + returned to
 // the caller. If blocking is false, the prune is run in the background and errors are just logged
 // instead.
 func (m *Manager) PruneAll(ctx context.Context, mediaCacheRemoteDays int, blocking bool) error {
 	const dry = false
 
 	f := func(innerCtx context.Context) error {
 		errs := gtserror.MultiError{}
 
 		pruned, err := m.PruneUnusedLocal(innerCtx, dry)
 		if err != nil {
 			errs = append(errs, fmt.Sprintf("error pruning unused local media (%s)", err))
 		} else {
 			log.Infof(ctx, "pruned %d unused local media", pruned)
 		}
 
 		pruned, err = m.PruneUnusedRemote(innerCtx, dry)
 		if err != nil {
 			errs = append(errs, fmt.Sprintf("error pruning unused remote media: (%s)", err))
 		} else {
 			log.Infof(ctx, "pruned %d unused remote media", pruned)
 		}
 
 		pruned, err = m.UncacheRemote(innerCtx, mediaCacheRemoteDays, dry)
 		if err != nil {
 			errs = append(errs, fmt.Sprintf("error uncacheing remote media older than %d day(s): (%s)", mediaCacheRemoteDays, err))
 		} else {
 			log.Infof(ctx, "uncached %d remote media older than %d day(s)", pruned, mediaCacheRemoteDays)
 		}
 
 		pruned, err = m.PruneOrphaned(innerCtx, dry)
 		if err != nil {
 			errs = append(errs, fmt.Sprintf("error pruning orphaned media: (%s)", err))
 		} else {
 			log.Infof(ctx, "pruned %d orphaned media", pruned)
 		}
 
 		if err := m.state.Storage.Storage.Clean(innerCtx); err != nil {
 			errs = append(errs, fmt.Sprintf("error cleaning storage: (%s)", err))
 		} else {
 			log.Info(ctx, "cleaned storage")
 		}
 
 		return errs.Combine()
 	}
 
 	if blocking {
 		return f(ctx)
 	}
 
 	go func() {
 		if err := f(context.Background()); err != nil {
 			log.Error(ctx, err)
 		}
 	}()
 
 	return nil
 }
 
 // PruneUnusedRemote prunes unused/out of date headers and avatars cached on this instance.
 //
 // The returned int is the amount of media that was pruned by this function.
 func (m *Manager) PruneUnusedRemote(ctx context.Context, dry bool) (int, error) {
 	var (
 		totalPruned int
 		maxID       string
 		attachments []*gtsmodel.MediaAttachment
 		err         error
 	)
 
 	// We don't know in advance how many remote attachments will meet
 	// our criteria for being 'unused'. So a dry run in this case just
 	// means we iterate through as normal, but do nothing with each entry
 	// instead of removing it. Define this here so we don't do the 'if dry'
 	// check inside the loop a million times.
 	var f func(ctx context.Context, attachment *gtsmodel.MediaAttachment) error
 	if !dry {
 		f = m.deleteAttachment
 	} else {
 		f = func(_ context.Context, _ *gtsmodel.MediaAttachment) error {
 			return nil // noop
 		}
 	}
 
 	for attachments, err = m.state.DB.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) {
 		maxID = attachments[len(attachments)-1].ID // use the id of the last attachment in the slice as the next 'maxID' value
 
 		for _, attachment := range attachments {
 			// Retrieve owning account if possible.
 			var account *gtsmodel.Account
 			if accountID := attachment.AccountID; accountID != "" {
 				account, err = m.state.DB.GetAccountByID(ctx, attachment.AccountID)
 				if err != nil && !errors.Is(err, db.ErrNoEntries) {
 					// Only return on a real error.
 					return 0, fmt.Errorf("PruneUnusedRemote: error fetching account with id %s: %w", accountID, err)
 				}
 			}
 
 			// Prune each attachment that meets one of the following criteria:
 			// - Has no owning account in the database.
 			// - Is a header but isn't the owning account's current header.
 			// - Is an avatar but isn't the owning account's current avatar.
 			if account == nil ||
 				(*attachment.Header && attachment.ID != account.HeaderMediaAttachmentID) ||
 				(*attachment.Avatar && attachment.ID != account.AvatarMediaAttachmentID) {
 				if err := f(ctx, attachment); err != nil {
 					return totalPruned, err
 				}
 				totalPruned++
 			}
 		}
 	}
 
 	// Make sure we don't have a real error when we leave the loop.
 	if err != nil && !errors.Is(err, db.ErrNoEntries) {
 		return totalPruned, err
 	}
 
 	return totalPruned, nil
 }
 
 // PruneOrphaned prunes files that exist in storage but which do not have a corresponding
 // entry in the database.
 //
 // If dry is true, then nothing will be changed, only the amount that *would* be removed
 // is returned to the caller.
 func (m *Manager) PruneOrphaned(ctx context.Context, dry bool) (int, error) {
 	// Emojis are stored under the instance account, so we
 	// need the ID of the instance account for the next part.
 	instanceAccount, err := m.state.DB.GetInstanceAccount(ctx, "")
 	if err != nil {
 		return 0, fmt.Errorf("PruneOrphaned: error getting instance account: %w", err)
 	}
 
 	instanceAccountID := instanceAccount.ID
 
 	var orphanedKeys []string
 
 	// Keys in storage will look like the following format:
 	// `[ACCOUNT_ID]/[MEDIA_TYPE]/[MEDIA_SIZE]/[MEDIA_ID].[EXTENSION]`
 	// We can filter out keys we're not interested in by matching through a regex.
 	if err := m.state.Storage.WalkKeys(ctx, func(ctx context.Context, key string) error {
 		if !regexes.FilePath.MatchString(key) {
 			// This is not our expected key format.
 			return nil
 		}
 
 		// Check whether this storage entry is orphaned.
 		orphaned, err := m.orphaned(ctx, key, instanceAccountID)
 		if err != nil {
 			return fmt.Errorf("error checking orphaned status: %w", err)
 		}
 
 		if orphaned {
 			// Add this orphaned entry to list of keys.
 			orphanedKeys = append(orphanedKeys, key)
 		}
 
 		return nil
 	}); err != nil {
 		return 0, fmt.Errorf("PruneOrphaned: error walking keys: %w", err)
 	}
 
 	totalPruned := len(orphanedKeys)
 
 	if dry {
 		// Dry run: don't remove anything.
 		return totalPruned, nil
 	}
 
 	// This is not a drill! We have to delete stuff!
 	return m.removeFiles(ctx, orphanedKeys...)
 }
 
 func (m *Manager) orphaned(ctx context.Context, key string, instanceAccountID string) (bool, error) {
 	pathParts := regexes.FilePath.FindStringSubmatch(key)
 	if len(pathParts) != 6 {
 		// This doesn't match our expectations so
 		// it wasn't created by gts; ignore it.
 		return false, nil
 	}
 
 	var (
 		mediaType = pathParts[2]
 		mediaID   = pathParts[4]
 		orphaned  = false
 	)
 
 	// Look for keys in storage that we don't have an attachment for.
 	switch Type(mediaType) {
 	case TypeAttachment, TypeHeader, TypeAvatar:
 		if _, err := m.state.DB.GetAttachmentByID(ctx, mediaID); err != nil {
 			if !errors.Is(err, db.ErrNoEntries) {
 				return false, fmt.Errorf("error calling GetAttachmentByID: %w", err)
 			}
 			orphaned = true
 		}
 	case TypeEmoji:
 		// Look using the static URL for the emoji. Emoji images can change, so
 		// the MEDIA_ID part of the key for emojis will not necessarily correspond
 		// to the file that's currently being used as the emoji image.
 		staticURL := uris.GenerateURIForAttachment(instanceAccountID, string(TypeEmoji), string(SizeStatic), mediaID, mimePng)
 		if _, err := m.state.DB.GetEmojiByStaticURL(ctx, staticURL); err != nil {
 			if !errors.Is(err, db.ErrNoEntries) {
 				return false, fmt.Errorf("error calling GetEmojiByStaticURL: %w", err)
 			}
 			orphaned = true
 		}
 	}
 
 	return orphaned, nil
 }
 
 // UncacheRemote uncaches all remote media attachments older than the given amount of days.
 //
 // In this context, uncacheing means deleting media files from storage and marking the attachment
 // as cached=false in the database.
 //
 // If 'dry' is true, then only a dry run will be performed: nothing will actually be changed.
 //
 // The returned int is the amount of media that was/would be uncached by this function.
 func (m *Manager) UncacheRemote(ctx context.Context, olderThanDays int, dry bool) (int, error) {
 	if olderThanDays < 0 {
 		return 0, nil
 	}
 
 	olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(olderThanDays))
 
 	if dry {
 		// Dry run, just count eligible entries without removing them.
 		return m.state.DB.CountRemoteOlderThan(ctx, olderThan)
 	}
 
 	var (
 		totalPruned int
 		attachments []*gtsmodel.MediaAttachment
 		err         error
 	)
 
 	for attachments, err = m.state.DB.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) {
 		olderThan = attachments[len(attachments)-1].CreatedAt // use the created time of the last attachment in the slice as the next 'olderThan' value
 
 		for _, attachment := range attachments {
 			if err := m.uncacheAttachment(ctx, attachment); err != nil {
 				return totalPruned, err
 			}
 			totalPruned++
 		}
 	}
 
 	// Make sure we don't have a real error when we leave the loop.
 	if err != nil && !errors.Is(err, db.ErrNoEntries) {
 		return totalPruned, err
 	}
 
 	return totalPruned, nil
 }
 
 // PruneUnusedLocal prunes unused media attachments that were uploaded by
 // a user on this instance, but never actually attached to a status, or attached but
 // later detached.
 //
 // The returned int is the amount of media that was pruned by this function.
 func (m *Manager) PruneUnusedLocal(ctx context.Context, dry bool) (int, error) {
 	olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(unusedLocalAttachmentDays))
 
 	if dry {
 		// Dry run, just count eligible entries without removing them.
 		return m.state.DB.CountLocalUnattachedOlderThan(ctx, olderThan)
 	}
 
 	var (
 		totalPruned int
 		attachments []*gtsmodel.MediaAttachment
 		err         error
 	)
 
 	for attachments, err = m.state.DB.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.state.DB.GetLocalUnattachedOlderThan(ctx, olderThan, selectPruneLimit) {
 		olderThan = attachments[len(attachments)-1].CreatedAt // use the created time of the last attachment in the slice as the next 'olderThan' value
 
 		for _, attachment := range attachments {
 			if err := m.deleteAttachment(ctx, attachment); err != nil {
 				return totalPruned, err
 			}
 			totalPruned++
 		}
 	}
 
 	// Make sure we don't have a real error when we leave the loop.
 	if err != nil && !errors.Is(err, db.ErrNoEntries) {
 		return totalPruned, err
 	}
 
 	return totalPruned, nil
 }
 
 /*
 	Handy little helpers
 */
 
 func (m *Manager) deleteAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
 	if _, err := m.removeFiles(ctx, attachment.File.Path, attachment.Thumbnail.Path); err != nil {
 		return err
 	}
 
 	// Delete attachment completely.
 	return m.state.DB.DeleteAttachment(ctx, attachment.ID)
 }
 
 func (m *Manager) uncacheAttachment(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
 	if _, err := m.removeFiles(ctx, attachment.File.Path, attachment.Thumbnail.Path); err != nil {
 		return err
 	}
 
 	// Update attachment to reflect that we no longer have it cached.
 	attachment.Cached = func() *bool { i := false; return &i }()
 	return m.state.DB.UpdateAttachment(ctx, attachment, "cached")
 }
 
 func (m *Manager) removeFiles(ctx context.Context, keys ...string) (int, error) {
 	errs := make(gtserror.MultiError, 0, len(keys))
 
 	for _, key := range keys {
 		if err := m.state.Storage.Delete(ctx, key); err != nil && !errors.Is(err, storage.ErrNotFound) {
 			errs = append(errs, "storage error removing "+key+": "+err.Error())
 		}
 	}
 
 	return len(keys) - len(errs), errs.Combine()
 }