robots.go (2021B)
1 // GoToSocial 2 // Copyright (C) GoToSocial Authors admin@gotosocial.org 3 // SPDX-License-Identifier: AGPL-3.0-or-later 4 // 5 // This program is free software: you can redistribute it and/or modify 6 // it under the terms of the GNU Affero General Public License as published by 7 // the Free Software Foundation, either version 3 of the License, or 8 // (at your option) any later version. 9 // 10 // This program is distributed in the hope that it will be useful, 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU Affero General Public License for more details. 14 // 15 // You should have received a copy of the GNU Affero General Public License 16 // along with this program. If not, see <http://www.gnu.org/licenses/>. 17 18 package web 19 20 import ( 21 "net/http" 22 23 "github.com/gin-gonic/gin" 24 ) 25 26 const ( 27 robotsPath = "/robots.txt" 28 robotsMetaAllowSome = "nofollow, noarchive, nositelinkssearchbox, max-image-preview:standard" // https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#robotsmeta 29 robotsTxt = `# GoToSocial robots.txt -- to edit, see internal/web/robots.go 30 # more info @ https://developers.google.com/search/docs/crawling-indexing/robots/intro 31 User-agent: * 32 Crawl-delay: 500 33 # api stuff 34 Disallow: /api/ 35 # auth/login stuff 36 Disallow: /auth/ 37 Disallow: /oauth/ 38 Disallow: /check_your_email 39 Disallow: /wait_for_approval 40 Disallow: /account_disabled 41 # well known stuff 42 Disallow: /.well-known/ 43 # files 44 Disallow: /fileserver/ 45 # s2s AP stuff 46 Disallow: /users/ 47 Disallow: /emoji/ 48 # panels 49 Disallow: /admin 50 Disallow: /user 51 Disallow: /settings/ 52 # domain blocklist 53 Disallow: /about/suspended` 54 ) 55 56 // robotsGETHandler returns a decent robots.txt that prevents crawling 57 // the api, auth pages, settings pages, etc. 58 // 59 // More granular robots meta tags are then applied for web pages 60 // depending on user preferences (see internal/web). 61 func (m *Module) robotsGETHandler(c *gin.Context) { 62 c.String(http.StatusOK, robotsTxt) 63 }