normalize_test.go (19571B)
1 // GoToSocial 2 // Copyright (C) GoToSocial Authors admin@gotosocial.org 3 // SPDX-License-Identifier: AGPL-3.0-or-later 4 // 5 // This program is free software: you can redistribute it and/or modify 6 // it under the terms of the GNU Affero General Public License as published by 7 // the Free Software Foundation, either version 3 of the License, or 8 // (at your option) any later version. 9 // 10 // This program is distributed in the hope that it will be useful, 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU Affero General Public License for more details. 14 // 15 // You should have received a copy of the GNU Affero General Public License 16 // along with this program. If not, see <http://www.gnu.org/licenses/>. 17 18 package ap_test 19 20 import ( 21 "context" 22 "encoding/json" 23 "testing" 24 25 "github.com/stretchr/testify/suite" 26 "github.com/superseriousbusiness/activity/streams" 27 "github.com/superseriousbusiness/activity/streams/vocab" 28 "github.com/superseriousbusiness/gotosocial/internal/ap" 29 "github.com/superseriousbusiness/gotosocial/testrig" 30 ) 31 32 type NormalizeTestSuite struct { 33 suite.Suite 34 } 35 36 func (suite *NormalizeTestSuite) jsonToType(rawJson string) (vocab.Type, map[string]interface{}) { 37 var raw map[string]interface{} 38 err := json.Unmarshal([]byte(rawJson), &raw) 39 if err != nil { 40 panic(err) 41 } 42 43 t, err := streams.ToType(context.Background(), raw) 44 if err != nil { 45 panic(err) 46 } 47 48 return t, raw 49 } 50 51 func (suite *NormalizeTestSuite) typeToJson(t vocab.Type) string { 52 m, err := ap.Serialize(t) 53 if err != nil { 54 suite.FailNow(err.Error()) 55 } 56 57 b, err := json.MarshalIndent(m, "", " ") 58 if err != nil { 59 suite.FailNow(err.Error()) 60 } 61 62 return string(b) 63 } 64 65 func (suite *NormalizeTestSuite) getStatusable() (vocab.ActivityStreamsNote, map[string]interface{}) { 66 t, raw := suite.jsonToType(`{ 67 "@context": [ 68 "https://www.w3.org/ns/activitystreams", 69 "https://example.org/schemas/litepub-0.1.jsonld", 70 { 71 "@language": "und" 72 } 73 ], 74 "actor": "https://example.org/users/someone", 75 "attachment": [], 76 "attributedTo": "https://example.org/users/someone", 77 "cc": [ 78 "https://example.org/users/someone/followers" 79 ], 80 "content": "UPDATE: As of this morning there are now more than 7 million Mastodon users, most from the <a class=\"hashtag\" data-tag=\"twittermigration\" href=\"https://example.org/tag/twittermigration\" rel=\"tag ugc\">#TwitterMigration</a>.<br><br>In fact, 100,000 new accounts have been created since last night.<br><br>Since last night's spike 8,000-12,000 new accounts are being created every hour.<br><br>Yesterday, I estimated that Mastodon would have 8 million users by the end of the week. That might happen a lot sooner if this trend continues.", 81 "context": "https://example.org/contexts/01GX0MSHPER1E0FT022Q209EJZ", 82 "conversation": "https://example.org/contexts/01GX0MSHPER1E0FT022Q209EJZ", 83 "id": "https://example.org/objects/01GX0MT2PA58JNSMK11MCS65YD", 84 "published": "2022-11-18T17:43:58.489995Z", 85 "replies": { 86 "items": [ 87 "https://example.org/objects/01GX0MV12MGEG3WF9SWB5K3KRJ" 88 ], 89 "type": "Collection" 90 }, 91 "repliesCount": 0, 92 "sensitive": null, 93 "source": "UPDATE: As of this morning there are now more than 7 million Mastodon users, most from the #TwitterMigration.\r\n\r\nIn fact, 100,000 new accounts have been created since last night.\r\n\r\nSince last night's spike 8,000-12,000 new accounts are being created every hour.\r\n\r\nYesterday, I estimated that Mastodon would have 8 million users by the end of the week. That might happen a lot sooner if this trend continues.", 94 "summary": "", 95 "tag": [ 96 { 97 "href": "https://example.org/tags/twittermigration", 98 "name": "#twittermigration", 99 "type": "Hashtag" 100 } 101 ], 102 "to": [ 103 "https://www.w3.org/ns/activitystreams#Public" 104 ], 105 "type": "Note" 106 }`) 107 108 return t.(vocab.ActivityStreamsNote), raw 109 } 110 111 func (suite *NormalizeTestSuite) getStatusableWithOneAttachment() (vocab.ActivityStreamsNote, map[string]interface{}) { 112 t, raw := suite.jsonToType(`{ 113 "@context": "https://www.w3.org/ns/activitystreams", 114 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 115 "type": "Note", 116 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ", 117 "attributedTo": "https://example.org/users/hourlycatbot", 118 "to": "https://www.w3.org/ns/activitystreams#Public", 119 "attachment": [ 120 { 121 "type": "Document", 122 "mediaType": "image/jpeg", 123 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", 124 "name": "DESCRIPTION: here's <<a>> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''" 125 } 126 ] 127 }`) 128 129 return t.(vocab.ActivityStreamsNote), raw 130 } 131 132 func (suite *NormalizeTestSuite) getStatusableWithOneAttachmentEmbedded() (vocab.ActivityStreamsNote, map[string]interface{}) { 133 t, raw := suite.jsonToType(`{ 134 "@context": "https://www.w3.org/ns/activitystreams", 135 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 136 "type": "Note", 137 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ", 138 "attributedTo": "https://example.org/users/hourlycatbot", 139 "to": "https://www.w3.org/ns/activitystreams#Public", 140 "attachment": { 141 "type": "Document", 142 "mediaType": "image/jpeg", 143 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", 144 "name": "DESCRIPTION: here's <<a>> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''" 145 } 146 }`) 147 148 return t.(vocab.ActivityStreamsNote), raw 149 } 150 151 func (suite *NormalizeTestSuite) getStatusableWithMultipleAttachments() (vocab.ActivityStreamsNote, map[string]interface{}) { 152 t, raw := suite.jsonToType(`{ 153 "@context": "https://www.w3.org/ns/activitystreams", 154 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 155 "type": "Note", 156 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ", 157 "attributedTo": "https://example.org/users/hourlycatbot", 158 "to": "https://www.w3.org/ns/activitystreams#Public", 159 "attachment": [ 160 { 161 "type": "Document", 162 "mediaType": "image/jpeg", 163 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", 164 "name": "DESCRIPTION: here's <<a>> picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''" 165 }, 166 { 167 "type": "Document", 168 "mediaType": "image/jpeg", 169 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", 170 "name": "hello: here's another #picture #of #a #cat, hope you like it!!!!!!!" 171 }, 172 { 173 "type": "Document", 174 "mediaType": "image/jpeg", 175 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 176 }, 177 { 178 "type": "Document", 179 "mediaType": "image/jpeg", 180 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg", 181 "name": "danger: #cute but will claw you :(" 182 } 183 ] 184 }`) 185 186 return t.(vocab.ActivityStreamsNote), raw 187 } 188 189 func (suite *NormalizeTestSuite) getStatusableWithWeirdSummaryAndName() (vocab.ActivityStreamsNote, map[string]interface{}) { 190 t, raw := suite.jsonToType(`{ 191 "@context": "https://www.w3.org/ns/activitystreams", 192 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 193 "type": "Note", 194 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ", 195 "attributedTo": "https://example.org/users/hourlycatbot", 196 "to": "https://www.w3.org/ns/activitystreams#Public", 197 "summary": "warning: #WEIRD #SUMMARY ;;;;a;;a;asv khop8273987(*^&^)", 198 "name": "WARNING: #WEIRD #nameEE ;;;;a;;a;asv khop8273987(*^&^)" 199 }`) 200 201 return t.(vocab.ActivityStreamsNote), raw 202 } 203 204 func (suite *NormalizeTestSuite) getAccountable() (vocab.ActivityStreamsPerson, map[string]interface{}) { 205 t, raw := suite.jsonToType(`{ 206 "@context": "https://www.w3.org/ns/activitystreams", 207 "id": "https://example.org/users/someone", 208 "summary": "about: I'm a #Barbie #girl in a #Barbie #world\nLife in plastic, it's fantastic\nYou can brush my hair, undress me everywhere\nImagination, life is your creation\nI'm a blonde bimbo girl\nIn a fantasy world\nDress me up, make it tight\nI'm your dolly\nYou're my doll, rock and roll\nFeel the glamour in pink\nKiss me here, touch me there\nHanky panky", 209 "type": "Person" 210 }`) 211 212 return t.(vocab.ActivityStreamsPerson), raw 213 } 214 215 func (suite *NormalizeTestSuite) TestNormalizeActivityObject() { 216 note, rawNote := suite.getStatusable() 217 suite.Equal(`update: As of this morning there are now more than 7 million Mastodon users, most from the <a class="hashtag" data-tag="twittermigration" href="https://example.org/tag/twittermigration" rel="tag ugc">#TwitterMigration%3C/a%3E.%3Cbr%3E%3Cbr%3EIn%20fact,%20100,000%20new%20accounts%20have%20been%20created%20since%20last%20night.%3Cbr%3E%3Cbr%3ESince%20last%20night&%2339;s%20spike%208,000-12,000%20new%20accounts%20are%20being%20created%20every%20hour.%3Cbr%3E%3Cbr%3EYesterday,%20I%20estimated%20that%20Mastodon%20would%20have%208%20million%20users%20by%20the%20end%20of%20the%20week.%20That%20might%20happen%20a%20lot%20sooner%20if%20this%20trend%20continues.`, ap.ExtractContent(note)) 218 219 create := testrig.WrapAPNoteInCreate( 220 testrig.URLMustParse("https://example.org/create_something"), 221 testrig.URLMustParse("https://example.org/users/someone"), 222 testrig.TimeMustParse("2022-11-18T17:43:58.489995Z"), 223 note, 224 ) 225 226 ap.NormalizeIncomingActivityObject(create, map[string]interface{}{"object": rawNote}) 227 suite.Equal(`UPDATE: As of this morning there are now more than 7 million Mastodon users, most from the <a class="hashtag" data-tag="twittermigration" href="https://example.org/tag/twittermigration" rel="tag ugc">#TwitterMigration</a>.<br><br>In fact, 100,000 new accounts have been created since last night.<br><br>Since last night's spike 8,000-12,000 new accounts are being created every hour.<br><br>Yesterday, I estimated that Mastodon would have 8 million users by the end of the week. That might happen a lot sooner if this trend continues.`, ap.ExtractContent(note)) 228 } 229 230 func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsOneAttachment() { 231 note, rawNote := suite.getStatusableWithOneAttachment() 232 233 // Without normalization, the 'name' field of 234 // the attachment(s) should be all jacked up. 235 suite.Equal(`{ 236 "@context": "https://www.w3.org/ns/activitystreams", 237 "attachment": { 238 "mediaType": "image/jpeg", 239 "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27", 240 "type": "Document", 241 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 242 }, 243 "attributedTo": "https://example.org/users/hourlycatbot", 244 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 245 "to": "https://www.w3.org/ns/activitystreams#Public", 246 "type": "Note", 247 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" 248 }`, suite.typeToJson(note)) 249 250 // Normalize it! 251 ap.NormalizeIncomingAttachments(note, rawNote) 252 253 // After normalization, the 'name' field of the 254 // attachment should no longer be all jacked up. 255 suite.Equal(`{ 256 "@context": "https://www.w3.org/ns/activitystreams", 257 "attachment": { 258 "mediaType": "image/jpeg", 259 "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''", 260 "type": "Document", 261 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 262 }, 263 "attributedTo": "https://example.org/users/hourlycatbot", 264 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 265 "to": "https://www.w3.org/ns/activitystreams#Public", 266 "type": "Note", 267 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" 268 }`, suite.typeToJson(note)) 269 } 270 271 func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsOneAttachmentEmbedded() { 272 note, rawNote := suite.getStatusableWithOneAttachmentEmbedded() 273 274 // Without normalization, the 'name' field of 275 // the attachment(s) should be all jacked up. 276 suite.Equal(`{ 277 "@context": "https://www.w3.org/ns/activitystreams", 278 "attachment": { 279 "mediaType": "image/jpeg", 280 "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27", 281 "type": "Document", 282 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 283 }, 284 "attributedTo": "https://example.org/users/hourlycatbot", 285 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 286 "to": "https://www.w3.org/ns/activitystreams#Public", 287 "type": "Note", 288 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" 289 }`, suite.typeToJson(note)) 290 291 // Normalize it! 292 ap.NormalizeIncomingAttachments(note, rawNote) 293 294 // After normalization, the 'name' field of the 295 // attachment should no longer be all jacked up. 296 suite.Equal(`{ 297 "@context": "https://www.w3.org/ns/activitystreams", 298 "attachment": { 299 "mediaType": "image/jpeg", 300 "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''", 301 "type": "Document", 302 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 303 }, 304 "attributedTo": "https://example.org/users/hourlycatbot", 305 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 306 "to": "https://www.w3.org/ns/activitystreams#Public", 307 "type": "Note", 308 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" 309 }`, suite.typeToJson(note)) 310 } 311 312 func (suite *NormalizeTestSuite) TestNormalizeStatusableAttachmentsMultipleAttachments() { 313 note, rawNote := suite.getStatusableWithMultipleAttachments() 314 315 // Without normalization, the 'name' field of 316 // the attachment(s) should be all jacked up. 317 suite.Equal(`{ 318 "@context": "https://www.w3.org/ns/activitystreams", 319 "attachment": [ 320 { 321 "mediaType": "image/jpeg", 322 "name": "description: here's \u003c\u003ca\u003e\u003e picture of a #cat,%20it%27s%20cute!%20here%27s%20some%20special%20characters:%20%22%22%20%5C%20weeee%27%27%27%27", 323 "type": "Document", 324 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 325 }, 326 { 327 "mediaType": "image/jpeg", 328 "name": "hello: here's another #picture%20%23of%20%23a%20%23cat,%20hope%20you%20like%20it!!!!!!!", 329 "type": "Document", 330 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 331 }, 332 { 333 "mediaType": "image/jpeg", 334 "type": "Document", 335 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 336 }, 337 { 338 "mediaType": "image/jpeg", 339 "name": "danger: #cute%20but%20will%20claw%20you%20:(", 340 "type": "Document", 341 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 342 } 343 ], 344 "attributedTo": "https://example.org/users/hourlycatbot", 345 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 346 "to": "https://www.w3.org/ns/activitystreams#Public", 347 "type": "Note", 348 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" 349 }`, suite.typeToJson(note)) 350 351 // Normalize it! 352 ap.NormalizeIncomingAttachments(note, rawNote) 353 354 // After normalization, the 'name' field of the 355 // attachment should no longer be all jacked up. 356 suite.Equal(`{ 357 "@context": "https://www.w3.org/ns/activitystreams", 358 "attachment": [ 359 { 360 "mediaType": "image/jpeg", 361 "name": "DESCRIPTION: here's \u003c\u003ca\u003e\u003e picture of a #cat, it's cute! here's some special characters: \"\" \\ weeee''''", 362 "type": "Document", 363 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 364 }, 365 { 366 "mediaType": "image/jpeg", 367 "name": "hello: here's another #picture #of #a #cat, hope you like it!!!!!!!", 368 "type": "Document", 369 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 370 }, 371 { 372 "mediaType": "image/jpeg", 373 "type": "Document", 374 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 375 }, 376 { 377 "mediaType": "image/jpeg", 378 "name": "danger: #cute but will claw you :(", 379 "type": "Document", 380 "url": "https://files.example.org/media_attachments/files/110/258/459/579/509/026/original/b65392ebe0fb04ef.jpeg" 381 } 382 ], 383 "attributedTo": "https://example.org/users/hourlycatbot", 384 "id": "https://example.org/users/hourlycatbot/statuses/01GYW48H311PZ78C5G856MGJJJ", 385 "to": "https://www.w3.org/ns/activitystreams#Public", 386 "type": "Note", 387 "url": "https://example.org/@hourlycatbot/01GYW48H311PZ78C5G856MGJJJ" 388 }`, suite.typeToJson(note)) 389 } 390 391 func (suite *NormalizeTestSuite) TestNormalizeAccountableSummary() { 392 accountable, rawAccount := suite.getAccountable() 393 suite.Equal(`about: I'm a #Barbie%20%23girl%20in%20a%20%23Barbie%20%23world%0ALife%20in%20plastic,%20it%27s%20fantastic%0AYou%20can%20brush%20my%20hair,%20undress%20me%20everywhere%0AImagination,%20life%20is%20your%20creation%0AI%27m%20a%20blonde%20bimbo%20girl%0AIn%20a%20fantasy%20world%0ADress%20me%20up,%20make%20it%20tight%0AI%27m%20your%20dolly%0AYou%27re%20my%20doll,%20rock%20and%20roll%0AFeel%20the%20glamour%20in%20pink%0AKiss%20me%20here,%20touch%20me%20there%0AHanky%20panky`, ap.ExtractSummary(accountable)) 394 395 ap.NormalizeIncomingSummary(accountable, rawAccount) 396 suite.Equal(`about: I'm a #Barbie #girl in a #Barbie #world 397 Life in plastic, it's fantastic 398 You can brush my hair, undress me everywhere 399 Imagination, life is your creation 400 I'm a blonde bimbo girl 401 In a fantasy world 402 Dress me up, make it tight 403 I'm your dolly 404 You're my doll, rock and roll 405 Feel the glamour in pink 406 Kiss me here, touch me there 407 Hanky panky`, ap.ExtractSummary(accountable)) 408 } 409 410 func (suite *NormalizeTestSuite) TestNormalizeStatusableSummary() { 411 statusable, rawAccount := suite.getStatusableWithWeirdSummaryAndName() 412 suite.Equal(`warning: #WEIRD%20%23SUMMARY%20;;;;a;;a;asv%20%20%20%20khop8273987(*%5E&%5E)`, ap.ExtractSummary(statusable)) 413 414 ap.NormalizeIncomingSummary(statusable, rawAccount) 415 suite.Equal(`warning: #WEIRD #SUMMARY ;;;;a;;a;asv khop8273987(*^&^)`, ap.ExtractSummary(statusable)) 416 } 417 418 func (suite *NormalizeTestSuite) TestNormalizeStatusableName() { 419 statusable, rawAccount := suite.getStatusableWithWeirdSummaryAndName() 420 suite.Equal(`warning: #WEIRD%20%23nameEE%20;;;;a;;a;asv%20%20%20%20khop8273987(*%5E&%5E)`, ap.ExtractName(statusable)) 421 422 ap.NormalizeIncomingName(statusable, rawAccount) 423 suite.Equal(`WARNING: #WEIRD #nameEE ;;;;a;;a;asv khop8273987(*^&^)`, ap.ExtractName(statusable)) 424 } 425 426 func TestNormalizeTestSuite(t *testing.T) { 427 suite.Run(t, new(NormalizeTestSuite)) 428 }