diff --git a/packages/backend/src/core/AvatarDecorationService.ts b/packages/backend/src/core/AvatarDecorationService.ts index 21e31d79a..fa3f63677 100644 --- a/packages/backend/src/core/AvatarDecorationService.ts +++ b/packages/backend/src/core/AvatarDecorationService.ts @@ -29,7 +29,7 @@ export class AvatarDecorationService implements OnApplicationShutdown { private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, ) { - this.cache = new MemorySingleCache(1000 * 60 * 30); + this.cache = new MemorySingleCache(1000 * 60 * 30); // 30s this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/core/CacheService.ts b/packages/backend/src/core/CacheService.ts index d008e7ec5..6725ebe75 100644 --- a/packages/backend/src/core/CacheService.ts +++ b/packages/backend/src/core/CacheService.ts @@ -56,10 +56,10 @@ export class CacheService implements OnApplicationShutdown { ) { //this.onMessage = this.onMessage.bind(this); - this.userByIdCache = new MemoryKVCache(Infinity); - this.localUserByNativeTokenCache = new MemoryKVCache(Infinity); - this.localUserByIdCache = new MemoryKVCache(Infinity); - this.uriPersonCache = new MemoryKVCache(Infinity); + this.userByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.localUserByNativeTokenCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.localUserByIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m + this.uriPersonCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.userProfileCache = new RedisKVCache(this.redisClient, 'userProfile', { lifetime: 1000 * 60 * 30, // 30m @@ -135,14 +135,14 @@ export class CacheService implements OnApplicationShutdown { if (user == null) { this.userByIdCache.delete(body.id); this.localUserByIdCache.delete(body.id); - for (const [k, v] of this.uriPersonCache.cache.entries()) { + for (const [k, v] of this.uriPersonCache.entries) { if (v.value?.id === body.id) { this.uriPersonCache.delete(k); } } } else { this.userByIdCache.set(user.id, user); - for (const [k, v] of this.uriPersonCache.cache.entries()) { + for (const [k, v] of this.uriPersonCache.entries) { if (v.value?.id === user.id) { this.uriPersonCache.set(k, user); } diff --git a/packages/backend/src/core/CustomEmojiService.ts b/packages/backend/src/core/CustomEmojiService.ts index feca0fe5e..27b85b10b 100644 --- a/packages/backend/src/core/CustomEmojiService.ts +++ b/packages/backend/src/core/CustomEmojiService.ts @@ -24,7 +24,7 @@ const parseEmojiStrRegexp = /^([-\w]+)(?:@([\w.-]+))?$/; @Injectable() export class CustomEmojiService implements OnApplicationShutdown { - private cache: MemoryKVCache; + private emojisCache: MemoryKVCache; public localEmojisCache: RedisSingleCache>; constructor( @@ -40,7 +40,7 @@ export class CustomEmojiService implements OnApplicationShutdown { private moderationLogService: ModerationLogService, private globalEventService: GlobalEventService, ) { - this.cache = new MemoryKVCache(1000 * 60 * 60 * 12); + this.emojisCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h this.localEmojisCache = new RedisSingleCache>(this.redisClient, 'localEmojis', { lifetime: 1000 * 60 * 30, // 30m @@ -346,7 +346,7 @@ export class CustomEmojiService implements OnApplicationShutdown { host, })) ?? null; - const emoji = await this.cache.fetch(`${name} ${host}`, queryOrNull); + const emoji = await this.emojisCache.fetch(`${name} ${host}`, queryOrNull); if (emoji == null) return null; return emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ） @@ -372,7 +372,7 @@ export class CustomEmojiService implements OnApplicationShutdown { */ @bindThis public async prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise { - const notCachedEmojis = emojis.filter(emoji => this.cache.get(`${emoji.name} ${emoji.host}`) == null); + const notCachedEmojis = emojis.filter(emoji => this.emojisCache.get(`${emoji.name} ${emoji.host}`) == null); const emojisQuery: any[] = []; const hosts = new Set(notCachedEmojis.map(e => e.host)); for (const host of hosts) { @@ -387,7 +387,7 @@ export class CustomEmojiService implements OnApplicationShutdown { select: ['name', 'host', 'originalUrl', 'publicUrl'], }) : []; for (const emoji of _emojis) { - this.cache.set(`${emoji.name} ${emoji.host}`, emoji); + this.emojisCache.set(`${emoji.name} ${emoji.host}`, emoji); } } @@ -412,7 +412,7 @@ export class CustomEmojiService implements OnApplicationShutdown { @bindThis public dispose(): void { - this.cache.dispose(); + this.emojisCache.dispose(); } @bindThis diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 21ae798f9..7543534d9 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -6,7 +6,6 @@ import * as fs from 'node:fs'; import * as stream from 'node:stream/promises'; import { Inject, Injectable } from '@nestjs/common'; -import ipaddr from 'ipaddr.js'; import chalk from 'chalk'; import got, * as Got from 'got'; import { parse } from 'content-disposition'; @@ -70,13 +69,6 @@ export class DownloadService { }, enableUnixSockets: false, }).on('response', (res: Got.Response) => { - if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) { - if (this.isPrivateIp(res.ip)) { - this.logger.warn(`Blocked address: ${res.ip}`); - req.destroy(); - } - } - const contentLength = res.headers['content-length']; if (contentLength != null) { const size = Number(contentLength); @@ -139,18 +131,4 @@ export class DownloadService { cleanup(); } } - - @bindThis - private isPrivateIp(ip: string): boolean { - const parsedIp = ipaddr.parse(ip); - - for (const net of this.config.allowedPrivateNetworks ?? []) { - const cidr = ipaddr.parseCIDR(net); - if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) { - return false; - } - } - - return parsedIp.range() !== 'unicast'; - } } diff --git a/packages/backend/src/core/EmailService.ts b/packages/backend/src/core/EmailService.ts index 1852536a8..832c2d55e 100644 --- a/packages/backend/src/core/EmailService.ts +++ b/packages/backend/src/core/EmailService.ts @@ -330,6 +330,7 @@ export class EmailService { Accept: 'application/json', Authorization: truemailAuthKey, }, + isLocalAddressAllowed: true, }); const json = (await res.json()) as { diff --git a/packages/backend/src/core/HttpRequestService.ts b/packages/backend/src/core/HttpRequestService.ts index 3eb2a8089..e3640216f 100644 --- a/packages/backend/src/core/HttpRequestService.ts +++ b/packages/backend/src/core/HttpRequestService.ts @@ -6,6 +6,7 @@ import * as http from 'node:http'; import * as https from 'node:https'; import * as net from 'node:net'; +import ipaddr from 'ipaddr.js'; import CacheableLookup from 'cacheable-lookup'; import fetch from 'node-fetch'; import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'; @@ -15,6 +16,7 @@ import type { Config } from '@/config.js'; import { StatusError } from '@/misc/status-error.js'; import { bindThis } from '@/decorators.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; +import { assertActivityMatchesUrls } from '@/core/activitypub/misc/check-against-url.js'; import type { IObject } from '@/core/activitypub/type.js'; import type { Response } from 'node-fetch'; import type { URL } from 'node:url'; @@ -24,8 +26,102 @@ export type HttpRequestSendOptions = { validators?: ((res: Response) => void)[]; }; +declare module 'node:http' { + interface Agent { + createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket; + } +} + +class HttpRequestServiceAgent extends http.Agent { + constructor( + private config: Config, + options?: http.AgentOptions, + ) { + super(options); + } + + @bindThis + public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { + const socket = super.createConnection(options, callback) + .on('connect', () => { + const address = socket.remoteAddress; + if (process.env.NODE_ENV === 'production') { + if (address && ipaddr.isValid(address)) { + if (this.isPrivateIp(address)) { + socket.destroy(new Error(`Blocked address: ${address}`)); + } + } + } + }); + return socket; + } + + @bindThis + private isPrivateIp(ip: string): boolean { + const parsedIp = ipaddr.parse(ip); + + for (const net of this.config.allowedPrivateNetworks ?? []) { + const cidr = ipaddr.parseCIDR(net); + if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) { + return false; + } + } + + return parsedIp.range() !== 'unicast'; + } +} + +class HttpsRequestServiceAgent extends https.Agent { + constructor( + private config: Config, + options?: https.AgentOptions, + ) { + super(options); + } + + @bindThis + public createConnection(options: net.NetConnectOpts, callback?: (err: unknown, stream: net.Socket) => void): net.Socket { + const socket = super.createConnection(options, callback) + .on('connect', () => { + const address = socket.remoteAddress; + if (process.env.NODE_ENV === 'production') { + if (address && ipaddr.isValid(address)) { + if (this.isPrivateIp(address)) { + socket.destroy(new Error(`Blocked address: ${address}`)); + } + } + } + }); + return socket; + } + + @bindThis + private isPrivateIp(ip: string): boolean { + const parsedIp = ipaddr.parse(ip); + + for (const net of this.config.allowedPrivateNetworks ?? []) { + const cidr = ipaddr.parseCIDR(net); + if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) { + return false; + } + } + + return parsedIp.range() !== 'unicast'; + } +} + @Injectable() export class HttpRequestService { + /** + * Get http non-proxy agent (without local address filtering) + */ + private httpNative: http.Agent; + + /** + * Get https non-proxy agent (without local address filtering) + */ + private httpsNative: https.Agent; + /** * Get http non-proxy agent */ @@ -56,19 +152,20 @@ export class HttpRequestService { lookup: false, // nativeのdns.lookupにfallbackしない }); - this.http = new http.Agent({ + const agentOption = { keepAlive: true, keepAliveMsecs: 30 * 1000, lookup: cache.lookup as unknown as net.LookupFunction, localAddress: config.outgoingAddress, - }); + }; - this.https = new https.Agent({ - keepAlive: true, - keepAliveMsecs: 30 * 1000, - lookup: cache.lookup as unknown as net.LookupFunction, - localAddress: config.outgoingAddress, - }); + this.httpNative = new http.Agent(agentOption); + + this.httpsNative = new https.Agent(agentOption); + + this.http = new HttpRequestServiceAgent(config, agentOption); + + this.https = new HttpsRequestServiceAgent(config, agentOption); const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128); @@ -103,16 +200,22 @@ export class HttpRequestService { * @param bypassProxy Allways bypass proxy */ @bindThis - public getAgentByUrl(url: URL, bypassProxy = false): http.Agent | https.Agent { + public getAgentByUrl(url: URL, bypassProxy = false, isLocalAddressAllowed = false): http.Agent | https.Agent { if (bypassProxy || (this.config.proxyBypassHosts ?? []).includes(url.hostname)) { + if (isLocalAddressAllowed) { + return url.protocol === 'http:' ? this.httpNative : this.httpsNative; + } return url.protocol === 'http:' ? this.http : this.https; } else { + if (isLocalAddressAllowed && (!this.config.proxy)) { + return url.protocol === 'http:' ? this.httpNative : this.httpsNative; + } return url.protocol === 'http:' ? this.httpAgent : this.httpsAgent; } } @bindThis - public async getActivityJson(url: string): Promise { + public async getActivityJson(url: string, isLocalAddressAllowed = false): Promise { const res = await this.send(url, { method: 'GET', headers: { @@ -120,16 +223,22 @@ export class HttpRequestService { }, timeout: 5000, size: 1024 * 256, + isLocalAddressAllowed: isLocalAddressAllowed, }, { throwErrorWhenResponseNotOk: true, validators: [validateContentTypeSetAsActivityPub], }); - return await res.json() as IObject; + const finalUrl = res.url; // redirects may have been involved + const activity = await res.json() as IObject; + + assertActivityMatchesUrls(activity, [finalUrl]); + + return activity; } @bindThis - public async getJson(url: string, accept = 'application/json, */*', headers?: Record): Promise { + public async getJson(url: string, accept = 'application/json, */*', headers?: Record, isLocalAddressAllowed = false): Promise { const res = await this.send(url, { method: 'GET', headers: Object.assign({ @@ -137,19 +246,21 @@ export class HttpRequestService { }, headers ?? {}), timeout: 5000, size: 1024 * 256, + isLocalAddressAllowed: isLocalAddressAllowed, }); return await res.json() as T; } @bindThis - public async getHtml(url: string, accept = 'text/html, */*', headers?: Record): Promise { + public async getHtml(url: string, accept = 'text/html, */*', headers?: Record, isLocalAddressAllowed = false): Promise { const res = await this.send(url, { method: 'GET', headers: Object.assign({ Accept: accept, }, headers ?? {}), timeout: 5000, + isLocalAddressAllowed: isLocalAddressAllowed, }); return await res.text(); @@ -164,6 +275,7 @@ export class HttpRequestService { headers?: Record, timeout?: number, size?: number, + isLocalAddressAllowed?: boolean, } = {}, extra: HttpRequestSendOptions = { throwErrorWhenResponseNotOk: true, @@ -178,6 +290,7 @@ export class HttpRequestService { }, timeout); const bearcaps = url.startsWith('bear:?') ? this.parseBearcaps(url) : undefined; + const isLocalAddressAllowed = args.isLocalAddressAllowed ?? false; const res = await fetch(bearcaps?.url ?? url, { method: args.method ?? 'GET', @@ -188,7 +301,7 @@ export class HttpRequestService { }, body: args.body, size: args.size ?? 10 * 1024 * 1024, - agent: (url) => this.getAgentByUrl(url), + agent: (url) => this.getAgentByUrl(url, false, isLocalAddressAllowed), signal: controller.signal, }); diff --git a/packages/backend/src/core/RelayService.ts b/packages/backend/src/core/RelayService.ts index e9dc9b57a..91857dc68 100644 --- a/packages/backend/src/core/RelayService.ts +++ b/packages/backend/src/core/RelayService.ts @@ -35,7 +35,7 @@ export class RelayService { private createSystemUserService: CreateSystemUserService, private apRendererService: ApRendererService, ) { - this.relaysCache = new MemorySingleCache(1000 * 60 * 10); + this.relaysCache = new MemorySingleCache(1000 * 60 * 10); // 10m } @bindThis diff --git a/packages/backend/src/core/RemoteUserResolveService.ts b/packages/backend/src/core/RemoteUserResolveService.ts index 38f45a03a..c7f5effd2 100644 --- a/packages/backend/src/core/RemoteUserResolveService.ts +++ b/packages/backend/src/core/RemoteUserResolveService.ts @@ -56,7 +56,7 @@ export class RemoteUserResolveService { host = this.utilityService.toPuny(host); - if (this.config.host === host) { + if (host === this.utilityService.toPuny(this.config.host)) { this.logger.info(`return local user: ${usernameLower}`); return await this.usersRepository.findOneBy({ usernameLower, host: IsNull() }).then(u => { if (u == null) { diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 45e8b641c..9089480f9 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -149,10 +149,8 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { private moderationLogService: ModerationLogService, private fanoutTimelineService: FanoutTimelineService, ) { - //this.onMessage = this.onMessage.bind(this); - - this.rolesCache = new MemorySingleCache(1000 * 60 * 60 * 1); - this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 1); + this.rolesCache = new MemorySingleCache(1000 * 60 * 60); // 1h + this.roleAssignmentByUserIdCache = new MemoryKVCache(1000 * 60 * 5); // 5m this.redisForSub.on('message', this.onMessage); } diff --git a/packages/backend/src/core/UserKeypairService.ts b/packages/backend/src/core/UserKeypairService.ts index 51ac99179..92d61cd10 100644 --- a/packages/backend/src/core/UserKeypairService.ts +++ b/packages/backend/src/core/UserKeypairService.ts @@ -25,7 +25,7 @@ export class UserKeypairService implements OnApplicationShutdown { ) { this.cache = new RedisKVCache(this.redisClient, 'userKeypair', { lifetime: 1000 * 60 * 60 * 24, // 24h - memoryCacheLifetime: Infinity, + memoryCacheLifetime: 1000 * 60 * 60, // 1h fetcher: (key) => this.userKeypairsRepository.findOneByOrFail({ userId: key }), toRedisConverter: (value) => JSON.stringify(value), fromRedisConverter: (value) => JSON.parse(value), diff --git a/packages/backend/src/core/UtilityService.ts b/packages/backend/src/core/UtilityService.ts index 16c71e971..7d3a74cf5 100644 --- a/packages/backend/src/core/UtilityService.ts +++ b/packages/backend/src/core/UtilityService.ts @@ -30,6 +30,11 @@ export class UtilityService { return this.toPuny(this.config.host) === this.toPuny(host); } + @bindThis + public isUriLocal(uri: string): boolean { + return this.punyHost(uri) === this.toPuny(this.config.host); + } + @bindThis public isBlockedHost(blockedHosts: string[], host: string | null): boolean { if (host == null) return false; @@ -90,7 +95,7 @@ export class UtilityService { @bindThis public extractDbHost(uri: string): string { const url = new URL(uri); - return this.toPuny(url.hostname); + return this.toPuny(url.host); } @bindThis @@ -103,4 +108,11 @@ export class UtilityService { if (host == null) return null; return punycode.toASCII(host.toLowerCase()); } + + @bindThis + public punyHost(url: string): string { + const urlObj = new URL(url); + const host = `${this.toPuny(urlObj.hostname)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`; + return host; + } } diff --git a/packages/backend/src/core/activitypub/ApDbResolverService.ts b/packages/backend/src/core/activitypub/ApDbResolverService.ts index f6b70ead4..5c16744a7 100644 --- a/packages/backend/src/core/activitypub/ApDbResolverService.ts +++ b/packages/backend/src/core/activitypub/ApDbResolverService.ts @@ -10,6 +10,7 @@ import type { Config } from '@/config.js'; import { MemoryKVCache } from '@/misc/cache.js'; import type { MiUserPublickey } from '@/models/UserPublickey.js'; import { CacheService } from '@/core/CacheService.js'; +import { UtilityService } from '@/core/UtilityService.js'; import type { MiNote } from '@/models/Note.js'; import { bindThis } from '@/decorators.js'; import { MiLocalUser, MiRemoteUser } from '@/models/User.js'; @@ -53,9 +54,10 @@ export class ApDbResolverService implements OnApplicationShutdown { private cacheService: CacheService, private apPersonService: ApPersonService, + private utilityService: UtilityService, ) { - this.publicKeyCache = new MemoryKVCache(Infinity); - this.publicKeyByUserIdCache = new MemoryKVCache(Infinity); + this.publicKeyCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h + this.publicKeyByUserIdCache = new MemoryKVCache(1000 * 60 * 60 * 12); // 12h } @bindThis @@ -63,7 +65,9 @@ export class ApDbResolverService implements OnApplicationShutdown { const separator = '/'; const uri = new URL(getApId(value)); - if (uri.origin !== this.config.url) return { local: false, uri: uri.href }; + if (this.utilityService.toPuny(uri.host) !== this.utilityService.toPuny(this.config.host)) { + return { local: false, uri: uri.href }; + } const [, type, id, ...rest] = uri.pathname.split(separator); return { diff --git a/packages/backend/src/core/activitypub/ApInboxService.ts b/packages/backend/src/core/activitypub/ApInboxService.ts index fd9c7c2ba..4d0342752 100644 --- a/packages/backend/src/core/activitypub/ApInboxService.ts +++ b/packages/backend/src/core/activitypub/ApInboxService.ts @@ -29,6 +29,7 @@ import { bindThis } from '@/decorators.js'; import type { MiLocalUser, MiRemoteUser } from '@/models/User.js'; import { isNotNull } from '@/misc/is-not-null.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; +import { IdentifiableError } from '@/misc/identifiable-error.js'; import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js'; import { ApNoteService } from './models/ApNoteService.js'; import { ApLoggerService } from './ApLoggerService.js'; @@ -38,7 +39,7 @@ import { ApAudienceService } from './ApAudienceService.js'; import { ApPersonService } from './models/ApPersonService.js'; import { ApQuestionService } from './models/ApQuestionService.js'; import type { Resolver } from './ApResolverService.js'; -import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove } from './type.js'; +import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove, IPost } from './type.js'; @Injectable() export class ApInboxService { @@ -90,13 +91,26 @@ export class ApInboxService { } @bindThis - public async performActivity(actor: MiRemoteUser, activity: IObject, additionalCc?: MiLocalUser['id']): Promise { + public async performActivity(actor: MiRemoteUser, activity: IObject, resolver?: Resolver, additionalCc?: MiLocalUser['id']): Promise { + let result = 'error'; if (isCollectionOrOrderedCollection(activity)) { - const resolver = this.apResolverService.createResolver(); - for (const item of toArray(isCollection(activity) ? activity.items : activity.orderedItems)) { + const results = [] as [string, string | void][]; + // eslint-disable-next-line no-param-reassign + resolver ??= this.apResolverService.createResolver(); + + const items = toArray(isCollection(activity) ? activity.items : activity.orderedItems); + if (items.length >= resolver.getRecursionLimit()) { + throw new Error(`skipping activity: collection would surpass recursion limit: ${this.utilityService.extractDbHost(actor.uri)}`); + } + + for (const item of items) { const act = await resolver.resolve(item); + if (act.id == null || this.utilityService.extractDbHost(act.id) !== this.utilityService.extractDbHost(actor.uri)) { + this.logger.warn('skipping activity: activity id is null or mismatching'); + continue; + } try { - await this.performOneActivity(actor, act, additionalCc); + results.push([getApId(item), await this.performOneActivity(actor, act, resolver, additionalCc)]); } catch (err) { if (err instanceof Error || typeof err === 'string') { this.logger.error(err); @@ -105,54 +119,62 @@ export class ApInboxService { } } } + + const hasReason = results.some(([, reason]) => (reason != null && !reason.startsWith('ok'))); + if (hasReason) { + result = results.map(([id, reason]) => `${id}: ${reason}`).join('\n'); + } } else { - await this.performOneActivity(actor, activity, additionalCc); + result = await this.performOneActivity(actor, activity, resolver, additionalCc); } // ついでにリモートユーザーの情報が古かったら更新しておく if (actor.uri) { if (actor.lastFetchedAt == null || Date.now() - actor.lastFetchedAt.getTime() > 1000 * 60 * 60 * 24) { setImmediate(() => { + // 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない this.apPersonService.updatePerson(actor.uri); }); } } + + return result; } @bindThis - public async performOneActivity(actor: MiRemoteUser, activity: IObject, additionalCc?: MiLocalUser['id']): Promise { - if (actor.isSuspended) return; + public async performOneActivity(actor: MiRemoteUser, activity: IObject, resolver?: Resolver, additionalCc?: MiLocalUser['id']): Promise { + if (actor.isSuspended) return 'skip: actor is suspended'; if (isCreate(activity)) { - await this.create(actor, activity, additionalCc); + return await this.create(actor, activity, resolver, additionalCc); } else if (isDelete(activity)) { - await this.delete(actor, activity); + return await this.delete(actor, activity); } else if (isUpdate(activity)) { - await this.update(actor, activity, additionalCc); + return await this.update(actor, activity, resolver, additionalCc); } else if (isFollow(activity)) { - await this.follow(actor, activity); + return await this.follow(actor, activity); } else if (isAccept(activity)) { - await this.accept(actor, activity); + return await this.accept(actor, activity, resolver); } else if (isReject(activity)) { - await this.reject(actor, activity); + return await this.reject(actor, activity, resolver); } else if (isAdd(activity)) { - await this.add(actor, activity).catch(err => this.logger.error(err)); + return await this.add(actor, activity, resolver).catch(err => { this.logger.error(err); return `error: ${err.message}`; }); } else if (isRemove(activity)) { - await this.remove(actor, activity).catch(err => this.logger.error(err)); + return await this.remove(actor, activity, resolver).catch(err => { this.logger.error(err); return `error: ${err.message}`; }); } else if (isAnnounce(activity)) { - await this.announce(actor, activity); + return await this.announce(actor, activity, resolver); } else if (isLike(activity)) { - await this.like(actor, activity); + return await this.like(actor, activity); } else if (isUndo(activity)) { - await this.undo(actor, activity); + return await this.undo(actor, activity, resolver); } else if (isBlock(activity)) { - await this.block(actor, activity); + return await this.block(actor, activity); } else if (isFlag(activity)) { - await this.flag(actor, activity); + return await this.flag(actor, activity); } else if (isMove(activity)) { - await this.move(actor, activity); + return await this.move(actor, activity, resolver); } else { - this.logger.warn(`unrecognized activity type: ${activity.type}`); + return `skip: unknown activity type ${activity.type}`; } } @@ -182,22 +204,26 @@ export class ApInboxService { await this.apNoteService.extractEmojis(activity.tag ?? [], actor.host).catch(() => null); - return await this.reactionService.create(actor, note, activity._misskey_reaction ?? activity.content ?? activity.name).catch(err => { - if (err.id === '51c42bb4-931a-456b-bff7-e5a8a70dd298') { + try { + await this.reactionService.create(actor, note, activity._misskey_reaction ?? activity.content ?? activity.name); + return 'ok'; + } catch (err) { + if (err instanceof IdentifiableError && err.id === '51c42bb4-931a-456b-bff7-e5a8a70dd298') { return 'skip: already reacted'; } else { throw err; } - }).then(() => 'ok'); + } } @bindThis - private async accept(actor: MiRemoteUser, activity: IAccept): Promise { + private async accept(actor: MiRemoteUser, activity: IAccept, resolver?: Resolver): Promise { const uri = activity.id ?? activity; this.logger.info(`Accept: ${uri}`); - const resolver = this.apResolverService.createResolver(); + // eslint-disable-next-line no-param-reassign + resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(err => { this.logger.error(`Resolution failed: ${err}`, { error: err }); @@ -206,7 +232,7 @@ export class ApInboxService { if (isFollow(object)) return await this.acceptFollow(actor, object); - return `skip: Unknown Accept type: ${getApType(object)}`; + return `skip: Unknown Accept type: ${getApType(object) ?? 'undefined'}}`; } @bindThis @@ -234,47 +260,57 @@ export class ApInboxService { } @bindThis - private async add(actor: MiRemoteUser, activity: IAdd): Promise { + private async add(actor: MiRemoteUser, activity: IAdd, resolver?: Resolver): Promise { if (actor.uri !== activity.actor) { - throw new Error('invalid actor'); + return 'skip: invalid actor'; } if (activity.target == null) { - throw new Error('target is null'); + return 'skip: target is null'; } if (activity.target === actor.featured) { - const note = await this.apNoteService.resolveNote(activity.object); - if (note == null) throw new Error('note not found'); + const note = await this.apNoteService.resolveNote(activity.object, { resolver }); + if (note == null) return 'skip: note not found'; await this.notePiningService.addPinned(actor, note.id); - return; + return 'ok'; } - throw new Error(`unknown target: ${activity.target}`); + return `skip: unknown target ${activity.target}`; } @bindThis - private async announce(actor: MiRemoteUser, activity: IAnnounce): Promise { + private async announce(actor: MiRemoteUser, activity: IAnnounce, resolver?: Resolver): Promise { const uri = getApId(activity); this.logger.info(`Announce: ${uri}`); - const targetUri = getApId(activity.object); + // eslint-disable-next-line no-param-reassign + resolver ??= this.apResolverService.createResolver(); - await this.announceNote(actor, activity, targetUri); + if (!activity.object) return 'skip: activity has no object property'; + + const target = await resolver.resolve(activity.object).catch(err => { + this.logger.error(`Resolution failed: ${err}`, { error: err }); + return err; + }); + + if (isPost(target)) await this.announceNote(actor, activity, target); + + return `skip: unknown object type ${getApType(target) ?? 'undefined'}}`; } @bindThis - private async announceNote(actor: MiRemoteUser, activity: IAnnounce, targetUri: string): Promise { + private async announceNote(actor: MiRemoteUser, activity: IAnnounce, target: IPost, resolver?: Resolver): Promise { const uri = getApId(activity); if (actor.isSuspended) { - return; + return 'skip: actor is suspended'; } // アナウンス先をブロックしてたら中断 const meta = await this.metaService.fetch(); - if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) return; + if (this.utilityService.isBlockedHost(meta.blockedHosts, this.utilityService.extractDbHost(uri))) return 'skip: blocked host'; const unlock = await this.appLockService.getApLock(uri); @@ -282,40 +318,37 @@ export class ApInboxService { // 既に同じURIを持つものが登録されていないかチェック const exist = await this.apNoteService.fetchNote(uri); if (exist) { - return; + return 'skip: note exists'; } // Announce対象をresolve let renote; try { - renote = await this.apNoteService.resolveNote(targetUri); - if (renote == null) throw new Error('announce target is null'); + renote = await this.apNoteService.resolveNote(target, { resolver }); + if (renote == null) return 'skip: target note not found'; } catch (err) { // 対象が4xxならスキップ if (err instanceof StatusError) { if (!err.isRetryable) { - this.logger.warn(`Ignored announce target ${targetUri} - ${err.statusCode}`); - return; + return `skip: Ignored announce target ${target} - ${err.statusCode}`; } - this.logger.warn(`Error in announce target ${targetUri} - ${err.statusCode}`); + this.logger.warn(`Error in announce target ${target} - ${err.statusCode}`); } throw err; } if (!await this.noteEntityService.isVisibleForMe(renote, actor.id)) { - this.logger.warn('skip: invalid actor for this activity'); - return; + return 'skip: invalid actor for this activity'; } this.logger.info(`Creating the (Re)Note: ${uri}`); - const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc); + const activityAudience = await this.apAudienceService.parseAudience(actor, activity.to, activity.cc, resolver); const createdAt = activity.published ? new Date(activity.published) : null; if (createdAt && createdAt < this.idService.parse(renote.id).date) { - this.logger.warn('skip: malformed createdAt'); - return; + return 'skip: malformed createdAt'; } await this.noteCreateService.create(actor, { @@ -328,6 +361,8 @@ export class ApInboxService { } finally { unlock(); } + + return 'ok'; } @bindThis @@ -349,11 +384,13 @@ export class ApInboxService { } @bindThis - private async create(actor: MiRemoteUser, activity: ICreate, additionalCc?: MiLocalUser['id']): Promise { + private async create(actor: MiRemoteUser, activity: ICreate, resolver?: Resolver, additionalCc?: MiLocalUser['id']): Promise { const uri = getApId(activity); this.logger.info(`Create: ${uri}`); + if (!activity.object) return 'skip: activity has no object property'; + // copy audiences between activity <=> object. if (typeof activity.object === 'object') { const to = unique(concat([toArray(activity.to), toArray(activity.object.to)])); @@ -370,7 +407,8 @@ export class ApInboxService { activity.object.attributedTo = activity.actor; } - const resolver = this.apResolverService.createResolver(); + // eslint-disable-next-line no-param-reassign + resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(e => { this.logger.error(`Resolution failed: ${e}`, { error: e }); @@ -378,9 +416,9 @@ export class ApInboxService { }); if (isPost(object)) { - await this.createNote(resolver, actor, object, false, activity, additionalCc); + return await this.createNote(resolver, actor, object, false, activity, additionalCc); } else { - this.logger.warn(`Unknown type: ${getApType(object)}`); + return `skip: Unknown type ${getApType(object) ?? 'undefined'}}`; } } @@ -397,6 +435,8 @@ export class ApInboxService { if (this.utilityService.extractDbHost(actor.uri) !== this.utilityService.extractDbHost(note.id)) { return 'skip: host in actor.uri !== note.id'; } + } else { + return 'skip: note.id is not a string'; } } @@ -412,7 +452,7 @@ export class ApInboxService { return 'skip: note exists'; } - const createdNote = await this.apNoteService.createNote(note, resolver, silent); + const createdNote = await this.apNoteService.createNote(note, actor, resolver, silent); if (createdNote && additionalCc && !await this.noteEntityService.isVisibleForMe(createdNote, additionalCc)) { await this.noteCreateService.appendNoteVisibleUser(actor, createdNote, additionalCc); return 'ok: note visible user appended'; @@ -433,7 +473,7 @@ export class ApInboxService { @bindThis private async delete(actor: MiRemoteUser, activity: IDelete): Promise { if (actor.uri !== activity.actor) { - throw new Error('invalid actor'); + return 'skip: invalid actor'; } // 削除対象objectのtype @@ -553,12 +593,13 @@ export class ApInboxService { } @bindThis - private async reject(actor: MiRemoteUser, activity: IReject): Promise { + private async reject(actor: MiRemoteUser, activity: IReject, resolver?: Resolver): Promise { const uri = activity.id ?? activity; this.logger.info(`Reject: ${uri}`); - const resolver = this.apResolverService.createResolver(); + // eslint-disable-next-line no-param-reassign + resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(e => { this.logger.error(`Resolution failed: ${e}`, { error: e }); @@ -567,7 +608,7 @@ export class ApInboxService { if (isFollow(object)) return await this.rejectFollow(actor, object); - return `skip: Unknown Reject type: ${getApType(object)}`; + return `skip: Unknown Reject type: ${getApType(object) ?? 'undefined'}}`; } @bindThis @@ -595,36 +636,37 @@ export class ApInboxService { } @bindThis - private async remove(actor: MiRemoteUser, activity: IRemove): Promise { + private async remove(actor: MiRemoteUser, activity: IRemove, resolver?: Resolver): Promise { if (actor.uri !== activity.actor) { - throw new Error('invalid actor'); + return 'skip: invalid actor'; } if (activity.target == null) { - throw new Error('target is null'); + return 'skip: target is null'; } if (activity.target === actor.featured) { - const note = await this.apNoteService.resolveNote(activity.object); - if (note == null) throw new Error('note not found'); + const note = await this.apNoteService.resolveNote(activity.object, { resolver }); + if (note == null) return 'skip: note not found'; await this.notePiningService.removePinned(actor, note.id); - return; + return 'ok'; } - throw new Error(`unknown target: ${activity.target}`); + return `skip: unknown target ${activity.target}`; } @bindThis - private async undo(actor: MiRemoteUser, activity: IUndo): Promise { + private async undo(actor: MiRemoteUser, activity: IUndo, resolver?: Resolver): Promise { if (actor.uri !== activity.actor) { - throw new Error('invalid actor'); + return 'skip: invalid actor'; } const uri = activity.id ?? activity; this.logger.info(`Undo: ${uri}`); - const resolver = this.apResolverService.createResolver(); + // eslint-disable-next-line no-param-reassign + resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(e => { this.logger.error(`Resolution failed: ${e}`, { error: e }); @@ -638,7 +680,7 @@ export class ApInboxService { if (isAnnounce(object)) return await this.undoAnnounce(actor, object); if (isAccept(object)) return await this.undoAccept(actor, object); - return `skip: unknown object type ${getApType(object)}`; + return `skip: unknown object type ${getApType(object) ?? 'undefined'}}`; } @bindThis @@ -748,14 +790,15 @@ export class ApInboxService { } @bindThis - private async update(actor: MiRemoteUser, activity: IUpdate, additionalCc?: MiLocalUser['id']): Promise { + private async update(actor: MiRemoteUser, activity: IUpdate, resolver?: Resolver, additionalCc?: MiLocalUser['id']): Promise { if (actor.uri !== activity.actor) { return 'skip: invalid actor'; } this.logger.debug('Update'); - const resolver = this.apResolverService.createResolver(); + // eslint-disable-next-line no-param-reassign + resolver ??= this.apResolverService.createResolver(); const object = await resolver.resolve(activity.object).catch(e => { this.logger.error(`Resolution failed: ${e}`, { error: e }); @@ -766,7 +809,7 @@ export class ApInboxService { await this.apPersonService.updatePerson(actor.uri, resolver, object); return 'ok: Person updated'; } else if (getApType(object) === 'Question') { - await this.apQuestionService.updateQuestion(object, resolver).catch(err => this.logger.error(`err: failed to update question: ${err}`, { error: err })); + await this.apQuestionService.updateQuestion(object, actor, resolver).catch(err => this.logger.error(`err: failed to update question: ${err}`, { error: err })); return 'ok: Question updated'; } else if (additionalCc && isPost(object)) { const uri = getApId(object); @@ -790,16 +833,16 @@ export class ApInboxService { unlock(); } } else { - return `skip: Unknown type: ${getApType(object)}`; + return `skip: Unknown type: ${getApType(object) ?? 'undefined'}}`; } } @bindThis - private async move(actor: MiRemoteUser, activity: IMove): Promise { + private async move(actor: MiRemoteUser, activity: IMove, resolver?: Resolver): Promise { // fetch the new and old accounts const targetUri = getApHrefNullable(activity.target); if (!targetUri) return 'skip: invalid activity target'; - return await this.apPersonService.updatePerson(actor.uri) ?? 'skip: nothing to do'; + return await this.apPersonService.updatePerson(actor.uri, resolver) ?? 'skip: nothing to do'; } } diff --git a/packages/backend/src/core/activitypub/ApRequestService.ts b/packages/backend/src/core/activitypub/ApRequestService.ts index b008a3ec5..f59fabba3 100644 --- a/packages/backend/src/core/activitypub/ApRequestService.ts +++ b/packages/backend/src/core/activitypub/ApRequestService.ts @@ -6,15 +6,19 @@ import * as crypto from 'node:crypto'; import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; +import { JSDOM } from 'jsdom'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import type { MiUser } from '@/models/User.js'; import { UserKeypairService } from '@/core/UserKeypairService.js'; +import { UtilityService } from '@/core/UtilityService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { LoggerService } from '@/core/LoggerService.js'; import { bindThis } from '@/decorators.js'; import type Logger from '@/logger.js'; import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js'; +import { assertActivityMatchesUrls } from '@/core/activitypub/misc/check-against-url.js'; +import type { IObject } from './type.js'; type Request = { url: string; @@ -144,6 +148,7 @@ export class ApRequestService { private userKeypairService: UserKeypairService, private httpRequestService: HttpRequestService, private loggerService: LoggerService, + private utilityService: UtilityService, ) { this.logger = this.loggerService.getLogger('ap:request'); } @@ -179,7 +184,8 @@ export class ApRequestService { * @param url URL to fetch */ @bindThis - public async signedGet(url: string, user: { id: MiUser['id'] }): Promise { + public async signedGet(url: string, user: { id: MiUser['id'] }, followAlternate?: boolean): Promise { + const _followAlternate = followAlternate ?? true; const keypair = await this.userKeypairService.getUserKeypair(user.id); const req = ApRequestCreator.createSignedGet({ @@ -197,9 +203,39 @@ export class ApRequestService { headers: req.request.headers, }, { throwErrorWhenResponseNotOk: true, - validators: [validateContentTypeSetAsActivityPub], }); - return await res.json(); + //#region リクエスト先がhtmlかつactivity+jsonへのalternate linkタグがあるとき + const contentType = res.headers.get('content-type'); + + if ( + res.ok && + (contentType ?? '').split(';')[0].trimEnd().toLowerCase() === 'text/html' && + _followAlternate + ) { + const html = await res.text(); + try { + const fragment = JSDOM.fragment(html); + + const alternate = fragment.querySelector('head > link[rel="alternate"][type="application/activity+json"]'); + if (alternate) { + const href = alternate.getAttribute('href'); + if (href && this.utilityService.punyHost(url) === this.utilityService.punyHost(href)) { + return await this.signedGet(href, user, false); + } + } + } catch (e) { + // something went wrong parsing the HTML, ignore the whole thing + } + } + //#endregion + + validateContentTypeSetAsActivityPub(res); + const finalUrl = res.url; // redirects may have been involved + const activity = await res.json() as IObject; + + assertActivityMatchesUrls(activity, [finalUrl]); + + return activity; } } diff --git a/packages/backend/src/core/activitypub/ApResolverService.ts b/packages/backend/src/core/activitypub/ApResolverService.ts index fbe32ba18..000f9d59f 100644 --- a/packages/backend/src/core/activitypub/ApResolverService.ts +++ b/packages/backend/src/core/activitypub/ApResolverService.ts @@ -42,7 +42,7 @@ export class Resolver { private apRendererService: ApRendererService, private apDbResolverService: ApDbResolverService, private loggerService: LoggerService, - private recursionLimit = 100, + private recursionLimit = 256, ) { this.history = new Set(); this.logger = this.loggerService.getLogger('ap:resolve'); @@ -53,6 +53,11 @@ export class Resolver { return Array.from(this.history); } + @bindThis + public getRecursionLimit(): number { + return this.recursionLimit; + } + @bindThis public async resolveCollection(value: string | IObject): Promise { const collection = typeof value === 'string' @@ -115,6 +120,18 @@ export class Resolver { throw new Error('invalid response'); } + // HttpRequestService / ApRequestService have already checked that + // `object.id` or `object.url` matches the URL used to fetch the + // object after redirects; here we double-check that no redirects + // bounced between hosts + if (object.id == null) { + throw new Error('invalid AP object: missing id'); + } + + if (this.utilityService.punyHost(object.id) !== this.utilityService.punyHost(value)) { + throw new Error(`invalid AP object ${value}: id ${object.id} has different host`); + } + return object; } diff --git a/packages/backend/src/core/activitypub/misc/check-against-url.ts b/packages/backend/src/core/activitypub/misc/check-against-url.ts new file mode 100644 index 000000000..78ba891a2 --- /dev/null +++ b/packages/backend/src/core/activitypub/misc/check-against-url.ts @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: dakkar and sharkey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ +import type { IObject } from '../type.js'; + +export function assertActivityMatchesUrls(activity: IObject, urls: string[]) { + const idOk = activity.id !== undefined && urls.includes(activity.id); + + // technically `activity.url` could be an `ApObject = IObject | + // string | (IObject | string)[]`, but if it's a complicated thing + // and the `activity.id` doesn't match, I think we're fine + // rejecting the activity + const urlOk = typeof(activity.url) === 'string' && urls.includes(activity.url); + + if (!idOk && !urlOk) { + throw new Error(`bad Activity: neither id(${activity?.id}) nor url(${activity?.url}) match location(${urls})`); + } +} diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 3d72c502e..3ce588e18 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -80,24 +80,33 @@ export class ApNoteService { } @bindThis - public validateNote(object: IObject, uri: string): Error | null { + public validateNote(object: IObject, uri: string, actor?: MiRemoteUser): Error | null { const expectHost = this.utilityService.extractDbHost(uri); + const apType = getApType(object); - if (!validPost.includes(getApType(object))) { - return new Error(`invalid Note: invalid object type ${getApType(object)}`); + if (apType == null || !validPost.includes(apType)) { + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: invalid object type ${apType ?? 'undefined'}`); } if (object.id && this.utilityService.extractDbHost(object.id) !== expectHost) { - return new Error(`invalid Note: id has different host. expected: ${expectHost}, actual: ${this.utilityService.extractDbHost(object.id)}`); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: id has different host. expected: ${expectHost}, actual: ${this.utilityService.extractDbHost(object.id)}`); } const actualHost = object.attributedTo && this.utilityService.extractDbHost(getOneApId(object.attributedTo)); if (object.attributedTo && actualHost !== expectHost) { - return new Error(`invalid Note: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: attributedTo has different host. expected: ${expectHost}, actual: ${actualHost}`); } if (object.published && !this.idService.isSafeT(new Date(object.published).valueOf())) { - return new Error('invalid Note: published timestamp is malformed'); + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', 'invalid Note: published timestamp is malformed'); + } + + if (actor) { + const attribution = (object.attributedTo) ? getOneApId(object.attributedTo) : actor.uri; + + if (attribution !== actor.uri) { + return new IdentifiableError('d450b8a9-48e4-4dab-ae36-f4db763fda7c', `invalid Note: attribution does not match the actor that send it. attribution: ${attribution}, actor: ${actor.uri}`); + } } return null; @@ -117,7 +126,7 @@ export class ApNoteService { * Noteを作成します。 */ @bindThis - public async createNote(value: string | IObject, resolver?: Resolver, silent = false): Promise { + public async createNote(value: string | IObject, actor?: MiRemoteUser, resolver?: Resolver, silent = false): Promise { // eslint-disable-next-line no-param-reassign if (resolver == null) resolver = this.apResolverService.createResolver(); @@ -126,7 +135,7 @@ export class ApNoteService { const object = await resolver.resolve(value); const entryUri = getApId(value); - const err = this.validateNote(object, entryUri); + const err = this.validateNote(object, entryUri, actor); if (err) { this.logger.error(err.message, { resolver: { history: resolver.getHistory() }, @@ -141,14 +150,24 @@ export class ApNoteService { this.logger.debug(`Note fetched: ${JSON.stringify(note, null, 2)}`); - if (note.id && !checkHttps(note.id)) { + if (note.id == null) { + throw new Error('Refusing to create note without id'); + } + + if (!checkHttps(note.id)) { throw new Error('unexpected schema of note.id: ' + note.id); } const url = getOneApHrefNullable(note.url); - if (url && !checkHttps(url)) { - throw new Error('unexpected schema of note url: ' + url); + if (url != null) { + if (!checkHttps(url)) { + throw new Error('unexpected schema of note url: ' + url); + } + + if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(note.id)) { + throw new Error(`note url & uri host mismatch: note url: ${url}, note uri: ${note.id}`); + } } this.logger.info(`Creating the Note: ${note.id}`); @@ -161,9 +180,10 @@ export class ApNoteService { const uri = getOneApId(note.attributedTo); // ローカルで投稿者を検索し、もし凍結されていたらスキップ - const cachedActor = await this.apPersonService.fetchPerson(uri) as MiRemoteUser; - if (cachedActor && cachedActor.isSuspended) { - throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `User ${cachedActor.id} has been suspended.`); + // eslint-disable-next-line no-param-reassign + actor ??= await this.apPersonService.fetchPerson(uri) as MiRemoteUser | undefined; + if (actor && actor.isSuspended) { + throw new IdentifiableError('85ab9bd7-3a41-4530-959d-f07073900109', `User ${actor.id} has been suspended.`); } const apMentions = await this.apMentionService.extractApMentions(note.tag, resolver); @@ -194,7 +214,8 @@ export class ApNoteService { } //#endregion - const actor = cachedActor ?? await this.apPersonService.resolvePerson(uri, resolver) as MiRemoteUser; + // eslint-disable-next-line no-param-reassign + actor ??= await this.apPersonService.resolvePerson(uri, resolver) as MiRemoteUser; // 解決した投稿者が凍結されていたらスキップ if (actor.isSuspended) { @@ -358,7 +379,7 @@ export class ApNoteService { if (exist) return exist; //#endregion - if (new URL(uri).origin === this.config.url) { + if (this.utilityService.isUriLocal(uri)) { throw new StatusError('cannot resolve local note', 400, 'cannot resolve local note'); } @@ -366,7 +387,7 @@ export class ApNoteService { // ここでuriの代わりに添付されてきたNote Objectが指定されていると、サーバーフェッチを経ずにノートが生成されるが // 添付されてきたNote Objectは偽装されている可能性があるため、常にuriを指定してサーバーフェッチを行う。 const createFrom = options.sentFrom?.origin === new URL(uri).origin ? value : uri; - return await this.createNote(createFrom, options.resolver, true); + return await this.createNote(createFrom, undefined, options.resolver, true); } finally { unlock(); } diff --git a/packages/backend/src/core/activitypub/models/ApPersonService.ts b/packages/backend/src/core/activitypub/models/ApPersonService.ts index 430ec714a..88c2dc5d2 100644 --- a/packages/backend/src/core/activitypub/models/ApPersonService.ts +++ b/packages/backend/src/core/activitypub/models/ApPersonService.ts @@ -49,7 +49,7 @@ import type { ApResolverService, Resolver } from '../ApResolverService.js'; import type { ApLoggerService } from '../ApLoggerService.js'; // eslint-disable-next-line @typescript-eslint/consistent-type-imports import type { ApImageService } from './ApImageService.js'; -import type { IActor, IObject } from '../type.js'; +import type { IActor, ICollection, IObject, IOrderedCollection } from '../type.js'; const nameLength = 128; const summaryLength = 2048; @@ -130,12 +130,6 @@ export class ApPersonService implements OnModuleInit { this.logger = this.apLoggerService.logger; } - private punyHost(url: string): string { - const urlObj = new URL(url); - const host = `${this.utilityService.toPuny(urlObj.hostname)}${urlObj.port.length > 0 ? ':' + urlObj.port : ''}`; - return host; - } - /** * Validate and convert to actor object * @param x Fetched object @@ -143,7 +137,7 @@ export class ApPersonService implements OnModuleInit { */ @bindThis private validateActor(x: IObject, uri: string): IActor { - const expectHost = this.punyHost(uri); + const expectHost = this.utilityService.punyHost(uri); if (!isActor(x)) { throw new Error(`invalid Actor type '${x.type}'`); @@ -157,18 +151,29 @@ export class ApPersonService implements OnModuleInit { throw new Error('invalid Actor: wrong inbox'); } - try { - new URL(x.inbox); - } catch { - throw new Error('invalid Actor: wrong inbox'); + if (this.utilityService.punyHost(x.inbox) !== expectHost) { + throw new Error('invalid Actor: inbox has different host'); } - const sharedInbox = x.sharedInbox ?? x.endpoints?.sharedInbox; - if (typeof sharedInbox === 'string') { - try { - new URL(sharedInbox); - } catch { - throw new Error('invalid Actor: wrong sharedInbox'); + const sharedInboxObject = x.sharedInbox ?? (x.endpoints ? x.endpoints.sharedInbox : undefined); + if (sharedInboxObject != null) { + const sharedInbox = getApId(sharedInboxObject); + if (!(typeof sharedInbox === 'string' && sharedInbox.length > 0 && this.utilityService.punyHost(sharedInbox) === expectHost)) { + throw new Error('invalid Actor: wrong shared inbox'); + } + } + + for (const collection of ['outbox', 'followers', 'following'] as (keyof IActor)[]) { + const xCollection = (x as IActor)[collection]; + if (xCollection != null) { + const collectionUri = getApId(xCollection); + if (typeof collectionUri === 'string' && collectionUri.length > 0) { + if (this.utilityService.punyHost(collectionUri) !== expectHost) { + throw new Error(`invalid Actor: ${collection} has different host`); + } + } else if (collectionUri != null) { + throw new Error(`invalid Actor: wrong ${collection}`); + } } } @@ -195,7 +200,7 @@ export class ApPersonService implements OnModuleInit { x.summary = truncate(x.summary, summaryLength); } - const idHost = this.punyHost(x.id); + const idHost = this.utilityService.punyHost(x.id); if (idHost !== expectHost) { throw new Error('invalid Actor: id has different host'); } @@ -205,7 +210,7 @@ export class ApPersonService implements OnModuleInit { throw new Error('invalid Actor: publicKey.id is not a string'); } - const publicKeyIdHost = this.punyHost(x.publicKey.id); + const publicKeyIdHost = this.utilityService.punyHost(x.publicKey.id); if (publicKeyIdHost !== expectHost) { throw new Error('invalid Actor: publicKey.id has different host'); } @@ -248,6 +253,12 @@ export class ApPersonService implements OnModuleInit { if (user == null) throw new Error('failed to create user: user is null'); const [avatar, banner] = await Promise.all([icon, image].map(img => { + // icon and image may be arrays + // see https://www.w3.org/TR/activitystreams-vocabulary/#dfn-icon + if (Array.isArray(img)) { + img = img.find(item => item && item.url) ?? null; + } + // if we have an explicitly missing image, return an // explicitly-null set of values if ((img == null) || (typeof img === 'object' && img.url == null)) { @@ -285,7 +296,8 @@ export class ApPersonService implements OnModuleInit { public async createPerson(uri: string, resolver?: Resolver): Promise { if (typeof uri !== 'string') throw new Error('uri is not string'); - if (new URL(uri).origin === this.config.url) { + const host = this.utilityService.punyHost(uri); + if (host === this.utilityService.toPuny(this.config.host)) { throw new StatusError('cannot resolve local user', 400, 'cannot resolve local user'); } @@ -299,20 +311,43 @@ export class ApPersonService implements OnModuleInit { this.logger.info(`Creating the Person: ${person.id}`); - const host = this.punyHost(object.id); - const fields = this.analyzeAttachments(person.attachment ?? []); const tags = extractApHashtags(person.tag).map(normalizeForSearch).splice(0, 32); const isBot = getApType(object) === 'Service' || getApType(object) === 'Application'; + const [followingVisibility, followersVisibility] = await Promise.all( + [ + this.isPublicCollection(person.following, resolver), + this.isPublicCollection(person.followers, resolver), + ].map((p): Promise<'public' | 'private'> => p + .then(isPublic => isPublic ? 'public' : 'private') + .catch(err => { + if (!(err instanceof StatusError) || err.isRetryable) { + this.logger.error('error occurred while fetching following/followers collection', { error: err }); + } + return 'private'; + }) + ) + ); + const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/); const url = getOneApHrefNullable(person.url); - if (url && !checkHttps(url)) { - throw new Error('unexpected schema of person url: ' + url); + if (person.id == null) { + throw new Error('Refusing to create person without id'); + } + + if (url != null) { + if (!checkHttps(url)) { + throw new Error('unexpected schema of person url: ' + url); + } + + if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) { + throw new Error(`person url <> uri host mismatch: ${url} <> ${person.id}`); + } } // Create user @@ -345,7 +380,7 @@ export class ApPersonService implements OnModuleInit { usernameLower: person.preferredUsername?.toLowerCase(), host, inbox: person.inbox, - sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox, + sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, followersUri: person.followers ? getApId(person.followers) : undefined, featured: person.featured ? getApId(person.featured) : undefined, uri: person.id, @@ -368,6 +403,8 @@ export class ApPersonService implements OnModuleInit { description: _description, url, fields, + followingVisibility, + followersVisibility, birthday: bday?.[0] ?? null, location: person['vcard:Address'] ?? null, userHost: host, @@ -447,7 +484,7 @@ export class ApPersonService implements OnModuleInit { if (typeof uri !== 'string') throw new Error('uri is not string'); // URIがこのサーバーを指しているならスキップ - if (new URL(uri).origin === this.config.url) return; + if (this.utilityService.isUriLocal(uri)) return; //#region このサーバーに既に登録されているか const exist = await this.fetchPerson(uri) as MiRemoteUser | null; @@ -475,12 +512,39 @@ export class ApPersonService implements OnModuleInit { const tags = extractApHashtags(person.tag).map(normalizeForSearch).splice(0, 32); + const [followingVisibility, followersVisibility] = await Promise.all( + [ + this.isPublicCollection(person.following, resolver), + this.isPublicCollection(person.followers, resolver), + ].map((p): Promise<'public' | 'private' | undefined> => p + .then(isPublic => isPublic ? 'public' : 'private') + .catch(err => { + if (!(err instanceof StatusError) || err.isRetryable) { + this.logger.error('error occurred while fetching following/followers collection', { error: err }); + // Do not update the visibiility on transient errors. + return undefined; + } + return 'private'; + }) + ) + ); + const bday = person['vcard:bday']?.match(/^\d{4}-\d{2}-\d{2}/); const url = getOneApHrefNullable(person.url); - if (url && !checkHttps(url)) { - throw new Error('unexpected schema of person url: ' + url); + if (person.id == null) { + throw new Error('Refusing to update person without id'); + } + + if (url != null) { + if (!checkHttps(url)) { + throw new Error('unexpected schema of person url: ' + url); + } + + if (this.utilityService.punyHost(url) !== this.utilityService.punyHost(person.id)) { + throw new Error(`person url <> uri host mismatch: ${url} <> ${person.id}`); + } } const policy = await this.roleService.getUserPolicies(exist.id); @@ -488,7 +552,7 @@ export class ApPersonService implements OnModuleInit { const updates = { lastFetchedAt: new Date(), inbox: person.inbox, - sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox, + sharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null, followersUri: person.followers ? getApId(person.followers) : undefined, featured: person.featured, emojis: emojiNames, @@ -545,6 +609,8 @@ export class ApPersonService implements OnModuleInit { url, fields, description: _description, + followingVisibility, + followersVisibility, birthday: bday?.[0] ?? null, location: person['vcard:Address'] ?? null, }); @@ -557,7 +623,7 @@ export class ApPersonService implements OnModuleInit { // 該当ユーザーが既にフォロワーになっていた場合はFollowingもアップデートする await this.followingsRepository.update( { followerId: exist.id }, - { followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox }, + { followerSharedInbox: person.sharedInbox ?? person.endpoints?.sharedInbox ?? null }, ); await this.updateFeatured(exist.id, resolver).catch(err => this.logger.error(err)); @@ -689,7 +755,7 @@ export class ApPersonService implements OnModuleInit { await this.updatePerson(src.movedToUri, undefined, undefined, [...movePreventUris, src.uri]); dst = await this.fetchPerson(src.movedToUri) ?? dst; } else { - if (new URL(src.movedToUri).origin === this.config.url) { + if (this.utilityService.isUriLocal(src.movedToUri)) { // ローカルユーザーっぽいのにfetchPersonで見つからないということはmovedToUriが間違っている return 'failed: movedTo is local but not found'; } @@ -715,4 +781,16 @@ export class ApPersonService implements OnModuleInit { return 'ok'; } + + @bindThis + private async isPublicCollection(collection: string | ICollection | IOrderedCollection | undefined, resolver: Resolver): Promise { + if (collection) { + const resolved = await resolver.resolveCollection(collection); + if (resolved.first || (resolved as ICollection).items || (resolved as IOrderedCollection).orderedItems) { + return true; + } + } + + return false; + } } diff --git a/packages/backend/src/core/activitypub/models/ApQuestionService.ts b/packages/backend/src/core/activitypub/models/ApQuestionService.ts index 4e3df00e0..d34c1a76c 100644 --- a/packages/backend/src/core/activitypub/models/ApQuestionService.ts +++ b/packages/backend/src/core/activitypub/models/ApQuestionService.ts @@ -5,17 +5,19 @@ import { Inject, Injectable } from '@nestjs/common'; import { DI } from '@/di-symbols.js'; -import type { NotesRepository, PollsRepository } from '@/models/_.js'; +import type { UsersRepository, NotesRepository, PollsRepository } from '@/models/_.js'; import type { Config } from '@/config.js'; import type { IPoll } from '@/models/Poll.js'; +import type { MiRemoteUser } from '@/models/User.js'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; import { isNotNull } from '@/misc/is-not-null.js'; -import { isQuestion } from '../type.js'; +import { UtilityService } from '@/core/UtilityService.js'; +import { getOneApId, isQuestion } from '../type.js'; import { ApLoggerService } from '../ApLoggerService.js'; import { ApResolverService } from '../ApResolverService.js'; import type { Resolver } from '../ApResolverService.js'; -import type { IObject, IQuestion } from '../type.js'; +import type { IObject } from '../type.js'; @Injectable() export class ApQuestionService { @@ -25,6 +27,9 @@ export class ApQuestionService { @Inject(DI.config) private config: Config, + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + @Inject(DI.notesRepository) private notesRepository: NotesRepository, @@ -33,6 +38,7 @@ export class ApQuestionService { private apResolverService: ApResolverService, private apLoggerService: ApLoggerService, + private utilityService: UtilityService, ) { this.logger = this.apLoggerService.logger; } @@ -66,28 +72,39 @@ export class ApQuestionService { * @returns true if updated */ @bindThis - public async updateQuestion(value: string | IObject, resolver?: Resolver): Promise { + public async updateQuestion(value: string | IObject, actor?: MiRemoteUser, resolver?: Resolver): Promise { const uri = typeof value === 'string' ? value : value.id; if (uri == null) throw new Error('uri is null'); // URIがこのサーバーを指しているならスキップ - if (new URL(uri).origin === this.config.url) throw new Error('uri points local'); + if (this.utilityService.isUriLocal(uri)) throw new Error('uri points local'); //#region このサーバーに既に登録されているか const note = await this.notesRepository.findOneBy({ uri }); if (note == null) throw new Error('Question is not registed'); const poll = await this.pollsRepository.findOneBy({ noteId: note.id }); - if (poll == null) throw new Error('Question is not registed'); + if (poll == null) throw new Error('Question is not registered'); + + const user = await this.usersRepository.findOneBy({ id: poll.userId }); + if (user == null) throw new Error('User is not registered'); //#endregion // resolve new Question object // eslint-disable-next-line no-param-reassign if (resolver == null) resolver = this.apResolverService.createResolver(); - const question = await resolver.resolve(value) as IQuestion; + const question = await resolver.resolve(value); this.logger.debug(`fetched question: ${JSON.stringify(question, null, 2)}`); - if (question.type !== 'Question') throw new Error('object is not a Question'); + if (!isQuestion(question)) throw new Error('object is not a Question'); + + const attribution = (question.attributedTo) ? getOneApId(question.attributedTo) : user.uri; + const attributionMatchesExisting = attribution === user.uri; + const actorMatchesAttribution = (actor) ? attribution === actor.uri : true; + + if (!attributionMatchesExisting || !actorMatchesAttribution) { + throw new Error('Refusing to ingest update for poll by different user'); + } const apChoices = question.oneOf ?? question.anyOf; if (apChoices == null) throw new Error('invalid apChoices: ' + apChoices); @@ -97,7 +114,7 @@ export class ApQuestionService { for (const choice of poll.choices) { const oldCount = poll.votes[poll.choices.indexOf(choice)]; const newCount = apChoices.filter(ap => ap.name === choice).at(0)?.replies?.totalItems; - if (newCount == null) throw new Error('invalid newCount: ' + newCount); + if (newCount == null || !(Number.isInteger(newCount) && newCount >= 0)) throw new Error('invalid newCount: ' + newCount); if (oldCount !== newCount) { changed = true; diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 09322888d..16812b7a4 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -60,11 +60,14 @@ export function getApId(value: string | IObject): string { /** * Get ActivityStreams Object type + * + * タイプ判定ができなかった場合に、あえてエラーではなくnullを返すようにしている。 + * 詳細: https://github.com/misskey-dev/misskey/issues/14239 */ -export function getApType(value: IObject): string { +export function getApType(value: IObject): string | null { if (typeof value.type === 'string') return value.type; if (Array.isArray(value.type) && typeof value.type[0] === 'string') return value.type[0]; - throw new Error('cannot detect type'); + return null; } export function getOneApHrefNullable(value: ApObject | undefined): string | undefined { @@ -97,19 +100,23 @@ export interface IActivity extends IObject { export interface ICollection extends IObject { type: 'Collection'; totalItems: number; - items: ApObject; + first?: IObject | string; + items?: ApObject; } export interface IOrderedCollection extends IObject { type: 'OrderedCollection'; totalItems: number; - orderedItems: ApObject; + first?: IObject | string; + orderedItems?: ApObject; } export const validPost = ['Note', 'Question', 'Article', 'Audio', 'Document', 'Image', 'Page', 'Video', 'Event']; -export const isPost = (object: IObject): object is IPost => - validPost.includes(getApType(object)); +export const isPost = (object: IObject): object is IPost => { + const type = getApType(object); + return type != null && validPost.includes(type); +}; export interface IPost extends IObject { type: 'Note' | 'Question' | 'Article' | 'Audio' | 'Document' | 'Image' | 'Page' | 'Video' | 'Event'; @@ -156,8 +163,10 @@ export const isTombstone = (object: IObject): object is ITombstone => export const validActor = ['Person', 'Service', 'Group', 'Organization', 'Application']; -export const isActor = (object: IObject): object is IActor => - validActor.includes(getApType(object)); +export const isActor = (object: IObject): object is IActor => { + const type = getApType(object); + return type != null && validActor.includes(type); +}; export interface IActor extends IObject { type: 'Person' | 'Service' | 'Organization' | 'Group' | 'Application'; @@ -240,12 +249,16 @@ export interface IKey extends IObject { publicKeyPem: string | Buffer; } +export const validDocumentTypes = ['Audio', 'Document', 'Image', 'Page', 'Video']; + export interface IApDocument extends IObject { type: 'Audio' | 'Document' | 'Image' | 'Page' | 'Video'; } -export const isDocument = (object: IObject): object is IApDocument => - ['Audio', 'Document', 'Image', 'Page', 'Video'].includes(getApType(object)); +export const isDocument = (object: IObject): object is IApDocument => { + const type = getApType(object); + return type != null && validDocumentTypes.includes(type); +}; export interface IApImage extends IApDocument { type: 'Image'; @@ -323,8 +336,12 @@ export const isAccept = (object: IObject): object is IAccept => getApType(object export const isReject = (object: IObject): object is IReject => getApType(object) === 'Reject'; export const isAdd = (object: IObject): object is IAdd => getApType(object) === 'Add'; export const isRemove = (object: IObject): object is IRemove => getApType(object) === 'Remove'; -export const isLike = (object: IObject): object is ILike => getApType(object) === 'Like' || getApType(object) === 'EmojiReaction' || getApType(object) === 'EmojiReact'; +export const isLike = (object: IObject): object is ILike => { + const type = getApType(object); + return type != null && ['Like', 'EmojiReaction', 'EmojiReact'].includes(type); +}; export const isAnnounce = (object: IObject): object is IAnnounce => getApType(object) === 'Announce'; export const isBlock = (object: IObject): object is IBlock => getApType(object) === 'Block'; export const isFlag = (object: IObject): object is IFlag => getApType(object) === 'Flag'; export const isMove = (object: IObject): object is IMove => getApType(object) === 'Move'; +export const isNote = (object: IObject): object is IPost => getApType(object) === 'Note'; diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 66ae69ed9..e9bf5a397 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -456,12 +456,12 @@ export class UserEntityService implements OnModuleInit { } const followingCount = profile == null ? null : - (profile.followingVisibility === 'public') || isMe ? user.followingCount : + (profile.followingVisibility === 'public') || isMe || iAmModerator ? user.followingCount : (profile.followingVisibility === 'followers') && (relation && relation.isFollowing) ? user.followingCount : null; const followersCount = profile == null ? null : - (profile.followersVisibility === 'public') || isMe ? user.followersCount : + (profile.followersVisibility === 'public') || isMe || iAmModerator ? user.followersCount : (profile.followersVisibility === 'followers') && (relation && relation.isFollowing) ? user.followersCount : null; diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index 4d7809570..07d4be7f3 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -7,23 +7,23 @@ import * as Redis from 'ioredis'; import { bindThis } from '@/decorators.js'; export class RedisKVCache { - private redisClient: Redis.Redis; - private name: string; - private lifetime: number; - private memoryCache: MemoryKVCache; - private fetcher: (key: string) => Promise; - private toRedisConverter: (value: T) => string; - private fromRedisConverter: (value: string) => T | undefined; + private readonly lifetime: number; + private readonly memoryCache: MemoryKVCache; + private readonly fetcher: (key: string) => Promise; + private readonly toRedisConverter: (value: T) => string; + private readonly fromRedisConverter: (value: string) => T | undefined; - constructor(redisClient: RedisKVCache['redisClient'], name: RedisKVCache['name'], opts: { - lifetime: RedisKVCache['lifetime']; - memoryCacheLifetime: number; - fetcher: RedisKVCache['fetcher']; - toRedisConverter: RedisKVCache['toRedisConverter']; - fromRedisConverter: RedisKVCache['fromRedisConverter']; - }) { - this.redisClient = redisClient; - this.name = name; + constructor( + private redisClient: Redis.Redis, + private name: string, + opts: { + lifetime: RedisKVCache['lifetime']; + memoryCacheLifetime: number; + fetcher: RedisKVCache['fetcher']; + toRedisConverter: RedisKVCache['toRedisConverter']; + fromRedisConverter: RedisKVCache['fromRedisConverter']; + }, + ) { this.lifetime = opts.lifetime; this.memoryCache = new MemoryKVCache(opts.memoryCacheLifetime); this.fetcher = opts.fetcher; @@ -55,10 +55,13 @@ export class RedisKVCache { const cached = await this.redisClient.get(`kvcache:${this.name}:${key}`); if (cached == null) return undefined; - const parsed = this.fromRedisConverter(cached); - if (parsed == null) return undefined; - this.memoryCache.set(key, parsed); - return parsed; + + const value = this.fromRedisConverter(cached); + if (value !== undefined) { + this.memoryCache.set(key, value); + } + + return value; } @bindThis @@ -69,6 +72,10 @@ export class RedisKVCache { /** * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します + * This awaits the call to Redis to ensure that the write succeeded, which is important for a few reasons: + * * Other code uses this to synchronize changes between worker processes. A failed write can internally de-sync the cluster. + * * Without an `await`, consecutive calls could race. An unlucky race could result in the older write overwriting the newer value. + * * Not awaiting here makes the entire cache non-consistent. The prevents many possible uses. */ @bindThis public async fetch(key: string): Promise { @@ -80,14 +87,14 @@ export class RedisKVCache { // Cache MISS const value = await this.fetcher(key); - this.set(key, value); + await this.set(key, value); return value; } @bindThis public async refresh(key: string) { const value = await this.fetcher(key); - this.set(key, value); + await this.set(key, value); // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } @@ -104,23 +111,23 @@ export class RedisKVCache { } export class RedisSingleCache { - private redisClient: Redis.Redis; - private name: string; - private lifetime: number; - private memoryCache: MemorySingleCache; - private fetcher: () => Promise; - private toRedisConverter: (value: T) => string; - private fromRedisConverter: (value: string) => T | undefined; + private readonly lifetime: number; + private readonly memoryCache: MemorySingleCache; + private readonly fetcher: () => Promise; + private readonly toRedisConverter: (value: T) => string; + private readonly fromRedisConverter: (value: string) => T | undefined; - constructor(redisClient: RedisSingleCache['redisClient'], name: RedisSingleCache['name'], opts: { - lifetime: RedisSingleCache['lifetime']; - memoryCacheLifetime: number; - fetcher: RedisSingleCache['fetcher']; - toRedisConverter: RedisSingleCache['toRedisConverter']; - fromRedisConverter: RedisSingleCache['fromRedisConverter']; - }) { - this.redisClient = redisClient; - this.name = name; + constructor( + private redisClient: Redis.Redis, + private name: string, + opts: { + lifetime: number; + memoryCacheLifetime: number; + fetcher: RedisSingleCache['fetcher']; + toRedisConverter: RedisSingleCache['toRedisConverter']; + fromRedisConverter: RedisSingleCache['fromRedisConverter']; + }, + ) { this.lifetime = opts.lifetime; this.memoryCache = new MemorySingleCache(opts.memoryCacheLifetime); this.fetcher = opts.fetcher; @@ -152,10 +159,13 @@ export class RedisSingleCache { const cached = await this.redisClient.get(`singlecache:${this.name}`); if (cached == null) return undefined; - const parsed = this.fromRedisConverter(cached); - if (parsed == null) return undefined; - this.memoryCache.set(parsed); - return parsed; + + const value = this.fromRedisConverter(cached); + if (value !== undefined) { + this.memoryCache.set(value); + } + + return value; } @bindThis @@ -166,6 +176,10 @@ export class RedisSingleCache { /** * キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します + * This awaits the call to Redis to ensure that the write succeeded, which is important for a few reasons: + * * Other code uses this to synchronize changes between worker processes. A failed write can internally de-sync the cluster. + * * Without an `await`, consecutive calls could race. An unlucky race could result in the older write overwriting the newer value. + * * Not awaiting here makes the entire cache non-consistent. The prevents many possible uses. */ @bindThis public async fetch(): Promise { @@ -177,14 +191,14 @@ export class RedisSingleCache { // Cache MISS const value = await this.fetcher(); - this.set(value); + await this.set(value); return value; } @bindThis public async refresh() { const value = await this.fetcher(); - this.set(value); + await this.set(value); // TODO: イベント発行して他プロセスのメモリキャッシュも更新できるようにする } @@ -197,18 +211,12 @@ export class MemoryKVCache { * データを持つマップ * これを直接操作するべきではない */ - public cache: Map; - private lifetime: number; - private gcIntervalHandle: NodeJS.Timeout; + private readonly cache = new Map(); + private readonly gcIntervalHandle = setInterval(() => this.gc(), 1000 * 60 * 3); // 3m - constructor(lifetime: MemoryKVCache['lifetime']) { - this.cache = new Map(); - this.lifetime = lifetime; - - this.gcIntervalHandle = setInterval(() => { - this.gc(); - }, 1000 * 60 * 3); - } + constructor( + private readonly lifetime: number, + ) {} @bindThis /** @@ -293,10 +301,14 @@ export class MemoryKVCache { @bindThis public gc(): void { const now = Date.now(); + for (const [key, { date }] of this.cache.entries()) { - if ((now - date) > this.lifetime) { - this.cache.delete(key); - } + // The map is ordered from oldest to youngest. + // We can stop once we find an entry that's still active, because all following entries must *also* be active. + const age = now - date; + if (age < this.lifetime) break; + + this.cache.delete(key); } } @@ -304,16 +316,19 @@ export class MemoryKVCache { public dispose(): void { clearInterval(this.gcIntervalHandle); } + + public get entries() { + return this.cache.entries(); + } } export class MemorySingleCache { private cachedAt: number | null = null; private value: T | undefined; - private lifetime: number; - constructor(lifetime: MemorySingleCache['lifetime']) { - this.lifetime = lifetime; - } + constructor( + private lifetime: number, + ) {} @bindThis public set(value: T): void { diff --git a/packages/backend/src/queue/processors/DeliverProcessorService.ts b/packages/backend/src/queue/processors/DeliverProcessorService.ts index 5fed07092..6f0cd718d 100644 --- a/packages/backend/src/queue/processors/DeliverProcessorService.ts +++ b/packages/backend/src/queue/processors/DeliverProcessorService.ts @@ -44,7 +44,7 @@ export class DeliverProcessorService { private queueLoggerService: QueueLoggerService, ) { this.logger = this.queueLoggerService.logger.createSubLogger('deliver'); - this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); + this.suspendedHostsCache = new MemorySingleCache(1000 * 60 * 60); // 1h } @bindThis diff --git a/packages/backend/src/queue/processors/InboxProcessorService.ts b/packages/backend/src/queue/processors/InboxProcessorService.ts index 259fd06f6..70f3e30c5 100644 --- a/packages/backend/src/queue/processors/InboxProcessorService.ts +++ b/packages/backend/src/queue/processors/InboxProcessorService.ts @@ -180,6 +180,8 @@ export class InboxProcessorService implements OnApplicationShutdown { if (signerHost !== activityIdHost) { throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`); } + } else { + throw new Bull.UnrecoverableError('skip: activity id is not a string'); } // Update stats @@ -198,7 +200,11 @@ export class InboxProcessorService implements OnApplicationShutdown { // アクティビティを処理 try { - await this.apInboxService.performActivity(authUser.user, activity, job.data.user?.id); + const result = await this.apInboxService.performActivity(authUser.user, activity, undefined, job.data.user?.id); + if (result && !result.startsWith('ok')) { + this.logger.warn(`inbox activity ignored (maybe): id=${activity.id} reason=${result}`); + return result; + } } catch (e) { if (e instanceof IdentifiableError) { if ([ @@ -206,6 +212,7 @@ export class InboxProcessorService implements OnApplicationShutdown { '689ee33f-f97c-479a-ac49-1b9f8140af99', '9f466dab-c856-48cd-9e65-ff90ff750580', '85ab9bd7-3a41-4530-959d-f07073900109', + 'd450b8a9-48e4-4dab-ae36-f4db763fda7c', ].includes(e.id)) return e.message; } throw e; diff --git a/packages/backend/src/server/ActivityPubServerService.ts b/packages/backend/src/server/ActivityPubServerService.ts index 8ba2daf79..0a6dc6370 100644 --- a/packages/backend/src/server/ActivityPubServerService.ts +++ b/packages/backend/src/server/ActivityPubServerService.ts @@ -105,7 +105,7 @@ export class ActivityPubServerService { let signature; try { - signature = httpSignature.parseRequest(request.raw, { 'headers': [] }); + signature = httpSignature.parseRequest(request.raw, { 'headers': ['(request-target)', 'host', 'date'], authorizationHeaderName: 'signature' }); } catch (e) { reply.code(401); return; diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index cc49c5ffa..22d673cd2 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -133,7 +133,7 @@ export class NodeinfoServerService { return document; }; - const cache = new MemorySingleCache>>(1000 * 60 * 10); + const cache = new MemorySingleCache>>(1000 * 60 * 10); // 10m fastify.get(nodeinfo2_1path, async (request, reply) => { const base = await cache.fetch(() => nodeinfo2(21)); diff --git a/packages/backend/src/server/api/AuthenticateService.ts b/packages/backend/src/server/api/AuthenticateService.ts index ddef8db98..690ff2e02 100644 --- a/packages/backend/src/server/api/AuthenticateService.ts +++ b/packages/backend/src/server/api/AuthenticateService.ts @@ -37,7 +37,7 @@ export class AuthenticateService implements OnApplicationShutdown { private cacheService: CacheService, ) { - this.appCache = new MemoryKVCache(Infinity); + this.appCache = new MemoryKVCache(1000 * 60 * 60 * 24 * 7); // 1w } @bindThis diff --git a/packages/backend/src/server/api/endpoints/ap/get.ts b/packages/backend/src/server/api/endpoints/ap/get.ts index d8c55de7e..14286bc23 100644 --- a/packages/backend/src/server/api/endpoints/ap/get.ts +++ b/packages/backend/src/server/api/endpoints/ap/get.ts @@ -11,6 +11,7 @@ import { ApResolverService } from '@/core/activitypub/ApResolverService.js'; export const meta = { tags: ['federation'], + requireAdmin: true, requireCredential: true, kind: 'read:federation', diff --git a/packages/backend/src/server/api/endpoints/ap/show.ts b/packages/backend/src/server/api/endpoints/ap/show.ts index d3c40dba5..cc7e24515 100644 --- a/packages/backend/src/server/api/endpoints/ap/show.ts +++ b/packages/backend/src/server/api/endpoints/ap/show.ts @@ -122,6 +122,11 @@ export default class extends Endpoint { // eslint- ])); if (local != null) return local; + const host = this.utilityService.extractDbHost(uri); + + // local object, not found in db? fail + if (this.utilityService.isSelfHost(host)) return null; + // リモートから一旦オブジェクトフェッチ const resolver = this.apResolverService.createResolver(); const object = await resolver.resolve(uri) as any; @@ -136,10 +141,11 @@ export default class extends Endpoint { // eslint- if (local != null) return local; } + // 同一ユーザーの情報を再度処理するので、使用済みのresolverを再利用してはいけない return await this.mergePack( me, isActor(object) ? await this.apPersonService.createPerson(getApId(object)) : null, - isPost(object) ? await this.apNoteService.createNote(getApId(object), undefined, true) : null, + isPost(object) ? await this.apNoteService.createNote(getApId(object), undefined, undefined, true) : null, ); } diff --git a/packages/backend/src/server/api/endpoints/users/followers.ts b/packages/backend/src/server/api/endpoints/users/followers.ts index 7ce7734f5..a8b4319a6 100644 --- a/packages/backend/src/server/api/endpoints/users/followers.ts +++ b/packages/backend/src/server/api/endpoints/users/followers.ts @@ -11,6 +11,7 @@ import { QueryService } from '@/core/QueryService.js'; import { FollowingEntityService } from '@/core/entities/FollowingEntityService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { DI } from '@/di-symbols.js'; +import { RoleService } from '@/core/RoleService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -81,6 +82,7 @@ export default class extends Endpoint { // eslint- private utilityService: UtilityService, private followingEntityService: FollowingEntityService, private queryService: QueryService, + private roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { const user = await this.usersRepository.findOneBy(ps.userId != null @@ -93,23 +95,25 @@ export default class extends Endpoint { // eslint- const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id }); - if (profile.followersVisibility === 'private') { - if (me == null || (me.id !== user.id)) { - throw new ApiError(meta.errors.forbidden); - } - } else if (profile.followersVisibility === 'followers') { - if (me == null) { - throw new ApiError(meta.errors.forbidden); - } else if (me.id !== user.id) { - const isFollowing = await this.followingsRepository.exists({ - where: { - followeeId: user.id, - followerId: me.id, - }, - }); - if (!isFollowing) { + if (profile.followersVisibility !== 'public' && !await this.roleService.isModerator(me)) { + if (profile.followersVisibility === 'private') { + if (me == null || (me.id !== user.id)) { throw new ApiError(meta.errors.forbidden); } + } else if (profile.followersVisibility === 'followers') { + if (me == null) { + throw new ApiError(meta.errors.forbidden); + } else if (me.id !== user.id) { + const isFollowing = await this.followingsRepository.exists({ + where: { + followeeId: user.id, + followerId: me.id, + }, + }); + if (!isFollowing) { + throw new ApiError(meta.errors.forbidden); + } + } } } diff --git a/packages/backend/src/server/api/endpoints/users/following.ts b/packages/backend/src/server/api/endpoints/users/following.ts index 084d27809..ee2e17a0e 100644 --- a/packages/backend/src/server/api/endpoints/users/following.ts +++ b/packages/backend/src/server/api/endpoints/users/following.ts @@ -12,6 +12,7 @@ import { QueryService } from '@/core/QueryService.js'; import { FollowingEntityService } from '@/core/entities/FollowingEntityService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { DI } from '@/di-symbols.js'; +import { RoleService } from '@/core/RoleService.js'; import { ApiError } from '../../error.js'; export const meta = { @@ -93,6 +94,7 @@ export default class extends Endpoint { // eslint- private utilityService: UtilityService, private followingEntityService: FollowingEntityService, private queryService: QueryService, + private roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { const user = await this.usersRepository.findOneBy(ps.userId != null @@ -105,23 +107,25 @@ export default class extends Endpoint { // eslint- const profile = await this.userProfilesRepository.findOneByOrFail({ userId: user.id }); - if (profile.followingVisibility === 'private') { - if (me == null || (me.id !== user.id)) { - throw new ApiError(meta.errors.forbidden); - } - } else if (profile.followingVisibility === 'followers') { - if (me == null) { - throw new ApiError(meta.errors.forbidden); - } else if (me.id !== user.id) { - const isFollowing = await this.followingsRepository.exists({ - where: { - followeeId: user.id, - followerId: me.id, - }, - }); - if (!isFollowing) { + if (profile.followingVisibility !== 'public' && !await this.roleService.isModerator(me)) { + if (profile.followingVisibility === 'private') { + if (me == null || (me.id !== user.id)) { throw new ApiError(meta.errors.forbidden); } + } else if (profile.followingVisibility === 'followers') { + if (me == null) { + throw new ApiError(meta.errors.forbidden); + } else if (me.id !== user.id) { + const isFollowing = await this.followingsRepository.exists({ + where: { + followeeId: user.id, + followerId: me.id, + }, + }); + if (!isFollowing) { + throw new ApiError(meta.errors.forbidden); + } + } } } diff --git a/packages/backend/src/server/web/UrlPreviewService.ts b/packages/backend/src/server/web/UrlPreviewService.ts index 4d8f212d8..f2be28de0 100644 --- a/packages/backend/src/server/web/UrlPreviewService.ts +++ b/packages/backend/src/server/web/UrlPreviewService.ts @@ -164,6 +164,6 @@ export class UrlPreviewService { contentLengthRequired: meta.urlPreviewRequireContentLength, }); - return this.httpRequestService.getJson(`${proxy}?${queryStr}`); + return this.httpRequestService.getJson(`${proxy}?${queryStr}`, 'application/json, */*', undefined, true); } } diff --git a/packages/backend/test/unit/activitypub.ts b/packages/backend/test/unit/activitypub.ts index 7f5e06b52..155ec671a 100644 --- a/packages/backend/test/unit/activitypub.ts +++ b/packages/backend/test/unit/activitypub.ts @@ -20,7 +20,8 @@ import { CoreModule } from '@/core/CoreModule.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { LoggerService } from '@/core/LoggerService.js'; import type { IActor, IApDocument, ICollection, IObject, IPost } from '@/core/activitypub/type.js'; -import { MiMeta, MiNote } from '@/models/_.js'; +import { MiMeta, MiNote, UserProfilesRepository } from '@/models/_.js'; +import { DI } from '@/di-symbols.js'; import { secureRndstr } from '@/misc/secure-rndstr.js'; import { DownloadService } from '@/core/DownloadService.js'; import { MetaService } from '@/core/MetaService.js'; @@ -86,6 +87,7 @@ async function createRandomRemoteUser( } describe('ActivityPub', () => { + let userProfilesRepository: UserProfilesRepository; let imageService: ApImageService; let noteService: ApNoteService; let personService: ApPersonService; @@ -127,6 +129,8 @@ describe('ActivityPub', () => { await app.init(); app.enableShutdownHooks(); + userProfilesRepository = app.get(DI.userProfilesRepository); + noteService = app.get(ApNoteService); personService = app.get(ApPersonService); rendererService = app.get(ApRendererService); @@ -169,7 +173,7 @@ describe('ActivityPub', () => { resolver.register(actor.id, actor); resolver.register(post.id, post); - const note = await noteService.createNote(post.id, resolver, true); + const note = await noteService.createNote(post.id, undefined, resolver, true); assert.deepStrictEqual(note?.uri, post.id); assert.deepStrictEqual(note.visibility, 'public'); @@ -205,6 +209,53 @@ describe('ActivityPub', () => { }); }); + describe('Collection visibility', () => { + test('Public following/followers', async () => { + const actor = createRandomActor(); + actor.following = { + id: `${actor.id}/following`, + type: 'OrderedCollection', + totalItems: 0, + first: `${actor.id}/following?page=1`, + }; + actor.followers = `${actor.id}/followers`; + + resolver.register(actor.id, actor); + resolver.register(actor.followers, { + id: actor.followers, + type: 'OrderedCollection', + totalItems: 0, + first: `${actor.followers}?page=1`, + }); + + const user = await personService.createPerson(actor.id, resolver); + const userProfile = await userProfilesRepository.findOneByOrFail({ userId: user.id }); + + assert.deepStrictEqual(userProfile.followingVisibility, 'public'); + assert.deepStrictEqual(userProfile.followersVisibility, 'public'); + }); + + test('Private following/followers', async () => { + const actor = createRandomActor(); + actor.following = { + id: `${actor.id}/following`, + type: 'OrderedCollection', + totalItems: 0, + // first: … + }; + actor.followers = `${actor.id}/followers`; + + resolver.register(actor.id, actor); + //resolver.register(actor.followers, { … }); + + const user = await personService.createPerson(actor.id, resolver); + const userProfile = await userProfilesRepository.findOneByOrFail({ userId: user.id }); + + assert.deepStrictEqual(userProfile.followingVisibility, 'private'); + assert.deepStrictEqual(userProfile.followersVisibility, 'private'); + }); + }); + describe('Renderer', () => { test('Render an announce with visibility: followers', () => { rendererService.renderAnnounce('https://example.com/notes/00example', { @@ -282,7 +333,7 @@ describe('ActivityPub', () => { resolver.register(actor.featured, featured); resolver.register(firstNote.id, firstNote); - const note = await noteService.createNote(firstNote.id as string, resolver); + const note = await noteService.createNote(firstNote.id as string, undefined, resolver); assert.strictEqual(note?.uri, firstNote.id); }); }); diff --git a/packages/frontend/src/scripts/isFfVisibleForMe.ts b/packages/frontend/src/scripts/isFfVisibleForMe.ts index 406404c46..e28e5725b 100644 --- a/packages/frontend/src/scripts/isFfVisibleForMe.ts +++ b/packages/frontend/src/scripts/isFfVisibleForMe.ts @@ -7,7 +7,7 @@ import * as Misskey from 'misskey-js'; import { $i } from '@/account.js'; export function isFollowingVisibleForMe(user: Misskey.entities.UserDetailed): boolean { - if ($i && $i.id === user.id) return true; + if ($i && ($i.id === user.id || $i.isAdmin || $i.isModerator)) return true; if (user.followingVisibility === 'private') return false; if (user.followingVisibility === 'followers' && !user.isFollowing) return false; @@ -15,7 +15,7 @@ export function isFollowingVisibleForMe(user: Misskey.entities.UserDetailed): bo return true; } export function isFollowersVisibleForMe(user: Misskey.entities.UserDetailed): boolean { - if ($i && $i.id === user.id) return true; + if ($i && ($i.id === user.id || $i.isAdmin || $i.isModerator)) return true; if (user.followersVisibility === 'private') return false; if (user.followersVisibility === 'followers' && !user.isFollowing) return false;