perf(timeline): Optimizing for CDN Caching (MisskeyIO#834)

Co-authored-by: あわわわとーにゅ <17376330+u1-liquid@users.noreply.github.com>

packages/backend/src/server/api/stream/channels/antenna.ts

@@ -15,6 +15,7 @@ class AntennaChannel extends Channel {
 	public static readonly requireCredential = true as const;
 	public static readonly kind = 'read:account';
 	private antennaId: string;
+	private idOnly: boolean;
 
 	constructor(
 		private noteEntityService: NoteEntityService,
@@ -29,6 +30,7 @@ class AntennaChannel extends Channel {
 	@bindThis
 	public async init(params: any) {
 		this.antennaId = params.antennaId as string;
+		this.idOnly = params.idOnly ?? false;
 
 		// Subscribe stream
 		this.subscriber.on(`antennaStream:${this.antennaId}`, this.onEvent);
@@ -49,9 +51,13 @@ class AntennaChannel extends Channel {
 
 			if (this.isNoteMutedOrBlocked(note)) return;
 
-			this.connection.cacheNote(note);
+			if (this.idOnly && ['public', 'home'].includes(note.visibility)) {
-
+				const idOnlyNote = { id: note.id };
-			this.send('note', note);
+				this.send('note', idOnlyNote);
+			} else {
+				this.connection.cacheNote(note);
+				this.send('note', note);
+			}
 		} else {
 			this.send(data.type, data.body);
 		}

packages/backend/src/server/api/stream/channels/channel.ts

@@ -15,6 +15,7 @@ class ChannelChannel extends Channel {
 	public static readonly shouldShare = false;
 	public static readonly requireCredential = false as const;
 	private channelId: string;
+	private idOnly: boolean;
 
 	constructor(
 		private noteEntityService: NoteEntityService,
@@ -29,6 +30,7 @@ class ChannelChannel extends Channel {
 	@bindThis
 	public async init(params: any) {
 		this.channelId = params.channelId as string;
+		this.idOnly = params.idOnly ?? false;
 
 		// Subscribe stream
 		this.subscriber.on('notesStream', this.onNote);
@@ -55,9 +57,13 @@ class ChannelChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
+		if (this.idOnly && ['public', 'home'].includes(note.visibility)) {
-
+			const idOnlyNote = { id: note.id };
-		this.send('note', note);
+			this.send('note', idOnlyNote);
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/api/stream/channels/global-timeline.ts

@@ -18,6 +18,7 @@ class GlobalTimelineChannel extends Channel {
 	public static readonly requireCredential = false as const;
 	private withRenotes: boolean;
 	private withFiles: boolean;
+	private idOnly: boolean;
 
 	constructor(
 		private metaService: MetaService,
@@ -38,6 +39,7 @@ class GlobalTimelineChannel extends Channel {
 
 		this.withRenotes = params.withRenotes ?? true;
 		this.withFiles = params.withFiles ?? false;
+		this.idOnly = params.idOnly ?? false;
 
 		// Subscribe events
 		this.subscriber.on('notesStream', this.onNote);
@@ -85,9 +87,13 @@ class GlobalTimelineChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
+		if (this.idOnly && ['public', 'home'].includes(note.visibility)) {
-
+			const idOnlyNote = { id: note.id };
-		this.send('note', note);
+			this.send('note', idOnlyNote);
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/api/stream/channels/home-timeline.ts

@@ -17,6 +17,7 @@ class HomeTimelineChannel extends Channel {
 	public static readonly kind = 'read:account';
 	private withRenotes: boolean;
 	private withFiles: boolean;
+	private idOnly: boolean;
 
 	constructor(
 		private noteEntityService: NoteEntityService,
@@ -32,6 +33,7 @@ class HomeTimelineChannel extends Channel {
 	public async init(params: any) {
 		this.withRenotes = params.withRenotes ?? true;
 		this.withFiles = params.withFiles ?? false;
+		this.idOnly = params.idOnly ?? false;
 
 		this.subscriber.on('notesStream', this.onNote);
 	}
@@ -89,9 +91,13 @@ class HomeTimelineChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
+		if (this.idOnly && ['public', 'home'].includes(note.visibility)) {
-
+			const idOnlyNote = { id: note.id };
-		this.send('note', note);
+			this.send('note', idOnlyNote);
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/api/stream/channels/hybrid-timeline.ts

@@ -20,6 +20,7 @@ class HybridTimelineChannel extends Channel {
 	private withRenotes: boolean;
 	private withReplies: boolean;
 	private withFiles: boolean;
+	private idOnly: boolean;
 
 	constructor(
 		private metaService: MetaService,
@@ -41,6 +42,7 @@ class HybridTimelineChannel extends Channel {
 		this.withRenotes = params.withRenotes ?? true;
 		this.withReplies = params.withReplies ?? false;
 		this.withFiles = params.withFiles ?? false;
+		this.idOnly = params.idOnly ?? false;
 
 		// Subscribe events
 		this.subscriber.on('notesStream', this.onNote);
@@ -103,9 +105,13 @@ class HybridTimelineChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
+		if (this.idOnly && ['public', 'home'].includes(note.visibility)) {
-
+			const idOnlyNote = { id: note.id };
-		this.send('note', note);
+			this.send('note', idOnlyNote);
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/api/stream/channels/local-timeline.ts

@@ -19,6 +19,7 @@ class LocalTimelineChannel extends Channel {
 	private withRenotes: boolean;
 	private withReplies: boolean;
 	private withFiles: boolean;
+	private idOnly: boolean;
 
 	constructor(
 		private metaService: MetaService,
@@ -40,6 +41,7 @@ class LocalTimelineChannel extends Channel {
 		this.withRenotes = params.withRenotes ?? true;
 		this.withReplies = params.withReplies ?? false;
 		this.withFiles = params.withFiles ?? false;
+		this.idOnly = params.idOnly ?? false;
 
 		// Subscribe events
 		this.subscriber.on('notesStream', this.onNote);
@@ -88,9 +90,13 @@ class LocalTimelineChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
+		if (this.idOnly && ['public', 'home'].includes(note.visibility)) {
-
+			const idOnlyNote = { id: note.id };
-		this.send('note', note);
+			this.send('note', idOnlyNote);
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/api/stream/channels/role-timeline.ts

@@ -16,6 +16,7 @@ class RoleTimelineChannel extends Channel {
 	public static readonly shouldShare = false;
 	public static readonly requireCredential = false as const;
 	private roleId: string;
+	private idOnly: boolean;
 
 	constructor(
 		private noteEntityService: NoteEntityService,
@@ -31,6 +32,7 @@ class RoleTimelineChannel extends Channel {
 	@bindThis
 	public async init(params: any) {
 		this.roleId = params.roleId as string;
+		this.idOnly = params.idOnly ?? false;
 
 		this.subscriber.on(`roleTimelineStream:${this.roleId}`, this.onEvent);
 	}
@@ -71,9 +73,13 @@ class RoleTimelineChannel extends Channel {
 				}
 			}
 
-			this.connection.cacheNote(note);
+			if (this.idOnly && ['public', 'home'].includes(note.visibility)) {
-
+				const idOnlyNote = { id: note.id };
-			this.send('note', note);
+				this.send('note', idOnlyNote);
+			} else {
+				this.connection.cacheNote(note);
+				this.send('note', note);
+			}
 		} else {
 			this.send(data.type, data.body);
 		}

packages/backend/src/server/api/stream/channels/user-list.ts

@@ -21,6 +21,7 @@ class UserListChannel extends Channel {
 	private listUsersClock: NodeJS.Timeout;
 	private withFiles: boolean;
 	private withRenotes: boolean;
+	private idOnly: boolean;
 
 	constructor(
 		private userListsRepository: UserListsRepository,
@@ -40,6 +41,7 @@ class UserListChannel extends Channel {
 		this.listId = params.listId as string;
 		this.withFiles = params.withFiles ?? false;
 		this.withRenotes = params.withRenotes ?? true;
+		this.idOnly = params.idOnly ?? false;
 
 		// Check existence and owner
 		const listExist = await this.userListsRepository.exists({
@@ -128,9 +130,13 @@ class UserListChannel extends Channel {
 			}
 		}
 
-		this.connection.cacheNote(note);
+		if (this.idOnly && ['public', 'home'].includes(note.visibility)) {
-
+			const idOnlyNote = { id: note.id };
-		this.send('note', note);
+			this.send('note', idOnlyNote);
+		} else {
+			this.connection.cacheNote(note);
+			this.send('note', note);
+		}
 	}
 
 	@bindThis

packages/backend/src/server/web/ClientServerService.ts

@@ -616,6 +616,81 @@ export class ClientServerService {
 			}
 		});
 
+		fastify.get<{ Params: { note: string; } }>('/notes/:note.json', async (request, reply) => {
+			const note = await this.notesRepository.findOneBy({
+				id: request.params.note,
+				visibility: In(['public', 'home']),
+			});
+
+			if (note) {
+				try {
+					const _note = await this.noteEntityService.pack(note, null);
+					reply.header('Content-Type', 'application/json; charset=utf-8');
+					reply.header('Cache-Control', 'public, max-age=600');
+					return reply.send(_note);
+				} catch (err) {
+					reply.header('Cache-Control', 'max-age=10, must-revalidate');
+					if (err instanceof IdentifiableError) {
+						this.clientLoggerService.logger.error(`Internal error occurred in ${request.routeOptions.url}: ${err.message}`, {
+							path: request.routeOptions.url,
+							params: request.params,
+							query: request.query,
+							id: err.id,
+							error: {
+								message: err.message,
+								code: 'INTERNAL_ERROR',
+								stack: err.stack,
+							},
+						});
+						const httpStatusCode = err.id === '85ab9bd7-3a41-4530-959d-f07073900109' ? 403 : 500;
+						reply.code(httpStatusCode);
+						return reply.send({
+							message: err.message,
+							code: 'INTERNAL_ERROR',
+							id: err.id,
+							kind: 'server',
+							httpStatusCode,
+							info: {
+								message: err.message,
+								code: err.name,
+								id: err.id,
+							},
+						});
+					} else {
+						const error = err as Error;
+						const errId = randomUUID();
+						this.clientLoggerService.logger.error(`Internal error occurred in ${request.routeOptions.url}: ${error.message}`, {
+							path: request.routeOptions.url,
+							params: request.params,
+							query: request.query,
+							id: errId,
+							error: {
+								message: error.message,
+								code: error.name,
+								stack: error.stack,
+							},
+						});
+						reply.code(500);
+						return reply.send({
+							message: 'Internal error occurred. Please contact us if the error persists.',
+							code: 'INTERNAL_ERROR',
+							id: 'b9f2a7f9-fe64-434b-9484-cb1f804d1a80',
+							kind: 'server',
+							httpStatusCode: 500,
+							info: {
+								message: error.message,
+								code: error.name,
+								id: errId,
+							},
+						});
+					}
+				}
+			} else {
+				reply.code(404);
+				return;
+			}
+		});
+
 		// Page
 		fastify.get<{ Params: { user: string; page: string; } }>('/@:user/pages/:page', async (request, reply) => {
 			const { username, host } = Acct.parse(request.params.user);

packages/backend/test/utils.ts

@@ -400,7 +400,7 @@ export const waitFire = async <C extends keyof misskey.Channels>(user: UserToken
 					if (timer) clearTimeout(timer);
 					res(true);
 				}
-			}, params);
+			}, { ...params, idOnly: false });
 		} catch (e) {
 			rej(e);
 		}

packages/frontend/src/components/MkTimeline.vue

@@ -18,15 +18,17 @@ SPDX-License-Identifier: AGPL-3.0-only
 
 <script lang="ts" setup>
 import { computed, watch, onUnmounted, provide, shallowRef } from 'vue';
+import { time as gtagTime } from 'vue-gtag';
 import * as Misskey from 'misskey-js';
 import MkNotes from '@/components/MkNotes.vue';
 import MkPullToRefresh from '@/components/MkPullToRefresh.vue';
 import { useStream } from '@/stream.js';
 import * as sound from '@/scripts/sound.js';
-import { $i } from '@/account.js';
+import { $i, iAmModerator } from '@/account.js';
 import { instance } from '@/instance.js';
 import { defaultStore } from '@/store.js';
 import { Paging } from '@/components/MkPagination.vue';
+import { generateClientTransactionId } from '@/scripts/misskey-api.js';
 
 const props = withDefaults(defineProps<{
 	src: 'home' | 'local' | 'media' | 'social' | 'global' | 'mentions' | 'directs' | 'list' | 'antenna' | 'channel' | 'role';
@@ -68,9 +70,36 @@ const tlComponent = shallowRef<InstanceType<typeof MkNotes>>();
 
 let tlNotesCount = 0;
 
-function prepend(note) {
+async function prepend(data) {
 	if (tlComponent.value == null) return;
 
+	let note = data;
+
+	// チェックするプロパティはなんでも良い
+	// idOnlyが有効でid以外が存在しない場合は取得する
+	if (!data.visibility) {
+		const initiateTime = Date.now();
+		const res = await window.fetch(`/notes/${data.id}.json`, {
+			method: 'GET',
+			credentials: 'omit',
+			headers: {
+				'Authorization': 'anonymous',
+				'X-Client-Transaction-Id': generateClientTransactionId('misskey'),
+			},
+		}).then(res => {
+			if (instance.googleAnalyticsId) {
+				gtagTime({
+					name: 'api-get',
+					event_category: `/notes/${data.id}.json`,
+					value: Date.now() - initiateTime,
+				});
+			}
+			return res;
+		});
+		if (!res.ok) return;
+		note = await res.json();
+	}
+
 	tlNotesCount++;
 
 	if (instance.notesPerOneAd > 0 && tlNotesCount % instance.notesPerOneAd === 0) {
@@ -89,6 +118,7 @@ function prepend(note) {
 let connection: Misskey.ChannelConnection | null = null;
 let connection2: Misskey.ChannelConnection | null = null;
 let paginationQuery: Paging | null = null;
+const idOnly = !iAmModerator;
 
 const stream = useStream();
 
@@ -97,11 +127,13 @@ function connectChannel() {
 		if (props.antenna == null) return;
 		connection = stream.useChannel('antenna', {
 			antennaId: props.antenna,
+			idOnly: idOnly,
 		});
 	} else if (props.src === 'home') {
 		connection = stream.useChannel('homeTimeline', {
 			withRenotes: props.withRenotes,
 			withFiles: props.onlyFiles ? true : undefined,
+			idOnly: idOnly,
 		});
 		connection2 = stream.useChannel('main');
 	} else if (props.src === 'local') {
@@ -109,23 +141,27 @@ function connectChannel() {
 			withRenotes: props.withRenotes,
 			withReplies: props.withReplies,
 			withFiles: props.onlyFiles ? true : undefined,
+			idOnly: idOnly,
 		});
 	} else if (props.src === 'media') {
 		connection = stream.useChannel('hybridTimeline', {
 			withRenotes: props.withRenotes,
 			withReplies: props.withReplies,
 			withFiles: true,
+			idOnly: idOnly,
 		});
 	} else if (props.src === 'social') {
 		connection = stream.useChannel('hybridTimeline', {
 			withRenotes: props.withRenotes,
 			withReplies: props.withReplies,
 			withFiles: props.onlyFiles ? true : undefined,
+			idOnly: idOnly,
 		});
 	} else if (props.src === 'global') {
 		connection = stream.useChannel('globalTimeline', {
 			withRenotes: props.withRenotes,
 			withFiles: props.onlyFiles ? true : undefined,
+			idOnly: idOnly,
 		});
 	} else if (props.src === 'mentions') {
 		connection = stream.useChannel('main');
@@ -144,16 +180,19 @@ function connectChannel() {
 			withRenotes: props.withRenotes,
 			withFiles: props.onlyFiles ? true : undefined,
 			listId: props.list,
+			idOnly: idOnly,
 		});
 	} else if (props.src === 'channel') {
 		if (props.channel == null) return;
 		connection = stream.useChannel('channel', {
 			channelId: props.channel,
+			idOnly: idOnly,
 		});
 	} else if (props.src === 'role') {
 		if (props.role == null) return;
 		connection = stream.useChannel('roleTimeline', {
 			roleId: props.role,
+			idOnly: idOnly,
 		});
 	}
 	if (props.src !== 'directs' && props.src !== 'mentions') connection?.on('note', prepend);

packages/misskey-js/etc/misskey-js.api.md

@@ -642,6 +642,7 @@ export type Channels = {
         params: {
             withRenotes?: boolean;
             withFiles?: boolean;
+            idOnly?: boolean;
         };
         events: {
             note: (payload: Note) => void;
@@ -653,6 +654,7 @@ export type Channels = {
             withRenotes?: boolean;
             withReplies?: boolean;
             withFiles?: boolean;
+            idOnly?: boolean;
         };
         events: {
             note: (payload: Note) => void;
@@ -664,6 +666,7 @@ export type Channels = {
             withRenotes?: boolean;
             withReplies?: boolean;
             withFiles?: boolean;
+            idOnly?: boolean;
         };
         events: {
             note: (payload: Note) => void;
@@ -674,6 +677,7 @@ export type Channels = {
         params: {
             withRenotes?: boolean;
             withFiles?: boolean;
+            idOnly?: boolean;
         };
         events: {
             note: (payload: Note) => void;
@@ -685,6 +689,7 @@ export type Channels = {
             listId: string;
             withFiles?: boolean;
             withRenotes?: boolean;
+            idOnly?: boolean;
         };
         events: {
             note: (payload: Note) => void;
@@ -703,6 +708,7 @@ export type Channels = {
     roleTimeline: {
         params: {
             roleId: string;
+            idOnly?: boolean;
         };
         events: {
             note: (payload: Note) => void;
@@ -712,6 +718,7 @@ export type Channels = {
     antenna: {
         params: {
             antennaId: string;
+            idOnly?: boolean;
         };
         events: {
             note: (payload: Note) => void;
@@ -721,6 +728,7 @@ export type Channels = {
     channel: {
         params: {
             channelId: string;
+            idOnly?: boolean;
         };
         events: {
             note: (payload: Note) => void;

packages/misskey-js/src/streaming.types.ts

@@ -64,6 +64,7 @@ export type Channels = {
 		params: {
 			withRenotes?: boolean;
 			withFiles?: boolean;
+			idOnly?: boolean,
 		};
 		events: {
 			note: (payload: Note) => void;
@@ -75,6 +76,7 @@ export type Channels = {
 			withRenotes?: boolean;
 			withReplies?: boolean;
 			withFiles?: boolean;
+			idOnly?: boolean,
 		};
 		events: {
 			note: (payload: Note) => void;
@@ -86,6 +88,7 @@ export type Channels = {
 			withRenotes?: boolean;
 			withReplies?: boolean;
 			withFiles?: boolean;
+			idOnly?: boolean,
 		};
 		events: {
 			note: (payload: Note) => void;
@@ -96,6 +99,7 @@ export type Channels = {
 		params: {
 			withRenotes?: boolean;
 			withFiles?: boolean;
+			idOnly?: boolean,
 		};
 		events: {
 			note: (payload: Note) => void;
@@ -107,6 +111,7 @@ export type Channels = {
 			listId: string;
 			withFiles?: boolean;
 			withRenotes?: boolean;
+			idOnly?: boolean,
 		};
 		events: {
 			note: (payload: Note) => void;
@@ -125,6 +130,7 @@ export type Channels = {
 	roleTimeline: {
 		params: {
 			roleId: string;
+			idOnly?: boolean,
 		};
 		events: {
 			note: (payload: Note) => void;
@@ -134,6 +140,7 @@ export type Channels = {
 	antenna: {
 		params: {
 			antennaId: string;
+			idOnly?: boolean,
 		};
 		events: {
 			note: (payload: Note) => void;
@@ -143,6 +150,7 @@ export type Channels = {
 	channel: {
 		params: {
 			channelId: string;
+			idOnly?: boolean,
 		};
 		events: {
 			note: (payload: Note) => void;