diff --git a/packages/backend/migration/1706827327619-add-language-selector.js b/packages/backend/migration/1706827327619-add-language-selector.js new file mode 100644 index 000000000..840ec7711 --- /dev/null +++ b/packages/backend/migration/1706827327619-add-language-selector.js @@ -0,0 +1,11 @@ +export class AddLanguageSelector1706827327619 { + async up(queryRunner) { + await queryRunner.query( + `ALTER TABLE "note" ADD "lang" character varying(10)`, + ); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" DROP COLUMN "lang"`); + } +} diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 8a43e100d..708f80d14 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -72,6 +72,7 @@ import { trackPromise } from '@/misc/promise-tracker.js'; import { isNotNull } from '@/misc/is-not-null.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { CollapsedQueue } from '@/misc/collapsed-queue.js'; +import { SUPPORTED_POST_LOCALES } from "@/misc/langmap.js"; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -342,6 +343,15 @@ export class NoteCreateService implements OnApplicationShutdown { data.text = null; } + if (data.lang) { + if (!Object.keys(SUPPORTED_POST_LOCALES).includes(data.lang.toLowerCase())) { + throw new Error("invalid lang param"); + } + data.lang = data.lang.toLowerCase(); + } else { + data.lang = null; + } + let tags = data.apHashtags; let emojis = data.apEmojis; let mentionedUsers = data.apMentions; @@ -458,6 +468,7 @@ export class NoteCreateService implements OnApplicationShutdown { : null, name: data.name, text: data.text, + lang: data.lang, hasPoll: data.poll != null, cw: data.cw ?? null, tags: tags.map(tag => normalizeForSearch(tag)), diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 9effcfac9..7eee22e4e 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -39,6 +39,7 @@ import { ApQuestionService } from './ApQuestionService.js'; import { ApImageService } from './ApImageService.js'; import type { Resolver } from '../ApResolverService.js'; import type { IObject, IPost } from '../type.js'; +import {Obj} from "@/misc/json-schema.js"; @Injectable() export class ApNoteService { @@ -204,12 +205,17 @@ export class ApNoteService { // テキストのパース let text: string | null = null; - if (note.source?.mediaType === 'text/x.misskeymarkdown' && typeof note.source.content === 'string') { - text = note.source.content; + let lang: string | null = null; + if (note.source?.mediaType === 'text/x.misskeymarkdown' && (typeof note.source.content === 'string' || note.source.contentMap)) { + const guessed = this.guessLang(note.source); + text = guessed.text; + lang = guessed.lang; } else if (typeof note._misskey_content !== 'undefined') { text = note._misskey_content; - } else if (typeof note.content === 'string') { - text = this.apMfmService.htmlToMfm(note.content, note.tag); + } else if (typeof note.content === 'string' || note.contentMap) { + const guessed = this.guessLang(note); + lang = guessed.lang; + if (guessed.text) text = this.apMfmService.htmlToMfm(guessed.text, note.tag); } const poll = await this.apQuestionService.extractPollFromQuestion(note, resolver).catch(() => undefined); @@ -343,6 +349,7 @@ export class ApNoteService { name: note.name, cw, text, + lang, localOnly: false, visibility, visibleUsers, @@ -460,4 +467,40 @@ export class ApNoteService { }).then(x => this.emojisRepository.findOneByOrFail(x.identifiers[0])); })); } + + @bindThis + private guessLang(source: { contentMap?: Record, content?: string | null }): { lang: string | null, text: string | null } { + // do we have a map? + if (source.contentMap) { + const entries = Object.entries(source.contentMap); + + // only one entry: take that + if (entries.length === 1) { + return { lang: entries[0][0], text: entries[0][1] }; + } + + // did the sender indicate a preferred language? + if (source.content) { + for (const e of entries) { + if (e[1] === source.content) { + return { lang: e[0], text: e[1] }; + } + } + } + + // can we find one of *our* preferred languages? + // for (const prefLang of this.config.langPref) { + // if (source.contentMap[prefLang]) { + // return { lang: prefLang, text: source.contentMap[prefLang] }; + // } + // } + + // bah, just pick one + return { lang: entries[0][0], text: entries[0][1] }; + } + + // no map, so we don't know the language, just take whatever + // content we got + return { lang: null, text: source.content ?? null }; + } } diff --git a/packages/backend/src/core/activitypub/type.ts b/packages/backend/src/core/activitypub/type.ts index 16812b7a4..aa1d7c306 100644 --- a/packages/backend/src/core/activitypub/type.ts +++ b/packages/backend/src/core/activitypub/type.ts @@ -11,6 +11,7 @@ export interface IObject { type: string | string[]; id?: string; name?: string | null; + contentMap?: Record; summary?: string; _misskey_summary?: string; published?: string; @@ -122,6 +123,7 @@ export interface IPost extends IObject { type: 'Note' | 'Question' | 'Article' | 'Audio' | 'Document' | 'Image' | 'Page' | 'Video' | 'Event'; source?: { content: string; + contentMap?: Record; mediaType: string; }; _misskey_quote?: string; @@ -134,6 +136,7 @@ export interface IQuestion extends IObject { actor: string; source?: { content: string; + contentMap?: Record; mediaType: string; }; _misskey_quote?: string; diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index f111b6540..b1526d468 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -331,6 +331,7 @@ export class NoteEntityService implements OnModuleInit { userId: note.userId, user: this.userEntityService.pack(note.user ?? note.userId, me), text: text, + lang: note.lang, cw: note.cw, visibility: note.visibility, localOnly: note.localOnly, diff --git a/packages/backend/src/misc/langmap.ts b/packages/backend/src/misc/langmap.ts index 5ff933865..4e9c2f69a 100644 --- a/packages/backend/src/misc/langmap.ts +++ b/packages/backend/src/misc/langmap.ts @@ -669,3 +669,631 @@ export const langmap = { nativeName: 'isiZulu', }, }; + + +export const ISO_639_1 = { + aa: { + nativeName: 'Afaraf', + }, + ab: { + nativeName: 'аҧсуа бызшәа', + }, + ae: { + nativeName: 'avesta', + }, + af: { + nativeName: 'Afrikaans', + }, + ak: { + nativeName: 'Akan', + }, + am: { + nativeName: 'አማርኛ', + }, + an: { + nativeName: 'aragonés', + }, + ar: { + nativeName: 'اللغة العربية', + }, + as: { + nativeName: 'অসমীয়া', + }, + av: { + nativeName: 'авар мацӀ', + }, + ay: { + nativeName: 'aymar aru', + }, + az: { + nativeName: 'azərbaycan dili', + }, + ba: { + nativeName: 'башҡорт теле', + }, + be: { + nativeName: 'беларуская мова', + }, + bg: { + nativeName: 'български език', + }, + bh: { + nativeName: 'भोजपुरी', + }, + bi: { + nativeName: 'Bislama', + }, + bm: { + nativeName: 'bamanankan', + }, + bn: { + nativeName: 'বাংলা', + }, + bo: { + nativeName: 'བོད་ཡིག', + }, + br: { + nativeName: 'brezhoneg', + }, + bs: { + nativeName: 'bosanski jezik', + }, + ca: { + nativeName: 'Català', + }, + ce: { + nativeName: 'нохчийн мотт', + }, + ch: { + nativeName: 'Chamoru', + }, + co: { + nativeName: 'corsu', + }, + cr: { + nativeName: 'ᓀᐦᐃᔭᐍᐏᐣ', + }, + cs: { + nativeName: 'čeština', + }, + cu: { + nativeName: 'ѩзыкъ словѣньскъ', + }, + cv: { + nativeName: 'чӑваш чӗлхи', + }, + cy: { + nativeName: 'Cymraeg', + }, + da: { + nativeName: 'dansk', + }, + de: { + nativeName: 'Deutsch', + }, + dv: { + nativeName: 'Dhivehi', + }, + dz: { + nativeName: 'རྫོང་ཁ', + }, + ee: { + nativeName: 'Eʋegbe', + }, + el: { + nativeName: 'Ελληνικά', + }, + en: { + nativeName: 'English', + }, + eo: { + nativeName: 'Esperanto', + }, + es: { + nativeName: 'Español', + }, + et: { + nativeName: 'eesti', + }, + eu: { + nativeName: 'euskara', + }, + fa: { + nativeName: 'فارسی', + }, + ff: { + nativeName: 'Fulfulde', + }, + fi: { + nativeName: 'suomi', + }, + fj: { + nativeName: 'Vakaviti', + }, + fo: { + nativeName: 'føroyskt', + }, + fr: { + nativeName: 'Français', + }, + fy: { + nativeName: 'Frysk', + }, + ga: { + nativeName: 'Gaeilge', + }, + gd: { + nativeName: 'Gàidhlig', + }, + gl: { + nativeName: 'galego', + }, + gu: { + nativeName: 'ગુજરાતી', + }, + gv: { + nativeName: 'Gaelg', + }, + ha: { + nativeName: 'هَوُسَ', + }, + he: { + nativeName: 'עברית', + }, + hi: { + nativeName: 'हिन्दी', + }, + ho: { + nativeName: 'Hiri Motu', + }, + hr: { + nativeName: 'Hrvatski', + }, + ht: { + nativeName: 'Kreyòl ayisyen', + }, + hu: { + nativeName: 'magyar', + }, + hy: { + nativeName: 'Հայերեն', + }, + hz: { + nativeName: 'Otjiherero', + }, + ia: { + nativeName: 'Interlingua', + }, + id: { + nativeName: 'Bahasa Indonesia', + }, + ie: { + nativeName: 'Interlingue', + }, + ig: { + nativeName: 'Asụsụ Igbo', + }, + ii: { + nativeName: 'ꆈꌠ꒿ Nuosuhxop', + }, + ik: { + nativeName: 'Iñupiaq', + }, + io: { + nativeName: 'Ido', + }, + is: { + nativeName: 'Íslenska', + }, + it: { + nativeName: 'Italiano', + }, + iu: { + nativeName: 'ᐃᓄᒃᑎᑐᑦ', + }, + ja: { + nativeName: '日本語', + }, + jv: { + nativeName: 'basa Jawa', + }, + ka: { + nativeName: 'ქართული', + }, + kg: { + nativeName: 'Kikongo', + }, + ki: { + nativeName: 'Gĩkũyũ', + }, + kj: { + nativeName: 'Kuanyama', + }, + kk: { + nativeName: 'қазақ тілі', + }, + kl: { + nativeName: 'kalaallisut', + }, + km: { + nativeName: 'ខេមរភាសា', + }, + kn: { + nativeName: 'ಕನ್ನಡ', + }, + ko: { + nativeName: '한국어', + }, + kr: { + nativeName: 'Kanuri', + }, + ks: { + nativeName: 'कश्मीरी', + }, + ku: { + nativeName: 'Kurmancî', + }, + kv: { + nativeName: 'коми кыв', + }, + kw: { + nativeName: 'Kernewek', + }, + ky: { + nativeName: 'Кыргызча', + }, + la: { + nativeName: 'latine', + }, + lb: { + nativeName: 'Lëtzebuergesch', + }, + lg: { + nativeName: 'Luganda', + }, + li: { + nativeName: 'Limburgs', + }, + ln: { + nativeName: 'Lingála', + }, + lo: { + nativeName: 'ລາວ', + }, + lt: { + nativeName: 'lietuvių kalba', + }, + lu: { + nativeName: 'Tshiluba', + }, + lv: { + nativeName: 'latviešu valoda', + }, + mg: { + nativeName: 'fiteny malagasy', + }, + mh: { + nativeName: 'Kajin M̧ajeļ', + }, + mi: { + nativeName: 'te reo Māori', + }, + mk: { + nativeName: 'македонски јазик', + }, + ml: { + nativeName: 'മലയാളം', + }, + mn: { + nativeName: 'Монгол хэл', + }, + mr: { + nativeName: 'मराठी', + }, + ms: { + nativeName: 'Bahasa Melayu', + }, + 'ms-Arab': { + nativeName: 'بهاس ملايو', + }, + mt: { + nativeName: 'Malti', + }, + my: { + nativeName: 'ဗမာစာ', + }, + na: { + nativeName: 'Ekakairũ Naoero', + }, + nb: { + nativeName: 'Norsk bokmål', + }, + nd: { + nativeName: 'isiNdebele', + }, + ne: { + nativeName: 'नेपाली', + }, + ng: { + nativeName: 'Owambo', + }, + nl: { + nativeName: 'Nederlands', + }, + nn: { + nativeName: 'Norsk Nynorsk', + }, + no: { + nativeName: 'Norsk', + }, + nr: { + nativeName: 'isiNdebele', + }, + nv: { + nativeName: 'Diné bizaad', + }, + ny: { + nativeName: 'chiCheŵa', + }, + oc: { + nativeName: 'occitan', + }, + oj: { + nativeName: 'ᐊᓂᔑᓈᐯᒧᐎᓐ', + }, + om: { + nativeName: 'Afaan Oromoo', + }, + or: { + nativeName: 'ଓଡ଼ିଆ', + }, + os: { + nativeName: 'ирон æвзаг', + }, + pa: { + nativeName: 'ਪੰਜਾਬੀ', + }, + pi: { + nativeName: 'पाऴि', + }, + pl: { + nativeName: 'Polski', + }, + ps: { + nativeName: 'پښتو', + }, + pt: { + nativeName: 'Português', + }, + qu: { + nativeName: 'Runa Simi', + }, + rm: { + nativeName: 'rumantsch grischun', + }, + rn: { + nativeName: 'Ikirundi', + }, + ro: { + nativeName: 'Română', + }, + ru: { + nativeName: 'Русский', + }, + rw: { + nativeName: 'Ikinyarwanda', + }, + sa: { + nativeName: 'संस्कृतम्', + }, + sc: { + nativeName: 'sardu', + }, + sd: { + nativeName: 'सिन्धी', + }, + se: { + nativeName: 'Davvisámegiella', + }, + sg: { + nativeName: 'yângâ tî sängö', + }, + si: { + nativeName: 'සිංහල', + }, + sk: { + nativeName: 'slovenčina', + }, + sl: { + nativeName: 'slovenščina', + }, + sn: { + nativeName: 'chiShona', + }, + so: { + nativeName: 'Soomaaliga', + }, + sq: { + nativeName: 'Shqip', + }, + sr: { + nativeName: 'српски језик', + }, + ss: { + nativeName: 'SiSwati', + }, + st: { + nativeName: 'Sesotho', + }, + su: { + nativeName: 'Basa Sunda', + }, + sv: { + nativeName: 'Svenska', + }, + sw: { + nativeName: 'Kiswahili', + }, + ta: { + nativeName: 'தமிழ்', + }, + te: { + nativeName: 'తెలుగు', + }, + tg: { + nativeName: 'тоҷикӣ', + }, + th: { + nativeName: 'ไทย', + }, + ti: { + nativeName: 'ትግርኛ', + }, + tk: { + nativeName: 'Türkmen', + }, + tl: { + nativeName: 'Tagalog', + }, + tn: { + nativeName: 'Setswana', + }, + to: { + nativeName: 'faka Tonga', + }, + tr: { + nativeName: 'Türkçe', + }, + ts: { + nativeName: 'Xitsonga', + }, + tt: { + nativeName: 'татар теле', + }, + tw: { + nativeName: 'Twi', + }, + ty: { + nativeName: 'Reo Tahiti', + }, + ug: { + nativeName: 'ئۇيغۇرچە‎', + }, + uk: { + nativeName: 'Українська', + }, + ur: { + nativeName: 'اردو', + }, + vi: { + nativeName: 'Tiếng Việt', + }, + vo: { + nativeName: 'Volapük', + }, + wa: { + nativeName: 'Walon', + }, + wo: { + nativeName: 'Wollof', + }, + xh: { + nativeName: 'isiXhosa', + }, + yi: { + nativeName: 'ייִדיש', + }, + zu: { + nativeName: 'isiZulu', + }, +}; + +export const ISO_639_3 = { + ast: { + nativeName: 'Asturianu', + }, + chr: { + nativeName: 'ᏣᎳᎩ ᎦᏬᏂᎯᏍᏗ', + }, + ckb: { + nativeName: 'سۆرانی', + }, + cnr: { + nativeName: 'crnogorski', + }, + csb: { + nativeName: 'Kaszëbsczi', + }, + gsw: { + nativeName: 'Schwiizertütsch', + }, + jbo: { + nativeName: 'la .lojban.', + }, + kab: { + nativeName: 'Taqbaylit', + }, + ldn: { + nativeName: 'Láadan', + }, + lfn: { + nativeName: 'lingua franca nova', + }, + moh: { + nativeName: 'Kanienʼkéha', + }, + nds: { + nativeName: 'Plattdüütsch', + }, + pdc: { + nativeName: 'Pennsilfaani-Deitsch', + }, + sco: { + nativeName: 'Scots', + }, + sma: { + nativeName: 'Åarjelsaemien Gïele', + }, + smj: { + nativeName: 'Julevsámegiella', + }, + szl: { + nativeName: 'ślůnsko godka', + }, + tok: { + nativeName: 'toki pona', + }, + vai: { + nativeName: 'ꕙꔤ', + }, + xal: { + nativeName: 'Хальмг келн', + }, + zba: { + nativeName: 'باليبلن', + }, + zgh: { + nativeName: 'ⵜⴰⵎⴰⵣⵉⵖⵜ', + }, +}; + +export const ISO_639_1_REGIONAL = { + 'zh-CN': { + nativeName: '简体中文', + }, + 'zh-HK': { + nativeName: '繁體中文(香港)', + }, + 'zh-TW': { + nativeName: '繁體中文(臺灣)', + }, + 'zh-YUE': { + nativeName: '廣東話', + }, +}; + +export const SUPPORTED_POST_LOCALES = { + ...ISO_639_1, + ...ISO_639_3, + ...ISO_639_1_REGIONAL, +}; diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index 7803a68c7..0b2d3169d 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -66,6 +66,12 @@ export class MiNote { }) public text: string | null; + @Column('varchar', { + length: 10, + nullable: true, + }) + public lang: string | null; + @Column('varchar', { length: 256, nullable: true, }) diff --git a/packages/backend/src/models/json-schema/note.ts b/packages/backend/src/models/json-schema/note.ts index 2641161c8..aead839e2 100644 --- a/packages/backend/src/models/json-schema/note.ts +++ b/packages/backend/src/models/json-schema/note.ts @@ -2,6 +2,7 @@ * SPDX-FileCopyrightText: syuilo and misskey-project * SPDX-License-Identifier: AGPL-3.0-only */ +import { SUPPORTED_POST_LOCALES } from "@/misc/langmap.js"; export const packedNoteSchema = { type: 'object', @@ -26,6 +27,11 @@ export const packedNoteSchema = { type: 'string', optional: false, nullable: true, }, + lang: { + type: 'string', + enum: [...Object.keys(SUPPORTED_POST_LOCALES)], + nullable: true, + }, cw: { type: 'string', optional: true, nullable: true, diff --git a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts index b45d4165c..4c530ac32 100644 --- a/packages/backend/src/queue/processors/ExportNotesProcessorService.ts +++ b/packages/backend/src/queue/processors/ExportNotesProcessorService.ts @@ -45,6 +45,7 @@ class NoteStream extends ReadableStream> { return { id: note.id, text: note.text, + lang: note.lang, createdAt: idService.parse(note.id).date.toISOString(), fileIds: note.fileIds, files: files, diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index eb537a59d..3341ede75 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -16,7 +16,7 @@ import type { MiLocalUser, MiUser } from '@/models/User.js'; import { birthdaySchema, descriptionSchema, locationSchema, nameSchema } from '@/models/User.js'; import type { MiUserProfile } from '@/models/UserProfile.js'; import { normalizeForSearch } from '@/misc/normalize-for-search.js'; -import { langmap } from '@/misc/langmap.js'; +import { SUPPORTED_POST_LOCALES } from '@/misc/langmap.js'; import { Endpoint } from '@/server/api/endpoint-base.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; import { GlobalEventService } from '@/core/GlobalEventService.js'; @@ -150,7 +150,7 @@ export const paramDef = { description: { ...descriptionSchema, nullable: true }, location: { ...locationSchema, nullable: true }, birthday: { ...birthdaySchema, nullable: true }, - lang: { type: 'string', enum: [null, ...Object.keys(langmap)] as string[], nullable: true }, + lang: { type: 'string', enum: [null, ...Object.keys(SUPPORTED_POST_LOCALES)] as string[], nullable: true }, avatarId: { type: 'string', format: 'misskey:id', nullable: true }, avatarDecorations: { type: 'array', maxItems: 16, items: { type: 'object', diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index ddccbdae5..a85aa0fa4 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -23,6 +23,7 @@ import { isQuote, isRenote } from '@/misc/is-renote.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; import { LoggerService } from '@/core/LoggerService.js'; import { ApiError } from '../../error.js'; +import { SUPPORTED_POST_LOCALES } from "@/misc/langmap.js"; export const meta = { tags: ['notes'], @@ -210,6 +211,11 @@ export const paramDef = { maxLength: MAX_NOTE_TEXT_LENGTH, nullable: true, }, + lang: { + type: "string", + enum: Object.keys(SUPPORTED_POST_LOCALES), + nullable: true, + }, fileIds: { type: 'array', uniqueItems: true, @@ -484,6 +490,7 @@ export default class extends Endpoint { // eslint- expiresAt: ps.poll.expiresAt ? new Date(ps.poll.expiresAt) : null, } : undefined, text: ps.text ?? undefined, + lang: ps.lang, reply, renote, cw: ps.cw, diff --git a/packages/backend/src/types.ts b/packages/backend/src/types.ts index dc13af42f..00c8dcace 100644 --- a/packages/backend/src/types.ts +++ b/packages/backend/src/types.ts @@ -375,6 +375,7 @@ export type NoteCreateOption = { scheduledAt?: Date | null; name?: string | null; text?: string | null; + lang?: string | null; reply?: MiNote | null; renote?: MiNote | null; files?: MiDriveFile[] | null; diff --git a/packages/frontend/src/components/MkMenu.vue b/packages/frontend/src/components/MkMenu.vue index d91239b9e..8decb8f2d 100644 --- a/packages/frontend/src/components/MkMenu.vue +++ b/packages/frontend/src/components/MkMenu.vue @@ -12,8 +12,15 @@ SPDX-License-Identifier: AGPL-3.0-only :style="{ width: (width && !asDrawer) ? width + 'px' : '', maxHeight: maxHeight ? maxHeight + 'px' : '' }" @contextmenu.self="e => e.preventDefault()" > +
+ + + +