From b1db18c319d12e274b3a4e1930f1b4a4304a464a Mon Sep 17 00:00:00 2001 From: Nuckyz <61953774+Nuckyz@users.noreply.github.com> Date: Sun, 22 Sep 2024 12:47:18 -0300 Subject: [PATCH] PronounDB: Rework API to avoid rate limits --- src/plugins/pronoundb/api.ts | 167 +++++++++++++++++ .../components/PronounsChatComponent.tsx | 30 ++-- src/plugins/pronoundb/index.ts | 8 +- src/plugins/pronoundb/pronoundbUtils.ts | 169 ------------------ src/plugins/pronoundb/settings.ts | 2 +- src/plugins/pronoundb/types.ts | 37 ++-- 6 files changed, 208 insertions(+), 205 deletions(-) create mode 100644 src/plugins/pronoundb/api.ts delete mode 100644 src/plugins/pronoundb/pronoundbUtils.ts diff --git a/src/plugins/pronoundb/api.ts b/src/plugins/pronoundb/api.ts new file mode 100644 index 00000000..da2bc651 --- /dev/null +++ b/src/plugins/pronoundb/api.ts @@ -0,0 +1,167 @@ +/* + * Vencord, a modification for Discord's desktop app + * Copyright (c) 2022 Vendicated and contributors + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . +*/ + +import { getCurrentChannel } from "@utils/discord"; +import { useAwaiter } from "@utils/react"; +import { findStoreLazy } from "@webpack"; +import { UserProfileStore } from "@webpack/common"; + +import { settings } from "./settings"; +import { PronounMapping, Pronouns, PronounsCache, PronounSets, PronounsFormat, PronounSource, PronounsResponse } from "./types"; + +const UserSettingsAccountStore = findStoreLazy("UserSettingsAccountStore"); + +const EmptyPronouns = { pronouns: undefined, source: "", hasPendingPronouns: false } as const satisfies Pronouns; + +type RequestCallback = (pronounSets?: PronounSets) => void; + +const pronounCache: Record = {}; +const requestQueue: Record = {}; +let isProcessing = false; + +async function processQueue() { + if (isProcessing) return; + isProcessing = true; + + let ids = Object.keys(requestQueue); + while (ids.length > 0) { + const idsChunk = ids.splice(0, 50); + const pronouns = await bulkFetchPronouns(idsChunk); + + for (const id of idsChunk) { + const callbacks = requestQueue[id]; + for (const callback of callbacks) { + callback(pronouns[id]?.sets); + } + + delete requestQueue[id]; + } + + ids = Object.keys(requestQueue); + await new Promise(r => setTimeout(r, 2000)); + } + + isProcessing = false; +} + +function fetchPronouns(id: string): Promise { + return new Promise(resolve => { + if (pronounCache[id] != null) { + resolve(extractPronouns(pronounCache[id].sets)); + return; + } + + function handlePronouns(pronounSets?: PronounSets) { + const pronouns = extractPronouns(pronounSets); + resolve(pronouns); + } + + if (requestQueue[id] != null) { + requestQueue[id].push(handlePronouns); + return; + } + + requestQueue[id] = [handlePronouns]; + processQueue(); + }); +} + +async function bulkFetchPronouns(ids: string[]): Promise { + const params = new URLSearchParams(); + params.append("platform", "discord"); + params.append("ids", ids.join(",")); + + try { + const req = await fetch("https://pronoundb.org/api/v2/lookup?" + String(params), { + method: "GET", + headers: { + "Accept": "application/json", + "X-PronounDB-Source": "WebExtension/0.14.5" + } + }); + + if (!req.ok) throw new Error(`Status ${req.status}`); + const res: PronounsResponse = await req.json(); + + Object.assign(pronounCache, res); + return res; + + } catch (e) { + console.error("PronounDB request failed:", e); + const dummyPronouns: PronounsResponse = Object.fromEntries(ids.map(id => [id, { sets: {} }])); + + Object.assign(pronounCache, dummyPronouns); + return dummyPronouns; + } +} + +function extractPronouns(pronounSets?: PronounSets): string | undefined { + if (pronounSets == null) return undefined; + if (pronounSets.en == null) return PronounMapping.unspecified; + + const pronouns = pronounSets.en; + if (pronouns.length === 0) return PronounMapping.unspecified; + + const { pronounsFormat } = settings.store; + + if (pronouns.length > 1) { + const pronounString = pronouns.map(p => p[0].toUpperCase() + p.slice(1)).join("/"); + return pronounsFormat === PronounsFormat.Capitalized ? pronounString : pronounString.toLowerCase(); + } + + const pronoun = pronouns[0]; + // For capitalized pronouns or special codes (any, ask, avoid), we always return the normal (capitalized) string + if (pronounsFormat === PronounsFormat.Capitalized || ["any", "ask", "avoid", "other", "unspecified"].includes(pronoun)) { + return PronounMapping[pronoun]; + } else { + return PronounMapping[pronoun].toLowerCase(); + } +} + +function getDiscordPronouns(id: string, useGlobalProfile: boolean = false): string | undefined { + const globalPronouns = UserProfileStore.getUserProfile(id)?.pronouns; + if (useGlobalProfile) return globalPronouns; + + return UserProfileStore.getGuildMemberProfile(id, getCurrentChannel()?.guild_id)?.pronouns || globalPronouns; +} + +export function useFormattedPronouns(id: string, useGlobalProfile: boolean = false): Pronouns { + const discordPronouns = getDiscordPronouns(id, useGlobalProfile)?.trim().replace(/\n+/g, ""); + const hasPendingPronouns = UserSettingsAccountStore.getPendingPronouns() != null; + + const [pronouns] = useAwaiter(() => fetchPronouns(id)); + + if (settings.store.pronounSource === PronounSource.PreferDiscord && discordPronouns) { + return { pronouns: discordPronouns, source: "Discord", hasPendingPronouns }; + } + + if (pronouns != null && pronouns !== PronounMapping.unspecified) { + return { pronouns, source: "PronounDB", hasPendingPronouns }; + } + + return { pronouns: discordPronouns, source: "Discord", hasPendingPronouns }; +} + +export function useProfilePronouns(id: string, useGlobalProfile: boolean = false): Pronouns { + const pronouns = useFormattedPronouns(id, useGlobalProfile); + + if (!settings.store.showInProfile) return EmptyPronouns; + if (!settings.store.showSelf && id === UserProfileStore.getCurrentUser()?.id) return EmptyPronouns; + + return pronouns; +} diff --git a/src/plugins/pronoundb/components/PronounsChatComponent.tsx b/src/plugins/pronoundb/components/PronounsChatComponent.tsx index 64fac18b..46c8a8a1 100644 --- a/src/plugins/pronoundb/components/PronounsChatComponent.tsx +++ b/src/plugins/pronoundb/components/PronounsChatComponent.tsx @@ -22,7 +22,7 @@ import { findByPropsLazy } from "@webpack"; import { UserStore } from "@webpack/common"; import { Message } from "discord-types/general"; -import { useFormattedPronouns } from "../pronoundbUtils"; +import { useFormattedPronouns } from "../api"; import { settings } from "../settings"; const styles: Record = findByPropsLazy("timestampInline"); @@ -53,25 +53,21 @@ export const CompactPronounsChatComponentWrapper = ErrorBoundary.wrap(({ message }, { noop: true }); function PronounsChatComponent({ message }: { message: Message; }) { - const [result] = useFormattedPronouns(message.author.id); + const { pronouns } = useFormattedPronouns(message.author.id); - return result - ? ( - • {result} - ) - : null; + return pronouns && ( + • {pronouns} + ); } export const CompactPronounsChatComponent = ErrorBoundary.wrap(({ message }: { message: Message; }) => { - const [result] = useFormattedPronouns(message.author.id); + const { pronouns } = useFormattedPronouns(message.author.id); - return result - ? ( - • {result} - ) - : null; + return pronouns && ( + • {pronouns} + ); }, { noop: true }); diff --git a/src/plugins/pronoundb/index.ts b/src/plugins/pronoundb/index.ts index 7dfa8cb4..511aeb1c 100644 --- a/src/plugins/pronoundb/index.ts +++ b/src/plugins/pronoundb/index.ts @@ -21,9 +21,9 @@ import "./styles.css"; import { Devs } from "@utils/constants"; import definePlugin from "@utils/types"; +import { useProfilePronouns } from "./api"; import PronounsAboutComponent from "./components/PronounsAboutComponent"; import { CompactPronounsChatComponentWrapper, PronounsChatComponentWrapper } from "./components/PronounsChatComponent"; -import { useProfilePronouns } from "./pronoundbUtils"; import { settings } from "./settings"; export default definePlugin({ @@ -53,15 +53,15 @@ export default definePlugin({ replacement: [ { match: /\.PANEL},/, - replace: "$&[vcPronoun,vcPronounSource,vcHasPendingPronouns]=$self.useProfilePronouns(arguments[0].user?.id)," + replace: "$&{pronouns:vcPronoun,source:vcPronounSource,hasPendingPronouns:vcHasPendingPronouns}=$self.useProfilePronouns(arguments[0].user?.id)," }, { match: /text:\i\.\i.Messages.USER_PROFILE_PRONOUNS/, - replace: '$&+(vcHasPendingPronouns?"":` (${vcPronounSource})`)' + replace: '$&+(vcPronoun==null||vcHasPendingPronouns?"":` (${vcPronounSource})`)' }, { match: /(\.pronounsText.+?children:)(\i)/, - replace: "$1vcHasPendingPronouns?$2:vcPronoun" + replace: "$1(vcPronoun==null||vcHasPendingPronouns)?$2:vcPronoun" } ] } diff --git a/src/plugins/pronoundb/pronoundbUtils.ts b/src/plugins/pronoundb/pronoundbUtils.ts deleted file mode 100644 index 991e9031..00000000 --- a/src/plugins/pronoundb/pronoundbUtils.ts +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Vencord, a modification for Discord's desktop app - * Copyright (c) 2022 Vendicated and contributors - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . -*/ - -import { Settings } from "@api/Settings"; -import { debounce } from "@shared/debounce"; -import { VENCORD_USER_AGENT } from "@shared/vencordUserAgent"; -import { getCurrentChannel } from "@utils/discord"; -import { useAwaiter } from "@utils/react"; -import { findStoreLazy } from "@webpack"; -import { UserProfileStore, UserStore } from "@webpack/common"; - -import { settings } from "./settings"; -import { CachePronouns, PronounCode, PronounMapping, PronounsResponse } from "./types"; - -const UserSettingsAccountStore = findStoreLazy("UserSettingsAccountStore"); - -type PronounsWithSource = [pronouns: string | null, source: string, hasPendingPronouns: boolean]; -const EmptyPronouns: PronounsWithSource = [null, "", false]; - -export const enum PronounsFormat { - Lowercase = "LOWERCASE", - Capitalized = "CAPITALIZED" -} - -export const enum PronounSource { - PreferPDB, - PreferDiscord -} - -// A map of cached pronouns so the same request isn't sent twice -const cache: Record = {}; -// A map of ids and callbacks that should be triggered on fetch -const requestQueue: Record void)[]> = {}; - -// Executes all queued requests and calls their callbacks -const bulkFetch = debounce(async () => { - const ids = Object.keys(requestQueue); - const pronouns = await bulkFetchPronouns(ids); - for (const id of ids) { - // Call all callbacks for the id - requestQueue[id]?.forEach(c => c(pronouns[id] ? extractPronouns(pronouns[id].sets) : "")); - delete requestQueue[id]; - } -}); - -function getDiscordPronouns(id: string, useGlobalProfile: boolean = false) { - const globalPronouns = UserProfileStore.getUserProfile(id)?.pronouns; - - if (useGlobalProfile) return globalPronouns; - - return ( - UserProfileStore.getGuildMemberProfile(id, getCurrentChannel()?.guild_id)?.pronouns - || globalPronouns - ); -} - -export function useFormattedPronouns(id: string, useGlobalProfile: boolean = false): PronounsWithSource { - // Discord is so stupid you can put tons of newlines in pronouns - const discordPronouns = getDiscordPronouns(id, useGlobalProfile)?.trim().replace(NewLineRe, " "); - - const [result] = useAwaiter(() => fetchPronouns(id), { - fallbackValue: getCachedPronouns(id), - onError: e => console.error("Fetching pronouns failed: ", e) - }); - - const hasPendingPronouns = UserSettingsAccountStore.getPendingPronouns() != null; - - if (settings.store.pronounSource === PronounSource.PreferDiscord && discordPronouns) - return [discordPronouns, "Discord", hasPendingPronouns]; - - if (result && result !== PronounMapping.unspecified) - return [result, "PronounDB", hasPendingPronouns]; - - return [discordPronouns, "Discord", hasPendingPronouns]; -} - -export function useProfilePronouns(id: string, useGlobalProfile: boolean = false): PronounsWithSource { - const pronouns = useFormattedPronouns(id, useGlobalProfile); - - if (!settings.store.showInProfile) return EmptyPronouns; - if (!settings.store.showSelf && id === UserStore.getCurrentUser().id) return EmptyPronouns; - - return pronouns; -} - - -const NewLineRe = /\n+/g; - -// Gets the cached pronouns, if you're too impatient for a promise! -export function getCachedPronouns(id: string): string | null { - const cached = cache[id] ? extractPronouns(cache[id].sets) : undefined; - - if (cached && cached !== PronounMapping.unspecified) return cached; - - return cached || null; -} - -// Fetches the pronouns for one id, returning a promise that resolves if it was cached, or once the request is completed -export function fetchPronouns(id: string): Promise { - return new Promise(res => { - const cached = getCachedPronouns(id); - if (cached) return res(cached); - - // If there is already a request added, then just add this callback to it - if (id in requestQueue) return requestQueue[id].push(res); - - // If not already added, then add it and call the debounced function to make sure the request gets executed - requestQueue[id] = [res]; - bulkFetch(); - }); -} - -async function bulkFetchPronouns(ids: string[]): Promise { - const params = new URLSearchParams(); - params.append("platform", "discord"); - params.append("ids", ids.join(",")); - - try { - const req = await fetch("https://pronoundb.org/api/v2/lookup?" + params.toString(), { - method: "GET", - headers: { - "Accept": "application/json", - "X-PronounDB-Source": VENCORD_USER_AGENT - } - }); - return await req.json() - .then((res: PronounsResponse) => { - Object.assign(cache, res); - return res; - }); - } catch (e) { - // If the request errors, treat it as if no pronouns were found for all ids, and log it - console.error("PronounDB fetching failed: ", e); - const dummyPronouns = Object.fromEntries(ids.map(id => [id, { sets: {} }] as const)); - Object.assign(cache, dummyPronouns); - return dummyPronouns; - } -} - -export function extractPronouns(pronounSet?: { [locale: string]: PronounCode[]; }): string { - if (!pronounSet || !pronounSet.en) return PronounMapping.unspecified; - // PronounDB returns an empty set instead of {sets: {en: ["unspecified"]}}. - const pronouns = pronounSet.en; - const { pronounsFormat } = Settings.plugins.PronounDB as { pronounsFormat: PronounsFormat, enabled: boolean; }; - - if (pronouns.length === 1) { - // For capitalized pronouns or special codes (any, ask, avoid), we always return the normal (capitalized) string - if (pronounsFormat === PronounsFormat.Capitalized || ["any", "ask", "avoid", "other", "unspecified"].includes(pronouns[0])) - return PronounMapping[pronouns[0]]; - else return PronounMapping[pronouns[0]].toLowerCase(); - } - const pronounString = pronouns.map(p => p[0].toUpperCase() + p.slice(1)).join("/"); - return pronounsFormat === PronounsFormat.Capitalized ? pronounString : pronounString.toLowerCase(); -} diff --git a/src/plugins/pronoundb/settings.ts b/src/plugins/pronoundb/settings.ts index 5d227978..ebacfbc8 100644 --- a/src/plugins/pronoundb/settings.ts +++ b/src/plugins/pronoundb/settings.ts @@ -19,7 +19,7 @@ import { definePluginSettings } from "@api/Settings"; import { OptionType } from "@utils/types"; -import { PronounsFormat, PronounSource } from "./pronoundbUtils"; +import { PronounsFormat, PronounSource } from "./types"; export const settings = definePluginSettings({ pronounsFormat: { diff --git a/src/plugins/pronoundb/types.ts b/src/plugins/pronoundb/types.ts index d099a7de..66bb13f0 100644 --- a/src/plugins/pronoundb/types.ts +++ b/src/plugins/pronoundb/types.ts @@ -25,22 +25,13 @@ export interface UserProfilePronounsProps { hidePersonalInformation: boolean; } -export interface PronounsResponse { - [id: string]: { - sets?: { - [locale: string]: PronounCode[]; - } - } -} +export type PronounSets = Record; +export type PronounsResponse = Record; -export interface CachePronouns { - sets?: { - [locale: string]: PronounCode[]; - } +export interface PronounsCache { + sets?: PronounSets; } -export type PronounCode = keyof typeof PronounMapping; - export const PronounMapping = { he: "He/Him", it: "It/Its", @@ -51,4 +42,22 @@ export const PronounMapping = { ask: "Ask me my pronouns", avoid: "Avoid pronouns, use my name", unspecified: "No pronouns specified.", -} as const; +} as const satisfies Record; + +export type PronounCode = keyof typeof PronounMapping; + +export interface Pronouns { + pronouns?: string; + source: string; + hasPendingPronouns: boolean; +} + +export const enum PronounsFormat { + Lowercase = "LOWERCASE", + Capitalized = "CAPITALIZED" +} + +export const enum PronounSource { + PreferPDB, + PreferDiscord +}