PronounDB: Rework API to avoid rate limits

src/plugins/pronoundb/api.ts

@@ -0,0 +1,167 @@
+/*
+ * Vencord, a modification for Discord's desktop app
+ * Copyright (c) 2022 Vendicated and contributors
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+import { getCurrentChannel } from "@utils/discord";
+import { useAwaiter } from "@utils/react";
+import { findStoreLazy } from "@webpack";
+import { UserProfileStore } from "@webpack/common";
+
+import { settings } from "./settings";
+import { PronounMapping, Pronouns, PronounsCache, PronounSets, PronounsFormat, PronounSource, PronounsResponse } from "./types";
+
+const UserSettingsAccountStore = findStoreLazy("UserSettingsAccountStore");
+
+const EmptyPronouns = { pronouns: undefined, source: "", hasPendingPronouns: false } as const satisfies Pronouns;
+
+type RequestCallback = (pronounSets?: PronounSets) => void;
+
+const pronounCache: Record<string, PronounsCache> = {};
+const requestQueue: Record<string, RequestCallback[]> = {};
+let isProcessing = false;
+
+async function processQueue() {
+    if (isProcessing) return;
+    isProcessing = true;
+
+    let ids = Object.keys(requestQueue);
+    while (ids.length > 0) {
+        const idsChunk = ids.splice(0, 50);
+        const pronouns = await bulkFetchPronouns(idsChunk);
+
+        for (const id of idsChunk) {
+            const callbacks = requestQueue[id];
+            for (const callback of callbacks) {
+                callback(pronouns[id]?.sets);
+            }
+
+            delete requestQueue[id];
+        }
+
+        ids = Object.keys(requestQueue);
+        await new Promise(r => setTimeout(r, 2000));
+    }
+
+    isProcessing = false;
+}
+
+function fetchPronouns(id: string): Promise<string | undefined> {
+    return new Promise(resolve => {
+        if (pronounCache[id] != null) {
+            resolve(extractPronouns(pronounCache[id].sets));
+            return;
+        }
+
+        function handlePronouns(pronounSets?: PronounSets) {
+            const pronouns = extractPronouns(pronounSets);
+            resolve(pronouns);
+        }
+
+        if (requestQueue[id] != null) {
+            requestQueue[id].push(handlePronouns);
+            return;
+        }
+
+        requestQueue[id] = [handlePronouns];
+        processQueue();
+    });
+}
+
+async function bulkFetchPronouns(ids: string[]): Promise<PronounsResponse> {
+    const params = new URLSearchParams();
+    params.append("platform", "discord");
+    params.append("ids", ids.join(","));
+
+    try {
+        const req = await fetch("https://pronoundb.org/api/v2/lookup?" + String(params), {
+            method: "GET",
+            headers: {
+                "Accept": "application/json",
+                "X-PronounDB-Source": "WebExtension/0.14.5"
+            }
+        });
+
+        if (!req.ok) throw new Error(`Status ${req.status}`);
+        const res: PronounsResponse = await req.json();
+
+        Object.assign(pronounCache, res);
+        return res;
+
+    } catch (e) {
+        console.error("PronounDB request failed:", e);
+        const dummyPronouns: PronounsResponse = Object.fromEntries(ids.map(id => [id, { sets: {} }]));
+
+        Object.assign(pronounCache, dummyPronouns);
+        return dummyPronouns;
+    }
+}
+
+function extractPronouns(pronounSets?: PronounSets): string | undefined {
+    if (pronounSets == null) return undefined;
+    if (pronounSets.en == null) return PronounMapping.unspecified;
+
+    const pronouns = pronounSets.en;
+    if (pronouns.length === 0) return PronounMapping.unspecified;
+
+    const { pronounsFormat } = settings.store;
+
+    if (pronouns.length > 1) {
+        const pronounString = pronouns.map(p => p[0].toUpperCase() + p.slice(1)).join("/");
+        return pronounsFormat === PronounsFormat.Capitalized ? pronounString : pronounString.toLowerCase();
+    }
+
+    const pronoun = pronouns[0];
+    // For capitalized pronouns or special codes (any, ask, avoid), we always return the normal (capitalized) string
+    if (pronounsFormat === PronounsFormat.Capitalized || ["any", "ask", "avoid", "other", "unspecified"].includes(pronoun)) {
+        return PronounMapping[pronoun];
+    } else {
+        return PronounMapping[pronoun].toLowerCase();
+    }
+}
+
+function getDiscordPronouns(id: string, useGlobalProfile: boolean = false): string | undefined {
+    const globalPronouns = UserProfileStore.getUserProfile(id)?.pronouns;
+    if (useGlobalProfile) return globalPronouns;
+
+    return UserProfileStore.getGuildMemberProfile(id, getCurrentChannel()?.guild_id)?.pronouns || globalPronouns;
+}
+
+export function useFormattedPronouns(id: string, useGlobalProfile: boolean = false): Pronouns {
+    const discordPronouns = getDiscordPronouns(id, useGlobalProfile)?.trim().replace(/\n+/g, "");
+    const hasPendingPronouns = UserSettingsAccountStore.getPendingPronouns() != null;
+
+    const [pronouns] = useAwaiter(() => fetchPronouns(id));
+
+    if (settings.store.pronounSource === PronounSource.PreferDiscord && discordPronouns) {
+        return { pronouns: discordPronouns, source: "Discord", hasPendingPronouns };
+    }
+
+    if (pronouns != null && pronouns !== PronounMapping.unspecified) {
+        return { pronouns, source: "PronounDB", hasPendingPronouns };
+    }
+
+    return { pronouns: discordPronouns, source: "Discord", hasPendingPronouns };
+}
+
+export function useProfilePronouns(id: string, useGlobalProfile: boolean = false): Pronouns {
+    const pronouns = useFormattedPronouns(id, useGlobalProfile);
+
+    if (!settings.store.showInProfile) return EmptyPronouns;
+    if (!settings.store.showSelf && id === UserProfileStore.getCurrentUser()?.id) return EmptyPronouns;
+
+    return pronouns;
+}

src/plugins/pronoundb/components/PronounsChatComponent.tsx

@@ -22,7 +22,7 @@ import { findByPropsLazy } from "@webpack";
 import { UserStore } from "@webpack/common";
 import { Message } from "discord-types/general";
 
-import { useFormattedPronouns } from "../pronoundbUtils";
+import { useFormattedPronouns } from "../api";
 import { settings } from "../settings";
 
 const styles: Record<string, string> = findByPropsLazy("timestampInline");
@@ -53,25 +53,21 @@ export const CompactPronounsChatComponentWrapper = ErrorBoundary.wrap(({ message
 }, { noop: true });
 
 function PronounsChatComponent({ message }: { message: Message; }) {
-    const [result] = useFormattedPronouns(message.author.id);
+    const { pronouns } = useFormattedPronouns(message.author.id);
 
-    return result
-        ? (
+    return pronouns && (
         <span
             className={classes(styles.timestampInline, styles.timestamp)}
-            >• {result}</span>
-        )
-        : null;
+        >• {pronouns}</span>
+    );
 }
 
 export const CompactPronounsChatComponent = ErrorBoundary.wrap(({ message }: { message: Message; }) => {
-    const [result] = useFormattedPronouns(message.author.id);
+    const { pronouns } = useFormattedPronouns(message.author.id);
 
-    return result
-        ? (
+    return pronouns && (
         <span
             className={classes(styles.timestampInline, styles.timestamp, "vc-pronoundb-compact")}
-            >• {result}</span>
-        )
-        : null;
+        >• {pronouns}</span>
+    );
 }, { noop: true });

src/plugins/pronoundb/index.ts

@@ -21,9 +21,9 @@ import "./styles.css";
 import { Devs } from "@utils/constants";
 import definePlugin from "@utils/types";
 
+import { useProfilePronouns } from "./api";
 import PronounsAboutComponent from "./components/PronounsAboutComponent";
 import { CompactPronounsChatComponentWrapper, PronounsChatComponentWrapper } from "./components/PronounsChatComponent";
-import { useProfilePronouns } from "./pronoundbUtils";
 import { settings } from "./settings";
 
 export default definePlugin({
@@ -53,15 +53,15 @@ export default definePlugin({
             replacement: [
                 {
                     match: /\.PANEL},/,
-                    replace: "$&[vcPronoun,vcPronounSource,vcHasPendingPronouns]=$self.useProfilePronouns(arguments[0].user?.id),"
+                    replace: "$&{pronouns:vcPronoun,source:vcPronounSource,hasPendingPronouns:vcHasPendingPronouns}=$self.useProfilePronouns(arguments[0].user?.id),"
                 },
                 {
                     match: /text:\i\.\i.Messages.USER_PROFILE_PRONOUNS/,
-                    replace: '$&+(vcHasPendingPronouns?"":` (${vcPronounSource})`)'
+                    replace: '$&+(vcPronoun==null||vcHasPendingPronouns?"":` (${vcPronounSource})`)'
                 },
                 {
                     match: /(\.pronounsText.+?children:)(\i)/,
-                    replace: "$1vcHasPendingPronouns?$2:vcPronoun"
+                    replace: "$1(vcPronoun==null||vcHasPendingPronouns)?$2:vcPronoun"
                 }
             ]
         }

src/plugins/pronoundb/pronoundbUtils.ts

@@ -1,169 +0,0 @@
-/*
- * Vencord, a modification for Discord's desktop app
- * Copyright (c) 2022 Vendicated and contributors
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
-*/
-
-import { Settings } from "@api/Settings";
-import { debounce } from "@shared/debounce";
-import { VENCORD_USER_AGENT } from "@shared/vencordUserAgent";
-import { getCurrentChannel } from "@utils/discord";
-import { useAwaiter } from "@utils/react";
-import { findStoreLazy } from "@webpack";
-import { UserProfileStore, UserStore } from "@webpack/common";
-
-import { settings } from "./settings";
-import { CachePronouns, PronounCode, PronounMapping, PronounsResponse } from "./types";
-
-const UserSettingsAccountStore = findStoreLazy("UserSettingsAccountStore");
-
-type PronounsWithSource = [pronouns: string | null, source: string, hasPendingPronouns: boolean];
-const EmptyPronouns: PronounsWithSource = [null, "", false];
-
-export const enum PronounsFormat {
-    Lowercase = "LOWERCASE",
-    Capitalized = "CAPITALIZED"
-}
-
-export const enum PronounSource {
-    PreferPDB,
-    PreferDiscord
-}
-
-// A map of cached pronouns so the same request isn't sent twice
-const cache: Record<string, CachePronouns> = {};
-// A map of ids and callbacks that should be triggered on fetch
-const requestQueue: Record<string, ((pronouns: string) => void)[]> = {};
-
-// Executes all queued requests and calls their callbacks
-const bulkFetch = debounce(async () => {
-    const ids = Object.keys(requestQueue);
-    const pronouns = await bulkFetchPronouns(ids);
-    for (const id of ids) {
-        // Call all callbacks for the id
-        requestQueue[id]?.forEach(c => c(pronouns[id] ? extractPronouns(pronouns[id].sets) : ""));
-        delete requestQueue[id];
-    }
-});
-
-function getDiscordPronouns(id: string, useGlobalProfile: boolean = false) {
-    const globalPronouns = UserProfileStore.getUserProfile(id)?.pronouns;
-
-    if (useGlobalProfile) return globalPronouns;
-
-    return (
-        UserProfileStore.getGuildMemberProfile(id, getCurrentChannel()?.guild_id)?.pronouns
-        || globalPronouns
-    );
-}
-
-export function useFormattedPronouns(id: string, useGlobalProfile: boolean = false): PronounsWithSource {
-    // Discord is so stupid you can put tons of newlines in pronouns
-    const discordPronouns = getDiscordPronouns(id, useGlobalProfile)?.trim().replace(NewLineRe, " ");
-
-    const [result] = useAwaiter(() => fetchPronouns(id), {
-        fallbackValue: getCachedPronouns(id),
-        onError: e => console.error("Fetching pronouns failed: ", e)
-    });
-
-    const hasPendingPronouns = UserSettingsAccountStore.getPendingPronouns() != null;
-
-    if (settings.store.pronounSource === PronounSource.PreferDiscord && discordPronouns)
-        return [discordPronouns, "Discord", hasPendingPronouns];
-
-    if (result && result !== PronounMapping.unspecified)
-        return [result, "PronounDB", hasPendingPronouns];
-
-    return [discordPronouns, "Discord", hasPendingPronouns];
-}
-
-export function useProfilePronouns(id: string, useGlobalProfile: boolean = false): PronounsWithSource {
-    const pronouns = useFormattedPronouns(id, useGlobalProfile);
-
-    if (!settings.store.showInProfile) return EmptyPronouns;
-    if (!settings.store.showSelf && id === UserStore.getCurrentUser().id) return EmptyPronouns;
-
-    return pronouns;
-}
-
-
-const NewLineRe = /\n+/g;
-
-// Gets the cached pronouns, if you're too impatient for a promise!
-export function getCachedPronouns(id: string): string | null {
-    const cached = cache[id] ? extractPronouns(cache[id].sets) : undefined;
-
-    if (cached && cached !== PronounMapping.unspecified) return cached;
-
-    return cached || null;
-}
-
-// Fetches the pronouns for one id, returning a promise that resolves if it was cached, or once the request is completed
-export function fetchPronouns(id: string): Promise<string> {
-    return new Promise(res => {
-        const cached = getCachedPronouns(id);
-        if (cached) return res(cached);
-
-        // If there is already a request added, then just add this callback to it
-        if (id in requestQueue) return requestQueue[id].push(res);
-
-        // If not already added, then add it and call the debounced function to make sure the request gets executed
-        requestQueue[id] = [res];
-        bulkFetch();
-    });
-}
-
-async function bulkFetchPronouns(ids: string[]): Promise<PronounsResponse> {
-    const params = new URLSearchParams();
-    params.append("platform", "discord");
-    params.append("ids", ids.join(","));
-
-    try {
-        const req = await fetch("https://pronoundb.org/api/v2/lookup?" + params.toString(), {
-            method: "GET",
-            headers: {
-                "Accept": "application/json",
-                "X-PronounDB-Source": VENCORD_USER_AGENT
-            }
-        });
-        return await req.json()
-            .then((res: PronounsResponse) => {
-                Object.assign(cache, res);
-                return res;
-            });
-    } catch (e) {
-        // If the request errors, treat it as if no pronouns were found for all ids, and log it
-        console.error("PronounDB fetching failed: ", e);
-        const dummyPronouns = Object.fromEntries(ids.map(id => [id, { sets: {} }] as const));
-        Object.assign(cache, dummyPronouns);
-        return dummyPronouns;
-    }
-}
-
-export function extractPronouns(pronounSet?: { [locale: string]: PronounCode[]; }): string {
-    if (!pronounSet || !pronounSet.en) return PronounMapping.unspecified;
-    // PronounDB returns an empty set instead of {sets: {en: ["unspecified"]}}.
-    const pronouns = pronounSet.en;
-    const { pronounsFormat } = Settings.plugins.PronounDB as { pronounsFormat: PronounsFormat, enabled: boolean; };
-
-    if (pronouns.length === 1) {
-        // For capitalized pronouns or special codes (any, ask, avoid), we always return the normal (capitalized) string
-        if (pronounsFormat === PronounsFormat.Capitalized || ["any", "ask", "avoid", "other", "unspecified"].includes(pronouns[0]))
-            return PronounMapping[pronouns[0]];
-        else return PronounMapping[pronouns[0]].toLowerCase();
-    }
-    const pronounString = pronouns.map(p => p[0].toUpperCase() + p.slice(1)).join("/");
-    return pronounsFormat === PronounsFormat.Capitalized ? pronounString : pronounString.toLowerCase();
-}

src/plugins/pronoundb/settings.ts

@@ -19,7 +19,7 @@
 import { definePluginSettings } from "@api/Settings";
 import { OptionType } from "@utils/types";
 
-import { PronounsFormat, PronounSource } from "./pronoundbUtils";
+import { PronounsFormat, PronounSource } from "./types";
 
 export const settings = definePluginSettings({
     pronounsFormat: {

src/plugins/pronoundb/types.ts

@@ -25,22 +25,13 @@ export interface UserProfilePronounsProps {
     hidePersonalInformation: boolean;
 }
 
-export interface PronounsResponse {
-    [id: string]: {
-        sets?: {
-            [locale: string]: PronounCode[];
-        }
-    }
-}
+export type PronounSets = Record<string, PronounCode[]>;
+export type PronounsResponse = Record<string, { sets?: PronounSets; }>;
 
-export interface CachePronouns {
-    sets?: {
-        [locale: string]: PronounCode[];
-    }
+export interface PronounsCache {
+    sets?: PronounSets;
 }
 
-export type PronounCode = keyof typeof PronounMapping;
-
 export const PronounMapping = {
     he: "He/Him",
     it: "It/Its",
@@ -51,4 +42,22 @@ export const PronounMapping = {
     ask: "Ask me my pronouns",
     avoid: "Avoid pronouns, use my name",
     unspecified: "No pronouns specified.",
-} as const;
+} as const satisfies Record<string, string>;
+
+export type PronounCode = keyof typeof PronounMapping;
+
+export interface Pronouns {
+    pronouns?: string;
+    source: string;
+    hasPendingPronouns: boolean;
+}
+
+export const enum PronounsFormat {
+    Lowercase = "LOWERCASE",
+    Capitalized = "CAPITALIZED"
+}
+
+export const enum PronounSource {
+    PreferPDB,
+    PreferDiscord
+}