diff --git a/package-lock.json b/package-lock.json index 8fdca082..285b8548 100644 --- a/package-lock.json +++ b/package-lock.json @@ -33,6 +33,7 @@ "xss": "^1.0.15" }, "devDependencies": { + "@total-typescript/shoehorn": "^0.1.2", "@types/node": "20.14.2", "@types/node-cron": "3.0.11", "@types/node-fetch": "2.6.11", @@ -1250,6 +1251,12 @@ "npm": ">=7.0.0" } }, + "node_modules/@total-typescript/shoehorn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@total-typescript/shoehorn/-/shoehorn-0.1.2.tgz", + "integrity": "sha512-p7nNZbOZIofpDNyP0u1BctFbjxD44Qc+oO5jufgQdFdGIXJLc33QRloJpq7k5T59CTgLWfQSUxsuqLcmeurYRw==", + "dev": true + }, "node_modules/@tsconfig/node10": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.11.tgz", diff --git a/package.json b/package.json index 0ce23c43..472db64a 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "xss": "^1.0.15" }, "devDependencies": { + "@total-typescript/shoehorn": "^0.1.2", "@types/node": "20.14.2", "@types/node-cron": "3.0.11", "@types/node-fetch": "2.6.11", diff --git a/src/features/duplicate-scanner/duplicate-scanner.test.ts b/src/features/duplicate-scanner/duplicate-scanner.test.ts new file mode 100644 index 00000000..ba5c92ab --- /dev/null +++ b/src/features/duplicate-scanner/duplicate-scanner.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import { messageDuplicateChecker } from "./duplicate-scanner"; +import { User } from "discord.js"; +import { fromPartial } from "@total-typescript/shoehorn"; +import { duplicateCache } from "./duplicate-scanner"; + +const maxMessagesPerUser = 5; +const maxCacheSize = 100; +const maxTrivialCharacters = 10; +// Mock dependencies +const mockBot = { + channels: { + fetch: vi.fn().mockResolvedValue({ + type: "GUILD_TEXT", + send: vi.fn().mockResolvedValue({ + delete: vi.fn().mockResolvedValue(undefined), + }), + }), + }, +}; +const mockMessage = (content: string, authorId: string, isBot = false) => { + return { + content, + author: { id: authorId, bot: isBot } as User, + delete: vi.fn(), + channel: { + send: vi.fn().mockResolvedValue({ + delete: vi.fn().mockResolvedValue(undefined), + }), + }, + }; +}; +describe("Duplicate Scanner Tests", () => { + beforeEach(() => { + // Reset the cache before each test + duplicateCache.clear(); + }); + it(`should not store messages less than ${maxTrivialCharacters} characters`, async () => { + const msg = mockMessage("Help me", "user1"); + const bot = mockBot; + messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot })); + const userMessages = duplicateCache.get("user1"); + expect(userMessages).toBeUndefined(); + }); + + it("should store messages correctly in the cache", async () => { + const msg = mockMessage("Hello world", "user1"); + const bot = mockBot; + + messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot })); + + const userMessages = duplicateCache.get("user1"); + expect(userMessages).toBeDefined(); + expect(userMessages?.has("hello world")).toBe(true); + }); + + it(`should enforce max size of ${maxMessagesPerUser} messages per user`, async () => { + const bot = mockBot; + for (let i = 1; i <= maxMessagesPerUser; i++) { + const msg = mockMessage(`Message to delete ${i}`, "user1"); + await messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot })); + } + + const userMessages = duplicateCache.get("user1"); + expect(userMessages).toBeDefined(); + expect(userMessages?.size).toBe(maxMessagesPerUser); + + const msg = mockMessage("New Message", "user1"); + await messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot })); + + expect(userMessages?.size).toBe(maxMessagesPerUser); + expect(userMessages?.has("message 1")).toBe(false); // First message should be removed + expect(userMessages?.has("new message")).toBe(true); // New message should be added + }); + + it(`should enforce max size of ${maxCacheSize} users in the cache`, async () => { + const bot = mockBot; + + for (let i = 1; i <= maxCacheSize; i++) { + const msg = mockMessage("Hello world", `user${i}`); + await messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot })); + } + + expect(duplicateCache.size).toBe(maxCacheSize); + + const msg = mockMessage("Hello world", "user101"); + await messageDuplicateChecker.handleMessage?.(fromPartial({ msg, bot })); + + expect(duplicateCache.size).toBe(maxCacheSize); + expect(duplicateCache.has("user1")).toBe(false); + expect(duplicateCache.has("user101")).toBe(true); + }); +}); diff --git a/src/features/duplicate-scanner/duplicate-scanner.ts b/src/features/duplicate-scanner/duplicate-scanner.ts new file mode 100644 index 00000000..3394ecb0 --- /dev/null +++ b/src/features/duplicate-scanner/duplicate-scanner.ts @@ -0,0 +1,93 @@ +import type { ChannelHandlers, HandleMessageArgs } from "../../types/index.js"; +import { EmbedType } from "discord.js"; + +import { EMBED_COLOR } from "../commands.js"; +import { LRUCache } from "lru-cache"; +import { isStaff, isHelpful } from "../../helpers/discord.js"; +import { logger } from "../log.js"; +import { truncateMessage } from "../../helpers/modLog.js"; +import { formatWithEllipsis } from "./helper.js"; +import cooldown from "../cooldown.js"; +const maxMessagesPerUser = 5; // Maximum number of messages per user to track +// Time (ms) to keep track of duplicates (e.g., 30 sec) +export const duplicateCache = new LRUCache>({ + max: 100, + ttl: 1000 * 60 * 0.5, + dispose: (value) => { + value.clear(); + }, +}); +const maxTrivialCharacters = 10; +const removeFirstElement = (messages: Set) => { + const iterator = messages.values(); + const firstElement = iterator.next().value; + if (firstElement) { + messages.delete(firstElement); + } +}; + +const handleDuplicateMessage = async ({ + msg, + userId, +}: HandleMessageArgs & { userId: string }) => { + await msg.delete().catch(console.error); + const cooldownKey = `resume-${msg.channelId}`; + if (cooldown.hasCooldown(userId, cooldownKey)) { + return; + } + cooldown.addCooldown(userId, cooldownKey); + const warningMsg = `Hey <@${userId}>, it looks like you've posted this message in another channel already. Please avoid cross-posting.`; + const warning = await msg.channel.send({ + embeds: [ + { + title: "Duplicate Message Detected", + type: EmbedType.Rich, + description: warningMsg, + color: EMBED_COLOR, + }, + ], + }); + + // Auto-delete warning after 30 seconds + setTimeout(() => { + warning.delete().catch(console.error); + }, 30_000); + + logger.log( + "duplicate message detected", + `${msg.author.username} in <#${msg.channel.id}> \n${formatWithEllipsis(truncateMessage(msg.content, 100))}`, + ); + return; +}; +const normalizeContent = (content: string) => + content.trim().toLowerCase().replace(/\s+/g, " "); +export const messageDuplicateChecker: ChannelHandlers = { + handleMessage: async ({ msg, bot }) => { + if (msg.author.bot || isStaff(msg.member) || isHelpful(msg.member)) return; + + const content = normalizeContent(msg.content); + const userId = msg.author.id; + + if (content.length < maxTrivialCharacters) return; + + const userMessages = duplicateCache.get(userId); + + if (!userMessages) { + const messages = new Set(); + messages.add(content); + duplicateCache.set(userId, messages); + return; + } + + if (userMessages.has(content)) { + await handleDuplicateMessage({ msg, bot, userId }); + return; + } + + if (userMessages.size >= maxMessagesPerUser) { + removeFirstElement(userMessages); + } + + userMessages.add(content); + }, +}; diff --git a/src/features/duplicate-scanner/helper.ts b/src/features/duplicate-scanner/helper.ts new file mode 100644 index 00000000..3c5c4dfe --- /dev/null +++ b/src/features/duplicate-scanner/helper.ts @@ -0,0 +1,11 @@ +export const formatWithEllipsis = (sentences: string): string => { + const ellipsis = "..."; + + if (sentences.length === 0) { + return ellipsis; + } + if (sentences.charAt(sentences.length - 1) !== ".") { + return sentences + ellipsis; + } + return sentences + ".."; +}; diff --git a/src/index.ts b/src/index.ts index 8dc34467..d9fc041f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -42,6 +42,9 @@ import { recommendBookCommand } from "./features/book-list.js"; import { mdnSearch } from "./features/mdn.js"; import "./server.js"; import { jobScanner } from "./features/job-scanner.js"; + +import { messageDuplicateChecker } from "./features/duplicate-scanner/duplicate-scanner.js"; + import { getMessage } from "./helpers/discord.js"; export const bot = new Client({ @@ -228,7 +231,10 @@ const threadChannels = [CHANNELS.helpJs, CHANNELS.helpThreadsReact]; addHandler(threadChannels, autothread); addHandler(CHANNELS.resumeReview, resumeReviewPdf); - +addHandler( + [CHANNELS.helpReact, CHANNELS.generalReact, CHANNELS.generalTech], + messageDuplicateChecker, +); bot.on("ready", () => { deployCommands(bot); jobsMod(bot);