import TurndownService, { Options } from "turndown"; import interval from "interval-promise"; import { client, FORUMS_DOMAIN, BotClient } from "../client"; import sha1 from "sha1"; import * as path from "path"; import * as fs from "fs"; import { HTML2BBCode } from "html2bbcode"; import { Dict } from "../util"; import { IAggregator, NewsPostItem } from "./aggregators/aggregator"; import { TextChannel, Message, Channel, ReactionCollector, MessageReaction, User, Collector, MessageEmbed } from "discord.js"; import { getRepository, IsNull, Not } from "typeorm"; import { KnownChannel } from "@shared/db/entity/KnownChannel"; import { AggroNewsItem } from "@shared/db/entity/AggroNewsItem"; import { v3beta1 } from "@google-cloud/translate"; import { CommandSet } from "src/model/command"; const { TranslationServiceClient } = v3beta1; const UPDATE_INTERVAL = process.env.NODE_ENV == "dev" ? 60 : 5; const MAX_PREVIEW_LENGTH = 300; const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager"; const FORUMS_STAGING_ID = 54; const FORUMS_NEWS_ID = 49; @CommandSet export class NewsAggregator { tlClient = new TranslationServiceClient(); aggregators: IAggregator[] = []; aggregateChannelID?: string; bbCodeParser = new HTML2BBCode(); turndown = new TurndownService(); reactionCollectors: Dict = {}; verifyMessageIdToPost: Dict = {}; constructor() { this.turndown.addRule("image", { filter: "img", replacement: () => "" }); this.turndown.addRule("link", { filter: (node: HTMLElement, opts: Options) => node.nodeName === "A" && node.getAttribute("href") != null, replacement: (content: string, node: Node) => (node instanceof HTMLElement ? node.getAttribute("href") : null) ?? "" }); } checkFeeds = async () => { console.log(`Aggregating feeds on ${new Date().toISOString()}`); let aggregatorJobs = []; for (let aggregator of this.aggregators) { aggregatorJobs.push(aggregator.aggregate()); } let aggregatedItems = await Promise.all(aggregatorJobs); for (let itemSet of aggregatedItems) { for (let item of itemSet) { let itemObj = { ...item, cacheMessageId: undefined, postedMessageId: undefined } as NewsPostItem; itemObj.hash = sha1(itemObj.contents); await this.addNewsItem(itemObj); } } } clipText(text: string) { if (text.length <= MAX_PREVIEW_LENGTH) return text; return `${text.substring(0, MAX_PREVIEW_LENGTH)}...`; } async addNewsItem(item: NewsPostItem) { if (!this.aggregateChannelID) return; let repo = getRepository(AggroNewsItem); let ch = client.bot.channels.resolve(this.aggregateChannelID); if (!(ch instanceof TextChannel)) return; let isNew = true; let newsItem = await repo.findOne({ where: { feedName: item.feedId, newsId: item.newsId } }); if (newsItem) { if (process.env.IGNORE_CHANGED_NEWS === "TRUE") { newsItem.hash = item.hash ?? ""; await repo.save(newsItem); return; } // No changes, skip if (newsItem.hash == item.hash) return; else if(newsItem.editMessageId) await this.deleteCacheMessage(newsItem.editMessageId); isNew = false; } else { newsItem = repo.create({ newsId: item.newsId, feedName: item.feedId, hash: item.hash }); } if (item.needsTranslation && process.env.GOOGLE_APP_ID) try { let request = { parent: this.tlClient.locationPath(process.env.GOOGLE_APP_ID, "global"), contents: [item.title, item.contents], mimeType: "text/html", sourceLanguageCode: "ja", targetLanguageCode: "en" }; let [res] = await this.tlClient.translateText(request); let [translatedTitle, translatedContents] = res.translations ?? [undefined, undefined]; if (translatedTitle?.translatedText && translatedContents?.translatedText) { item.title = translatedTitle.translatedText; item.contents = translatedContents.translatedText; } } catch (err) { console.log(`Failed to translate because ${err}`); } item.contents = this.bbCodeParser.feed(item.contents).toString(); if (!newsItem.forumsEditPostId) { let createResponse = await client.forum.createThread(FORUMS_STAGING_ID, item.title, item.contents); newsItem.forumsEditPostId = createResponse.thread.thread_id; } else if(newsItem.forumsNewsPostId){ await client.forum.postReply(newsItem.forumsNewsPostId, item.contents); } let msg = await ch.send(new MessageEmbed({ title: item.title, url: item.link, color: item.embedColor, timestamp: new Date(), description: `${(isNew ? "**[NEW]**" : "**[EDIT]**")}\n[**Edit on forums**](${FORUMS_DOMAIN}/index.php?threads/.${newsItem.forumsEditPostId}/)`, author: { name: item.author }, footer: { text: "NoctBot News Aggregator" } })) as Message; newsItem.editMessageId = msg.id; await msg.react("✅"); await msg.react("❌"); let collector = msg.createReactionCollector(this.isVerifyReaction, { maxEmojis: 1 }); collector.on("collect", this.collectReaction) this.reactionCollectors[msg.id] = collector; this.verifyMessageIdToPost[msg.id] = newsItem; await repo.save(newsItem); } isVerifyReaction(reaction: MessageReaction, user: User) { return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot && user.id != client.botUser.id; } collectReaction = async (reaction: MessageReaction, collector: Collector) => { let repo = getRepository(AggroNewsItem); let m = reaction.message; collector.stop(); delete this.reactionCollectors[m.id]; let post = this.verifyMessageIdToPost[m.id]; if(!post.forumsEditPostId) { throw new Error("No forum edit post found!"); } if (reaction.emoji.name == "✅") { let res = await client.forum.getThread(post.forumsEditPostId); let forumPost = await client.forum.getPost(res.thread.first_post_id); if (!post.forumsNewsPostId) { let newThread = await client.forum.createThread(FORUMS_NEWS_ID, res.thread.title, forumPost.message); post.forumsNewsPostId = newThread.thread.thread_id; } else { let curThread = await client.forum.editThread(post.forumsNewsPostId, { title: res.thread.title }); await client.forum.editPost(curThread.thread.first_post_id, { message: forumPost.message }); } } await client.forum.deleteThread(post.forumsEditPostId); await repo.update({ newsId: post.newsId, feedName: post.feedName }, { editMessageId: undefined, forumsEditPostId: undefined, forumsNewsPostId: post.forumsNewsPostId }); await reaction.message.delete(); delete this.verifyMessageIdToPost[m.id]; }; async deleteCacheMessage(messageId: string) { if(!this.aggregateChannelID) return; let ch = client.bot.channels.resolve(this.aggregateChannelID); if (!(ch instanceof TextChannel)) return; let msg = await this.tryFetchMessage(ch, messageId); if (msg) await msg.delete(); } async tryFetchMessage(channel: Channel, messageId: string) { try { if (!(channel instanceof TextChannel)) return null; return await channel.messages.fetch(messageId); } catch (error) { return null; } } initAggregators() { let aggregatorsPath = path.join(path.dirname(module.filename), "aggregators"); let files = fs.readdirSync(aggregatorsPath); for (let file of files) { let ext = path.extname(file); let name = path.basename(file); if (name == "aggregator.js") continue; if (ext != ".js") continue; let obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator; if (obj) this.aggregators.push(obj); if (obj.init) obj.init(); } } async initPendingReactors() { if(!this.aggregateChannelID) return; let verifyChannel = client.bot.channels.resolve(this.aggregateChannelID); if(!verifyChannel) throw new Error("Couldn't find verify channel!"); let repo = getRepository(AggroNewsItem); let pendingVerifyMessages = await repo.find({ where: { editMessageId: Not(IsNull()) } }); for (let msg of pendingVerifyMessages) { let m = await this.tryFetchMessage(verifyChannel, msg.editMessageId!); if (!m) { await repo.update({ feedName: msg.feedName, newsId: msg.newsId }, { editMessageId: undefined }); continue; } let collector = m.createReactionCollector(this.isVerifyReaction, { maxEmojis: 1 }); collector.on("collect", this.collectReaction); this.reactionCollectors[m.id] = collector; this.verifyMessageIdToPost[m.id] = msg; } } async onStart() { let repo = getRepository(KnownChannel); let ch = await repo.findOne({ where: { channelType: AGGREGATOR_MANAGER_CHANNEL } }); if (!ch) return; this.aggregateChannelID = ch.channelId; await this.initPendingReactors(); this.initAggregators(); interval(this.checkFeeds, UPDATE_INTERVAL * 60 * 1000); } }