import TurndownService, { Options } from "turndown"; import interval from "interval-promise"; import { client, forumClient, FORUMS_DOMAIN } from "../client"; import sha1 from "sha1"; import * as path from "path"; import * as fs from "fs"; import { HTML2BBCode } from "html2bbcode"; import { Dict } from "../util"; import { IAggregator, NewsPostItem, INewsPostData } from "./aggregators/aggregator"; import { ICommand } from "./command"; import { RichEmbed, TextChannel, Message, Channel, ReactionCollector, MessageReaction, User, Collector } from "discord.js"; import { getRepository, IsNull, Not } from "typeorm"; import { KnownChannel } from "@db/entity/KnownChannel"; import { AggroNewsItem } from "@db/entity/AggroNewsItem"; import { v3beta1 } from "@google-cloud/translate"; const { TranslationServiceClient } = v3beta1; const tlClient = new TranslationServiceClient(); const UPDATE_INTERVAL = 5; const MAX_PREVIEW_LENGTH = 300; const aggregators : IAggregator[] = []; let aggregateChannelID : string = null; const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager"; const FORUMS_STAGING_ID = 54; const FORUMS_NEWS_ID = 49; const bbCodeParser = new HTML2BBCode(); const reactionCollectors: Dict = {}; const verifyMessageIdToPost: Dict = {}; // TODO: Run BBCode converter instead const turndown = new TurndownService(); turndown.addRule("image", { filter: "img", replacement: () => "" }); turndown.addRule("link", { filter: (node : HTMLElement, opts: Options) => node.nodeName === "A" &&node.getAttribute("href") != null, replacement: (content: string, node: HTMLElement) => node.getAttribute("href") }); function markdownify(htmStr: string) { return turndown.turndown(htmStr)/*.replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "")*/; } async function checkFeeds() { console.log(`Aggregating feeds on ${new Date().toISOString()}`); let aggregatorJobs = []; for(let aggregator of aggregators) { aggregatorJobs.push(aggregator.aggregate()); } let aggregatedItems = await Promise.all(aggregatorJobs); for(let itemSet of aggregatedItems) { for(let item of itemSet) { let itemObj = { ...item, cacheMessageId: null, postedMessageId: null } as NewsPostItem; itemObj.hash = sha1(itemObj.contents); await addNewsItem(itemObj); } } } function clipText(text: string) { if(text.length <= MAX_PREVIEW_LENGTH) return text; return `${text.substring(0, MAX_PREVIEW_LENGTH)}...`; } // TODO: Replace with proper forum implementation async function addNewsItem(item: NewsPostItem) { let repo = getRepository(AggroNewsItem); let ch = client.channels.get(aggregateChannelID); if(!(ch instanceof TextChannel)) return; let isNew = true; let newsItem = await repo.findOne({ where: { feedName: item.feedId, newsId: item.newsId } }); if(newsItem) { // No changes, skip if(newsItem.hash == item.hash) return; else await deleteCacheMessage(newsItem.editMessageId); isNew = false; } else { newsItem = repo.create({ newsId: item.newsId, feedName: item.feedId, hash: item.hash }); } if(item.needsTranslation) try { let request = { parent: tlClient.locationPath(process.env.GOOGLE_APP_ID, "global"), contents: [ item.title, item.contents ], mimeType: "text/html", sourceLanguageCode: "ja", targetLanguageCode: "en" }; let [ res ] = await tlClient.translateText(request); item.title = res.translations[0].translatedText item.contents = res.translations[1].translatedText; } catch(err) { console.log(`Failed to translate because ${err}`); } item.contents = bbCodeParser.feed(item.contents).toString(); if(!newsItem.forumsEditPostId) { let createResponse = await forumClient.createThread(FORUMS_STAGING_ID, item.title, item.contents); newsItem.forumsEditPostId = createResponse.thread.thread_id; } else { await forumClient.postReply(newsItem.forumsNewsPostId, item.contents); } let msg = await ch.send(new RichEmbed({ title: item.title, url: item.link, color: item.embedColor, timestamp: new Date(), description: `${(isNew ? "**[NEW]**" : "**[EDIT]**")}\n[**Edit on forums**](${FORUMS_DOMAIN}/index.php?threads/.${newsItem.forumsEditPostId}/)`, author: { name: item.author }, footer: { text: "NoctBot News Aggregator" } })) as Message; newsItem.editMessageId = msg.id; await msg.react("✅"); await msg.react("❌"); let collector = msg.createReactionCollector(isVerifyReaction, { maxEmojis: 1 }); collector.on("collect", collectReaction) reactionCollectors[msg.id] = collector; verifyMessageIdToPost[msg.id] = newsItem; await repo.save(newsItem); } function isVerifyReaction(reaction: MessageReaction, user: User) { return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot && user.id != client.user.id; } async function collectReaction(reaction: MessageReaction, collector: Collector) { let repo = getRepository(AggroNewsItem); let m = reaction.message; collector.stop(); delete reactionCollectors[m.id]; let post = verifyMessageIdToPost[m.id]; if (reaction.emoji.name == "✅") { let res = await forumClient.getThread(post.forumsEditPostId); let forumPost = await forumClient.getPost(res.thread.first_post_id); if(!post.forumsNewsPostId) { let newThread = await forumClient.createThread(FORUMS_NEWS_ID, res.thread.title, forumPost.message); post.forumsNewsPostId = newThread.thread.thread_id; } else { let curThread = await forumClient.editThread(post.forumsNewsPostId, { title: res.thread.title }); await forumClient.editPost(curThread.thread.first_post_id, { message: forumPost.message }); } } await forumClient.deleteThread(post.forumsEditPostId); await repo.update({ newsId: post.newsId, feedName: post.feedName }, { editMessageId: null, forumsEditPostId: null, forumsNewsPostId: post.forumsNewsPostId }); await reaction.message.delete(); delete verifyMessageIdToPost[m.id]; } async function deleteCacheMessage(messageId: string) { let ch = client.channels.get(aggregateChannelID); if(!(ch instanceof TextChannel)) return; let msg = await tryFetchMessage(ch, messageId); if(msg) await msg.delete(); } async function tryFetchMessage(channel: Channel, messageId: string) { try { if (!(channel instanceof TextChannel)) return null; return await channel.fetchMessage(messageId); } catch (error) { return null; } } function initAggregators() { let aggregatorsPath = path.join(path.dirname(module.filename), "aggregators"); let files = fs.readdirSync(aggregatorsPath); for(let file of files) { let ext = path.extname(file); let name = path.basename(file); if(name == "aggregator.js") continue; if(ext != ".js") continue; let obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator; if(obj) aggregators.push(obj); if(obj.init) obj.init(); } } async function initPendingReactors() { let verifyChannel = client.channels.get(aggregateChannelID); let repo = getRepository(AggroNewsItem); let pendingVerifyMessages = await repo.find({ where: { editMessageId: Not(IsNull()) } }); for (let msg of pendingVerifyMessages) { let m = await tryFetchMessage(verifyChannel, msg.editMessageId); if (!m) { await repo.update({ feedName: msg.feedName, newsId: msg.newsId }, { editMessageId: null }); continue; } let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 }); collector.on("collect", collectReaction); reactionCollectors[m.id] = collector; verifyMessageIdToPost[m.id] = msg; } } export default { onStart : async () => { let repo = getRepository(KnownChannel); let ch = await repo.findOne({ where: { channelType: AGGREGATOR_MANAGER_CHANNEL } }); if(!ch) return; aggregateChannelID = ch.channelId; await initPendingReactors(); initAggregators(); interval(checkFeeds, UPDATE_INTERVAL * 60 * 1000); } } as ICommand;