const TurndownService = require("turndown"); const RSSParser = require("rss-parser"); const db = require("../db.js"); const interval = require("interval-promise"); const client = require("../client.js"); const sha1 = require("sha1"); const html = require("node-html-parser"); const axios = require("axios"); const PREVIEW_CHAR_LIMIT = 300; const verifyChannelID = db.get("newsPostVerifyChannel").value(); const reactionCollectors = {}; const verifyMessageIdToPostId = {}; const turndown = new TurndownService(); turndown.addRule("image", { filter: "img", replacement: () => "" }); turndown.addRule("link", { filter: node => node.nodeName === "A" &&node.getAttribute("href"), replacement: (content, node) => node.getAttribute("href") }); const parser = new RSSParser(); const RSS_UPDATE_INTERVAL_MIN = 5; function getThreadId(url) { let result = url.substring(url.lastIndexOf(".") + 1); if(result.endsWith("/")) result = result.substring(0, result.length - 1); return result; } async function checkFeeds() { console.log(`Checking feeds on ${new Date().toISOString()}`); let feeds = db.get("rssFeeds").value(); let oldNews = db.get("postedNewsGuids"); for(let feedEntry of feeds) { let feed = await parser.parseURL(feedEntry.url); if(feed.items.length == 0) continue; let printableItems = feed.items.sort((a, b) => a.isoDate.localeCompare(b.isoDate)); if(printableItems.length > 0) { for(let item of printableItems) { let itemID = getThreadId(item.guid); let contents = null; try { let res = await axios.get(item.link); if(res.status != 200) { console.log(`Post ${itemID} could not be loaded because request returned status ${res.status}`); continue; } let rootNode = html.parse(res.data, { pre: true, script: false, style: false }); let opDiv = rootNode.querySelector("div.bbWrapper"); if (!opDiv) { console.log(`No posts found for ${itemID}!`); continue; } contents = markdownify(opDiv.outerHTML, item.link); } catch(err){ console.log(`Failed to get html for item ${itemID} because ${err}`); continue; } let itemObj = { id: itemID, title: item.title, link: item.link, creator: item.creator, contents: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`, hash: sha1(contents), messageId: null, verifyMessageId: null, type: null }; if(oldNews.has(itemObj.id).value()){ let data = oldNews.get(itemObj.id).value(); // Old type, don't care if(data === true) continue; // Add message ID to mark for edit if(data.hash != itemObj.hash) itemObj.messageId = data.messageId; else continue; } if(!shouldVerify()) await sendNews(itemObj); else await addVerifyMessage(itemObj); } let lastUpdateDate = printableItems[printableItems.length - 1].isoDate; console.log(`Setting last update marker on ${feedEntry.url} to ${lastUpdateDate}`); db.get("rssFeeds").find({ url: feedEntry.url}).assign({lastUpdate: lastUpdateDate}).write(); } } } function initPendingReactors() { let verifyChannel = client.channels.get(verifyChannelID); db.get("newsCache").forOwn(async i => { let m = await tryFetchMessage(verifyChannel, i.verifyMessageId); let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 }); collector.on("collect", collectReaction) reactionCollectors[m.id] = collector; verifyMessageIdToPostId[m.id] = i.id; }).value(); } async function addVerifyMessage(item) { let verifyChannel = client.channels.get(verifyChannelID); let cache = db.get("newsCache"); let oldNews = db.get("postedNewsGuids"); let postedNews = db.get("postedNewsGuids"); item.type = "🆕 ADD"; if(postedNews.has(item.id).value()) item.type = "✏️ EDIT"; if(cache.has(item.id).value()) { let oldItem = cache.get(item.id).value(); let oldMessage = await tryFetchMessage(verifyChannel, oldItem.verifyMessageId); if(oldMessage) await oldMessage.delete(); } let newMessage = await verifyChannel.send(toVerifyString(item)); await newMessage.react("✅"); await newMessage.react("❌"); var collector = newMessage.createReactionCollector(isVerifyReaction, { maxEmojis: 1 }); collector.on("collect", collectReaction) reactionCollectors[newMessage.id] = collector; verifyMessageIdToPostId[newMessage.id] = item.id; item.verifyMessageId = newMessage.id; cache.set(item.id, item).write(); oldNews.set(item.id, { hash: item.hash, messageId: null }).write(); } function collectReaction(reaction, collector) { let cache = db.get("newsCache"); let m = reaction.message; collector.stop(); delete reactionCollectors[m.id]; let postId = verifyMessageIdToPostId[m.id]; let item = cache.get(postId).value(); cache.unset(postId).write(); m.delete(); if(reaction.emoji.name == "✅") sendNews(item); } async function sendNews(item) { let outChannel = db.get("feedOutputChannel").value(); let oldNews = db.get("postedNewsGuids"); let sentMessage = await postNewsItem(outChannel, item); oldNews.set(item.id, { hash: item.hash, messageId: sentMessage.id }).write(); } function isVerifyReaction(reaction, user) { return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot; } async function tryFetchMessage(channel, messageId) { try { return await channel.fetchMessage(messageId); }catch(error){ return null; } } function shouldVerify() { return verifyChannelID != ""; } async function postNewsItem(channel, item) { let newsMessage = toNewsString(item); let ch = client.channels.get(channel); if(item.messageId) { let message = await tryFetchMessage(ch, item.messageId); if(message) return await message.edit(newsMessage); else return await ch.send(newsMessage); } else return await ch.send(newsMessage); } function markdownify(htmStr, link) { return turndown.turndown(htmStr).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, ""); } function toNewsString(item) { return `**${item.title}** Posted by ${item.creator} ${item.link} ${item.contents}`; } function toVerifyString(item) { return `[${item.type}] Post ID: **${item.id}** ${toNewsString(item)} React with ✅ (approve) or ❌ (deny).`; } const onStart = () => { initPendingReactors(); interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000); }; const commands = [ { pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i, action: async (msg, s, match) => { if(msg.channel.id != verifyChannelID) return; let id = match[1]; let newContents = match[2].trim(); if(!db.get("newsCache").has(id).value()) { msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`); return; } let item = db.get("newsCache").get(id).value(); let editMsg = await tryFetchMessage(client.channels.get(verifyChannelID), item.verifyMessageId); if(!editMsg){ msg.channel.send(`${msg.author.toString()} No verify messafe found for ${id}! This is a bug: report to horse.`); return; } item.contents = newContents; db.get("newsCache").set(id, item).write(); await editMsg.edit(toVerifyString(item)); await msg.delete(); } } ]; module.exports = { onStart: onStart, commands: commands };