noctbot/commands/rss_checker.js

const TurndownService = require("turndown");
const RSSParser = require("rss-parser");
const db = require("../db.js");
const interval = require("interval-promise");
const client = require("../client.js");
const sha1 = require("sha1");
const html = require("node-html-parser"); 
const axios = require("axios");

const PREVIEW_CHAR_LIMIT = 300;
const verifyChannelID = db.get("newsPostVerifyChannel").value();

const reactionCollectors = {};
const verifyMessageIdToPostId = {};

const turndown = new TurndownService();
turndown.addRule("image", {
    filter: "img",
    replacement: () => ""
});
turndown.addRule("link", {
    filter: node => node.nodeName === "A" &&node.getAttribute("href"),
    replacement: (content, node) => node.getAttribute("href")
});

const parser = new RSSParser();
const RSS_UPDATE_INTERVAL_MIN = 5;

function getThreadId(url) {
    let result = url.substring(url.lastIndexOf(".") + 1);
    if(result.endsWith("/"))
        result = result.substring(0, result.length - 1);
    return result;
}

async function checkFeeds() {
    console.log(`Checking feeds on ${new Date().toISOString()}`);
    let feeds = db.get("rssFeeds").value();
    let oldNews = db.get("postedNewsGuids");
    for(let feedEntry of feeds) {
        let feed = await parser.parseURL(feedEntry.url);
        if(feed.items.length == 0)
            continue;
        let printableItems = feed.items.sort((a, b) => a.isoDate.localeCompare(b.isoDate));
        if(printableItems.length > 0) {
            for(let item of printableItems) {
                let itemID = getThreadId(item.guid);
                let contents = null;
                
                try {
                    let res = await axios.get(item.link);
                    if(res.status != 200) {
                        console.log(`Post ${itemID} could not be loaded because request returned status ${res.status}`);
                        continue;
                    }

                    let rootNode = html.parse(res.data, {
                        pre: true,
                        script: false,
                        style: false
                    });

                    let opDiv = rootNode.querySelector("div.bbWrapper");

                    if (!opDiv) {
                        console.log(`No posts found for ${itemID}!`);
                        continue;
                    }
                    
                    contents = markdownify(opDiv.outerHTML, item.link);
                } catch(err){
                    console.log(`Failed to get html for item ${itemID} because ${err}`);
                    continue;
                }

                let itemObj = {
                    id: itemID,
                    title: item.title,
                    link: item.link,
                    creator: item.creator,
                    contents: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
                    hash: sha1(contents),
                    messageId: null,
                    verifyMessageId: null,
                    type: null
                };

                if(oldNews.has(itemObj.id).value()){
                    let data = oldNews.get(itemObj.id).value();
                    // Old type, don't care
                    if(data === true)
                        continue;

                    // Add message ID to mark for edit
                    if(data.hash != itemObj.hash)
                        itemObj.messageId = data.messageId;
                    else 
                        continue;
                }

                if(!shouldVerify())
                    await sendNews(itemObj);
                else
                    await addVerifyMessage(itemObj);
            }
            let lastUpdateDate = printableItems[printableItems.length - 1].isoDate;
            console.log(`Setting last update marker on ${feedEntry.url} to ${lastUpdateDate}`);
            db.get("rssFeeds").find({ url: feedEntry.url}).assign({lastUpdate: lastUpdateDate}).write();
        }
    }
}

function initPendingReactors() {
    let verifyChannel = client.channels.get(verifyChannelID);
    db.get("newsCache").forOwn(async i => {
        let m = await tryFetchMessage(verifyChannel, i.verifyMessageId);
        let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
        collector.on("collect", collectReaction)
        reactionCollectors[m.id] = collector;
        verifyMessageIdToPostId[m.id] = i.id;
    }).value();
}

async function addVerifyMessage(item) {
    let verifyChannel = client.channels.get(verifyChannelID);
    let cache = db.get("newsCache");
    let oldNews = db.get("postedNewsGuids");
    let postedNews = db.get("postedNewsGuids");
    item.type = "🆕 ADD";

    if(postedNews.has(item.id).value())
        item.type = "✏️ EDIT";

    if(cache.has(item.id).value()) {
        let oldItem = cache.get(item.id).value();
        let oldMessage = await tryFetchMessage(verifyChannel, oldItem.verifyMessageId);
        if(oldMessage)
            await oldMessage.delete();
    }

    let newMessage = await verifyChannel.send(toVerifyString(item));
    
    await newMessage.react("✅");
    await newMessage.react("❌");

    var collector = newMessage.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
    collector.on("collect", collectReaction)
    reactionCollectors[newMessage.id] = collector;
    verifyMessageIdToPostId[newMessage.id] = item.id;
    item.verifyMessageId = newMessage.id;
    cache.set(item.id, item).write();

    oldNews.set(item.id, {
        hash: item.hash,
        messageId: null
    }).write();
}

function collectReaction(reaction, collector) {
    let cache = db.get("newsCache");
    let m = reaction.message;
    collector.stop();
    delete reactionCollectors[m.id];
    let postId = verifyMessageIdToPostId[m.id];
    let item = cache.get(postId).value();
    cache.unset(postId).write();
    m.delete();

    if(reaction.emoji.name == "✅")
        sendNews(item);
}

async function sendNews(item) {
    let outChannel = db.get("feedOutputChannel").value();
    let oldNews = db.get("postedNewsGuids");

    let sentMessage = await postNewsItem(outChannel, item);
    oldNews.set(item.id, {
                    hash: item.hash,
                    messageId: sentMessage.id
                }).write();
}

function isVerifyReaction(reaction, user) {
    return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot;
}

async function tryFetchMessage(channel, messageId) {
    try {
        return await channel.fetchMessage(messageId);
    }catch(error){
        return null;
    }
}

function shouldVerify() {
    return verifyChannelID != "";
}

async function postNewsItem(channel, item) {
    let newsMessage = toNewsString(item);
    let ch = client.channels.get(channel);

    if(item.messageId) {
        let message = await tryFetchMessage(ch, item.messageId);
        if(message)
            return await message.edit(newsMessage);
        else 
            return await ch.send(newsMessage);
    } 
    else 
        return await ch.send(newsMessage);
}

function markdownify(htmStr, link) {
    return turndown.turndown(htmStr).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "");
}

function toNewsString(item) {
    return `**${item.title}**
Posted by ${item.creator}
${item.link}

${item.contents}`;
}

function toVerifyString(item) {
    return `[${item.type}]
Post ID: **${item.id}**
    
${toNewsString(item)}
            
React with ✅ (approve) or ❌ (deny).`;
}

const onStart = () => {
    initPendingReactors();
    interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);
};

const commands = [
    {
        pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i,
        action: async (msg, s, match) => {
            if(msg.channel.id != verifyChannelID)
                return;

            let id = match[1];
            let newContents = match[2].trim();
            
            if(!db.get("newsCache").has(id).value()) {
                msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`);
                return;
            }

            let item = db.get("newsCache").get(id).value();

            let editMsg = await tryFetchMessage(client.channels.get(verifyChannelID), item.verifyMessageId);

            if(!editMsg){
                msg.channel.send(`${msg.author.toString()} No verify messafe found for ${id}! This is a bug: report to horse.`);
                return;
            }

            item.contents = newContents;

            db.get("newsCache").set(id, item).write();
            await editMsg.edit(toVerifyString(item));
            await msg.delete();
        }
    }
];

module.exports = {
    onStart: onStart,
    commands: commands
};