noctbot/commands/rss_checker.js

const TurndownService = require("turndown");
const RSSParser = require("rss-parser");
const db = require("../db.js");
const interval = require("interval-promise");
const client = require("../client.js");
const sha1 = require("sha1");

const verifyChannelID = db.get("newsPostVerifyChannel").value();

const reactionCollectors = {};
const verifyMessageIdToPostId = {};

const turndown = new TurndownService();
turndown.addRule("image", {
    filter: "img",
    replacement: () => ""
});
turndown.addRule("link", {
    filter: node => node.nodeName === "A" &&node.getAttribute("href"),
    replacement: (content, node) => node.getAttribute("href")
});

const parser = new RSSParser();
const RSS_UPDATE_INTERVAL_MIN = 5;

function getThreadId(url) {
    let result = url.substring(url.lastIndexOf(".") + 1);
    if(result.endsWith("/"))
        result = result.substring(0, result.length - 1);
    return result;
}

async function checkFeeds() {
    console.log(`Checking feeds on ${new Date().toISOString()}`);
    let feeds = db.get("rssFeeds").value();
    let oldNews = db.get("postedNewsGuids");
    for(let feedEntry of feeds) {
        let feed = await parser.parseURL(feedEntry.url);
        if(feed.items.length == 0)
            continue;
        let printableItems = feed.items.sort((a, b) => a.isoDate.localeCompare(b.isoDate));
        if(printableItems.length > 0) {
            for(let item of printableItems) {
                let itemID = getThreadId(item.guid);

                let contents = item[feedEntry.contentElement];
                let itemObj = {
                    id: itemID,
                    title: item.title,
                    link: item.link,
                    creator: item.creator,
                    contents: contents,
                    hash: null,
                    messageId: null,
                    verifyMessageId: null
                };

                itemObj.contents = newsToString(itemObj);
                itemObj.hash = sha1(itemObj.contents);

                if(oldNews.has(itemObj.id).value()){
                    let data = oldNews.get(itemObj.id).value();
                    // Old type, don't care
                    if(data === true)
                        continue;

                    // Add message ID to mark for edit
                    if(data.hash != itemObj.hash)
                        itemObj.messageId = data.messageId;
                    else 
                        continue;
                }

                if(!shouldVerify())
                    await sendNews(itemObj);
                else
                    await addVerifyMessage(itemObj);
            }
            let lastUpdateDate = printableItems[printableItems.length - 1].isoDate;
            console.log(`Setting last update marker on ${feedEntry.url} to ${lastUpdateDate}`);
            db.get("rssFeeds").find({ url: feedEntry.url}).assign({lastUpdate: lastUpdateDate}).write();
        }
    }
}

function initPendingReactors() {
    let verifyChannel = client.channels.get(verifyChannelID);
    db.get("newsCache").forOwn(async i => {
        let m = await tryFetchMessage(verifyChannel, i.verifyChannelID);
        let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
        collector.on("collect", collectReaction)
        reactionCollectors[m.id] = collector;
        verifyMessageIdToPostId[m.id] = i.id;
    }).value();
}

async function addVerifyMessage(item) {
    let verifyChannel = client.channels.get(verifyChannelID);
    let cache = db.get("newsCache");
    let oldNews = db.get("postedNewsGuids");
    let postedNews = db.get("postedNewsGuids");
    let process = "🆕 ADD";

    if(postedNews.has(item.id).value())
        process = "✏️ EDIT";

    if(cache.has(item.id).value()) {
        let oldItem = cache.get(item.id).value();
        let oldMessage = await tryFetchMessage(verifyChannel, oldItem.verifyMessageId);
        if(oldMessage)
            await oldMessage.delete();
    }

    let newMessage = await verifyChannel.send(`[${process}]
Post ID: **${item.id}**

${item.contents}
        
React with ✅ (approve) or ❌ (deny).`
);

    
    await newMessage.react("✅");
    await newMessage.react("❌");

    var collector = newMessage.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
    collector.on("collect", collectReaction)
    reactionCollectors[newMessage.id] = collector;
    verifyMessageIdToPostId[newMessage.id] = item.id;
    item.verifyMessageId = newMessage.id;
    cache.set(item.id, item).write();

    oldNews.set(item.id, {
        hash: item.hash,
        messageId: null
    }).write();
}

function collectReaction(reaction, collector) {
    let cache = db.get("newsCache");
    let m = reaction.message;
    collector.stop();
    delete reactionCollectors[m.id];
    let postId = verifyMessageIdToPostId[m.id];
    let item = cache.get(postId).value();
    cache.unset(postId).write();
    m.delete();

    if(reaction.emoji.name == "✅")
        sendNews(item);
}

async function sendNews(item) {
    let outChannel = db.get("feedOutputChannel").value();
    let oldNews = db.get("postedNewsGuids");

    let sentMessage = await postNewsItem(outChannel, item);
    oldNews.set(item.id, {
                    hash: item.hash,
                    messageId: sentMessage.id
                }).write();
}

function isVerifyReaction(reaction, user) {
    return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot;
}

async function tryFetchMessage(channel, messageId) {
    try {
        return await channel.fetchMessage(messageId);
    }catch(error){
        return null;
    }
}

function shouldVerify() {
    return verifyChannelID != "";
}

async function postNewsItem(channel, item) {
    let ch = client.channels.get(channel);

    if(item.messageId) {
        let message = await tryFetchMessage(ch, item.messageId);
        if(message)
            return await message.edit(item.contents);
        else 
            return await ch.send(item.contents);
    } 
    else 
        return await ch.send(item.contents);
}

function newsToString(item) {
    return `**${item.title}**
Posted by ${item.creator}
${item.link}

${turndown.turndown(item.contents).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(item.link, "")}`;
}

const onStart = () => {
    initPendingReactors();
    interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);
};

module.exports = {
    onStart: onStart
};