noctbot/commands/rss_checker.js

const TurndownService = require("turndown");
const RSSParser = require("rss-parser");
const db = require("../db.js");
const interval = require("interval-promise");
const client = require("../client.js");

const turndown = new TurndownService();
turndown.addRule("image", {
    filter: "img",
    replacement: () => ""
});
turndown.addRule("link", {
    filter: node => node.nodeName === "A" &&node.getAttribute("href"),
    replacement: (content, node) => node.getAttribute("href")
});

const parser = new RSSParser();
const RSS_UPDATE_INTERVAL_MIN = 5;

function getThreadId(url) {
    let result = url.substring(url.lastIndexOf(".") + 1);
    if(result.endsWith("/"))
        result = result.substring(0, result.length - 1);
    return result;
}

async function checkFeeds() {
    console.log(`Checking feeds on ${new Date().toISOString()}`);
    let feeds = db.get("rssFeeds").value();
    let outlets = db.get("feedOutputs").value();
    let oldNews = db.get("postedNewsGuids");
    for(let feedEntry of feeds) {
        let feed = await parser.parseURL(feedEntry.url);
        if(feed.items.length == 0)
            continue;
        let printableItems = feed.items.filter(i => i.isoDate > feedEntry.lastUpdate).sort((a, b) => a.isoDate.localeCompare(b.isoDate));
        if(printableItems.length > 0) {
            printableItems.forEach(item => {
                let itemID = getThreadId(item.guid);
                if(oldNews.has(itemID).value())
                    return;
                outlets.forEach(ch => {
                    client.channels.get(ch).send(`**${item.title}**\nPosted by ${item.creator}\n${item.link}\n\n${turndown.turndown(item[feedEntry.contentElement]).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(item.link, "")}`);
                });
                oldNews.set(itemID, true).write();
            });
            let lastUpdateDate = printableItems[printableItems.length - 1].isoDate;
            console.log(`Setting last update marker on ${feedEntry.url} to ${lastUpdateDate}`);
            db.get("rssFeeds").find({ url: feedEntry.url}).assign({lastUpdate: lastUpdateDate}).write();
        }
    }
}

const onStart = () => {
    interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);
};

module.exports = {
    onStart: onStart
};