noctbot/commands/news_aggregator.js

const TurndownService = require("turndown");
const RSSParser = require("rss-parser");
const db = require("../db.js");
const interval = require("interval-promise");
const client = require("../client.js");
const sha1 = require("sha1");
const html = require("node-html-parser"); 
const axios = require("axios");
const path = require("path");
const fs = require("fs");
const Discord = require("discord.js");

const UPDATE_INTERVAL = 5;

const aggregators = [];
const aggregateChannelID = db.get("aggregateChannel").value();

// TODO: Run BBCode converter instead
const turndown = new TurndownService();
turndown.addRule("image", {
    filter: "img",
    replacement: () => ""
});
turndown.addRule("link", {
    filter: node => node.nodeName === "A" &&node.getAttribute("href"),
    replacement: (content, node) => node.getAttribute("href")
});

function markdownify(htmStr, link) {
    return turndown.turndown(htmStr)/*.replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "")*/;
}

async function checkFeeds() {
    console.log(`Aggregating feeds on ${new Date().toISOString()}`);

    let aggregatorJobs = [];

    for(let aggregator of aggregators) {
        if(aggregator.aggregate)
            aggregatorJobs.push(aggregator.aggregate());    
    }
    let aggregatedItems = await Promise.all(aggregatorJobs);

    for(let itemSet of aggregatedItems) {
        for(let item of itemSet) {
            let itemObj = {
                id: item.id,
                link: item.link || "",
                title: item.title || "",
                author: item.author,
                contents: markdownify(item.contents, item.link),
                hash: null,
                cacheMessageId: null,
                postedMessageId: null,
                embedColor: item.embedColor || 0xffffffff
            };
            itemObj.hash = sha1(itemObj.contents);

            await addNewsItem(itemObj);
        }
    }
}

// TODO: Replace with proper forum implementation
async function addNewsItem(item) {
    let aggrItems = db.get("aggregatedItemsCache");

    if(aggrItems.has(item.id).value()) {
        let postedItem = aggrItems.get(item.id).value();

        // No changes, skip
        if(postedItem.hash == item.hash)
            return;
        else
            await deleteCacheMessage(postedItem.cacheMessageId);
    }

    let ch = client.channels.get(aggregateChannelID);

    let msg = await ch.send(new Discord.RichEmbed({
        title: item.title,
        url: item.link,
        color: item.embedColor,
        timestamp: new Date(),
        description: `${item.contents.substring(0, Math.min(item.contents.length, 300))}...`,
        author: {
            name: item.author
        },
        footer: {
            text: "NoctBot News Aggregator"
        }
    }));

    aggrItems.set(item.id, {
        hash: item.hash,
        cacheMessageId: msg.id,
        postedMessageId: null
    }).write();
}

async function deleteCacheMessage(messageId) {
    let ch = client.channels.get(aggregateChannelID);
    let msg = await tryFetchMessage(ch, messageId);

    if(msg)
        await msg.delete();
}

async function tryFetchMessage(channel, messageId) {
    try {
        return await channel.fetchMessage(messageId);
    }catch(error){
        return null;
    }
}

function initAggregators() {
    let aggregatorsPath = path.join(path.dirname(module.filename), "aggregators");
    let files = fs.readdirSync(aggregatorsPath);

    for(let file of files) {
        let ext  = path.extname(file);
        if(ext != ".js")
            continue;

        let obj = require(path.resolve(aggregatorsPath, file));

        if(obj)
            aggregators.push(obj);

        if(obj.init)
            obj.init();
    }
}

function onStart() {
    initAggregators();
    interval(checkFeeds, UPDATE_INTERVAL * 1000);
};

module.exports = {
    onStart: onStart
};