noctbot/src/commands/news_aggregator.ts

import TurndownService, { Options } from "turndown";
import interval from "interval-promise";
import client from "../client";
import sha1 from "sha1";
import * as path from "path";
import * as fs from "fs";

import { IAggregator, NewsPostItem, INewsPostData } from "./aggregators/aggregator";
import { ICommand } from "./command";
import { RichEmbed, TextChannel, Message, Channel } from "discord.js";
import { getRepository } from "typeorm";
import { KnownChannel } from "../entity/KnownChannel";
import { AggroNewsItem } from "../entity/AggroNewsItem";

const UPDATE_INTERVAL = 5;
const MAX_PREVIEW_LENGTH = 300; 

const aggregators : IAggregator[] = [];
let aggregateChannelID : string = null;
const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager";

// TODO: Run BBCode converter instead
const turndown = new TurndownService();
turndown.addRule("image", {
    filter: "img",
    replacement: () => ""
});
turndown.addRule("link", {
    filter: (node : HTMLElement, opts: Options) => node.nodeName === "A" &&node.getAttribute("href") != null,
    replacement: (content: string, node: HTMLElement) => node.getAttribute("href")
});

function markdownify(htmStr: string) {
    return turndown.turndown(htmStr)/*.replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "")*/;
}

async function checkFeeds() {
    console.log(`Aggregating feeds on ${new Date().toISOString()}`);

    let aggregatorJobs = [];

    for(let aggregator of aggregators) {
        aggregatorJobs.push(aggregator.aggregate());    
    }
    let aggregatedItems = await Promise.all(aggregatorJobs);

    for(let itemSet of aggregatedItems) {
        for(let item of itemSet) {
            let itemObj = {
                ...item,
                cacheMessageId: null,
                postedMessageId: null
            } as NewsPostItem;
            itemObj.contents = markdownify(item.contents);
            itemObj.hash = sha1(itemObj.contents);

            await addNewsItem(itemObj);
        }
    }
}

function clipText(text: string) {
    if(text.length <= MAX_PREVIEW_LENGTH)
        return text;

    return `${text.substring(0, MAX_PREVIEW_LENGTH)}...`;
}

// TODO: Replace with proper forum implementation
async function addNewsItem(item: NewsPostItem) {
    let repo = getRepository(AggroNewsItem);

    let newsItem = await repo.findOne({
        where: { feedName: item.feedId, newsId: item.newsId }
    });

    if(newsItem) {
        // No changes, skip
        if(newsItem.hash == item.hash)
            return;
        else
            await deleteCacheMessage(newsItem.editMessageId);
    } else {
        newsItem = repo.create({
            newsId: item.newsId,
            feedName: item.feedId,
            hash: item.hash
        });
    }

    let ch = client.channels.get(aggregateChannelID);
    
    if(!(ch instanceof TextChannel))
        return;

    let msg = await ch.send(new RichEmbed({
        title: item.title,
        url: item.link,
        color: item.embedColor,
        timestamp: new Date(),
        description: clipText(item.contents),
        author: {
            name: item.author
        },
        footer: {
            text: "NoctBot News Aggregator"
        }
    })) as Message;

    newsItem.editMessageId = msg.id;

    await repo.save(newsItem);
}

async function deleteCacheMessage(messageId: string) {
    let ch = client.channels.get(aggregateChannelID);
    if(!(ch instanceof TextChannel))
        return;

    let msg = await tryFetchMessage(ch, messageId);

    if(msg)
        await msg.delete();
}

async function tryFetchMessage(channel : TextChannel, messageId: string) {
    try {
        return await channel.fetchMessage(messageId);
    }catch(error){
        return null;
    }
}

function initAggregators() {
    let aggregatorsPath = path.join(path.dirname(module.filename), "aggregators");
    let files = fs.readdirSync(aggregatorsPath);

    for(let file of files) {
        let ext  = path.extname(file);
        let name = path.basename(file);

        if(name == "aggregator.js")
            continue;
        if(ext != ".js")
            continue;

        let obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator;

        if(obj)
            aggregators.push(obj);

        if(obj.init)
            obj.init();
    }
}

export default {
    onStart : async () => {
        let repo = getRepository(KnownChannel);

        let ch = await repo.findOne({
            where: { channelType: AGGREGATOR_MANAGER_CHANNEL }
        });
        aggregateChannelID = ch.channelId;

        initAggregators();
        interval(checkFeeds, UPDATE_INTERVAL * 60 * 1000);
    }
} as ICommand;