noctbot/bot/src/plugins/news_aggregator.ts

import TurndownService from "turndown";
import interval from "interval-promise";
import { client } from "../client";
import sha1 from "sha1";
import * as path from "path";
import * as fs from "fs";
import { HTML2BBCode } from "html2bbcode";
import { IAggregator, NewsPostItem } from "./aggregators/aggregator";
import { TextChannel, Message, MessageEmbed } from "discord.js";
import { getRepository, } from "typeorm";
import { KnownChannel } from "@shared/db/entity/KnownChannel";
import { AggroNewsItem } from "@shared/db/entity/AggroNewsItem";
import { logger } from "src/logging";
import { Plugin } from "src/model/plugin";
import { assertOk, tryDo } from "@shared/common/async_utils";

const UPDATE_INTERVAL = process.env.NODE_ENV == "dev" ? 60 : 5;
const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager";

@Plugin
export class NewsAggregator {
    aggregators: IAggregator[] = [];
    aggregateChannelID?: string;
    bbCodeParser = new HTML2BBCode();
    turndown = new TurndownService();

    constructor() {
        this.turndown.addRule("image", {
            filter: "img",
            replacement: () => ""
        });
        this.turndown.addRule("link", {
            filter: (node: HTMLElement) => node.nodeName === "A" && node.getAttribute("href") != null,
            replacement: (content: string, node: Node) => (node as HTMLElement).getAttribute("htef") ?? ""
        });
    }

    checkFeeds = async (): Promise<void> => {
        logger.info(`Aggregating feeds on ${new Date().toISOString()}`);

        const aggregatorJobs = [];

        for (const aggregator of this.aggregators) {
            aggregatorJobs.push(aggregator.aggregate());
        }
        const aggregatedItems = await Promise.all(aggregatorJobs);

        for (const itemSet of aggregatedItems) {
            for (const item of itemSet) {
                const itemObj = {
                    ...item,
                    cacheMessageId: undefined,
                    postedMessageId: undefined
                } as NewsPostItem;

                itemObj.hash = sha1(itemObj.contents);

                await this.addNewsItem(itemObj);
            }
        }
    }

    async addNewsItem(item: NewsPostItem): Promise<void> {
        if (!this.aggregateChannelID)
            return;

        const repo = getRepository(AggroNewsItem);

        const ch = client.bot.channels.resolve(this.aggregateChannelID);

        if (!(ch instanceof TextChannel))
            return;

        let isNew = true;
        let newsItem = await repo.findOne({
            where: { feedName: item.feedId, newsId: item.newsId }
        });

        if (newsItem) {
            if (process.env.IGNORE_CHANGED_NEWS === "TRUE") {
                newsItem.hash = item.hash ?? "";
                await repo.save(newsItem);
                return;
            }

            // No changes, skip
            if (newsItem.hash == item.hash)
                return;
            else if(newsItem.editMessageId)
                await this.deleteCacheMessage(newsItem.editMessageId);
            isNew = false;
        } else {
            newsItem = repo.create({
                newsId: item.newsId,
                feedName: item.feedId,
                hash: item.hash
            });
        }

        const msg = await assertOk(ch.send(new MessageEmbed({
            title: `${(isNew ? "**[NEW]**" : "**[EDIT]**")} ${item.title}`,
            url: item.link,
            color: item.embedColor,
            timestamp: new Date(),
            author: {
                name: item.author
            },
            footer: {
                text: "NoctBot News Aggregator"
            }
        }))) as Message;

        newsItem.editMessageId = msg.id;

        await repo.save(newsItem);
    }

    async deleteCacheMessage(messageId: string): Promise<void> {
        if(!this.aggregateChannelID)
            return;
        const ch = client.bot.channels.resolve(this.aggregateChannelID);
        if (!(ch instanceof TextChannel))
            return;

        const msg = await tryDo(ch.messages.fetch(messageId));

        if (msg.ok) {
            const deleteResult = await tryDo(msg.result.delete());
            if (!deleteResult.ok)
                logger.error("NewsAggregator: failed to delete message %s: %s", messageId, deleteResult.error);
        } 
        else
            logger.error("NewsAggregator: failed to fetch messate %s: %s", this.aggregateChannelID, msg.error);
    }

    initAggregators(): void {
        const aggregatorsPath = path.join(path.dirname(module.filename), "aggregators");
        const files = fs.readdirSync(aggregatorsPath);

        for (const file of files) {
            const ext = path.extname(file);
            const name = path.basename(file);

            if (name == "aggregator.js")
                continue;
            if (ext != ".js")
                continue;

            // eslint-disable-next-line @typescript-eslint/no-var-requires
            const obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator;

            if (obj)
                this.aggregators.push(obj);

            if (obj.init)
                obj.init();
        }
    }


    async start(): Promise<void> {
        const repo = getRepository(KnownChannel);

        const ch = await repo.findOne({
            where: { channelType: AGGREGATOR_MANAGER_CHANNEL }
        });

        if (!ch)
            return;

        this.aggregateChannelID = ch.channelId;

        this.initAggregators();
        interval(this.checkFeeds, UPDATE_INTERVAL * 60 * 1000);
    }
}