noctbot/bot/src/commands/news_aggregator.ts

import TurndownService, { Options } from "turndown";
import interval from "interval-promise";
import { client, FORUMS_DOMAIN, BotClient } from "../client";
import sha1 from "sha1";
import * as path from "path";
import * as fs from "fs";
import { HTML2BBCode } from "html2bbcode";
import { Dict } from "../util";
import { IAggregator, NewsPostItem } from "./aggregators/aggregator";
import { TextChannel, Message, Channel, ReactionCollector, MessageReaction, User, Collector, MessageEmbed } from "discord.js";
import { getRepository, IsNull, Not } from "typeorm";
import { KnownChannel } from "@shared/db/entity/KnownChannel";
import { AggroNewsItem } from "@shared/db/entity/AggroNewsItem";
import { v3beta1 } from "@google-cloud/translate";
import { CommandSet } from "src/model/command";
const { TranslationServiceClient } = v3beta1;

const UPDATE_INTERVAL = process.env.NODE_ENV == "dev" ? 60 : 5;
const MAX_PREVIEW_LENGTH = 300;
const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager";
const FORUMS_STAGING_ID = 54;
const FORUMS_NEWS_ID = 49;

@CommandSet
export class NewsAggregator {
    tlClient = new TranslationServiceClient();
    aggregators: IAggregator[] = [];
    aggregateChannelID?: string;
    bbCodeParser = new HTML2BBCode();
    turndown = new TurndownService();
    reactionCollectors: Dict<ReactionCollector> = {};
    verifyMessageIdToPost: Dict<AggroNewsItem> = {};

    constructor() {
        this.turndown.addRule("image", {
            filter: "img",
            replacement: () => ""
        });
        this.turndown.addRule("link", {
            filter: (node: HTMLElement, opts: Options) => node.nodeName === "A" && node.getAttribute("href") != null,
            replacement: (content: string, node: Node) => (node instanceof HTMLElement ? node.getAttribute("href") : null) ?? ""
        });
    }

    checkFeeds = async () => {
        console.log(`Aggregating feeds on ${new Date().toISOString()}`);

        let aggregatorJobs = [];

        for (let aggregator of this.aggregators) {
            aggregatorJobs.push(aggregator.aggregate());
        }
        let aggregatedItems = await Promise.all(aggregatorJobs);

        for (let itemSet of aggregatedItems) {
            for (let item of itemSet) {
                let itemObj = {
                    ...item,
                    cacheMessageId: undefined,
                    postedMessageId: undefined
                } as NewsPostItem;

                itemObj.hash = sha1(itemObj.contents);

                await this.addNewsItem(itemObj);
            }
        }
    }

    clipText(text: string) {
        if (text.length <= MAX_PREVIEW_LENGTH)
            return text;

        return `${text.substring(0, MAX_PREVIEW_LENGTH)}...`;
    }

    async addNewsItem(item: NewsPostItem) {
        if (!this.aggregateChannelID)
            return;

        let repo = getRepository(AggroNewsItem);

        let ch = client.bot.channels.resolve(this.aggregateChannelID);

        if (!(ch instanceof TextChannel))
            return;

        let isNew = true;
        let newsItem = await repo.findOne({
            where: { feedName: item.feedId, newsId: item.newsId }
        });

        if (newsItem) {
            if (process.env.IGNORE_CHANGED_NEWS === "TRUE") {
                newsItem.hash = item.hash ?? "";
                await repo.save(newsItem);
                return;
            }

            // No changes, skip
            if (newsItem.hash == item.hash)
                return;
            else if(newsItem.editMessageId)
                await this.deleteCacheMessage(newsItem.editMessageId);
            isNew = false;
        } else {
            newsItem = repo.create({
                newsId: item.newsId,
                feedName: item.feedId,
                hash: item.hash
            });
        }

        if (item.needsTranslation && process.env.GOOGLE_APP_ID)
            try {
                let request = {
                    parent: this.tlClient.locationPath(process.env.GOOGLE_APP_ID, "global"),
                    contents: [item.title, item.contents],
                    mimeType: "text/html",
                    sourceLanguageCode: "ja",
                    targetLanguageCode: "en"
                };

                let [res] = await this.tlClient.translateText(request);
                let [translatedTitle, translatedContents] = res.translations ?? [undefined, undefined];
                
                if (translatedTitle?.translatedText && translatedContents?.translatedText) {
                    item.title = translatedTitle.translatedText;
                    item.contents = translatedContents.translatedText;
                }
            } catch (err) {
                console.log(`Failed to translate because ${err}`);
            }

        item.contents = this.bbCodeParser.feed(item.contents).toString();

        if (!newsItem.forumsEditPostId) {
            let createResponse = await client.forum.createThread(FORUMS_STAGING_ID, item.title, item.contents);
            newsItem.forumsEditPostId = createResponse.thread.thread_id;
        } else if(newsItem.forumsNewsPostId){
            await client.forum.postReply(newsItem.forumsNewsPostId, item.contents);
        }


        let msg = await ch.send(new MessageEmbed({
            title: item.title,
            url: item.link,
            color: item.embedColor,
            timestamp: new Date(),
            description: `${(isNew ? "**[NEW]**" : "**[EDIT]**")}\n[**Edit on forums**](${FORUMS_DOMAIN}/index.php?threads/.${newsItem.forumsEditPostId}/)`,
            author: {
                name: item.author
            },
            footer: {
                text: "NoctBot News Aggregator"
            }
        })) as Message;

        newsItem.editMessageId = msg.id;

        await msg.react("✅");
        await msg.react("❌");

        let collector = msg.createReactionCollector(this.isVerifyReaction, { maxEmojis: 1 });
        collector.on("collect", this.collectReaction)
        this.reactionCollectors[msg.id] = collector;
        this.verifyMessageIdToPost[msg.id] = newsItem;

        await repo.save(newsItem);
    }

    isVerifyReaction(reaction: MessageReaction, user: User) {
        return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot && user.id != client.botUser.id;
    }

    collectReaction = async (reaction: MessageReaction, collector: Collector<string, MessageReaction>) => {
        let repo = getRepository(AggroNewsItem);

        let m = reaction.message;
        collector.stop();
        delete this.reactionCollectors[m.id];
        let post = this.verifyMessageIdToPost[m.id];

        if(!post.forumsEditPostId) {
            throw new Error("No forum edit post found!");
        }

        if (reaction.emoji.name == "✅") {
            let res = await client.forum.getThread(post.forumsEditPostId);
            let forumPost = await client.forum.getPost(res.thread.first_post_id);

            if (!post.forumsNewsPostId) {
                let newThread = await client.forum.createThread(FORUMS_NEWS_ID, res.thread.title, forumPost.message);
                post.forumsNewsPostId = newThread.thread.thread_id;
            } else {
                let curThread = await client.forum.editThread(post.forumsNewsPostId, {
                    title: res.thread.title
                });

                await client.forum.editPost(curThread.thread.first_post_id, {
                    message: forumPost.message
                });
            }
        }

        await client.forum.deleteThread(post.forumsEditPostId);
        await repo.update({ newsId: post.newsId, feedName: post.feedName }, { 
                editMessageId: undefined, 
                forumsEditPostId: undefined, 
                forumsNewsPostId: post.forumsNewsPostId });
        await reaction.message.delete();
        delete this.verifyMessageIdToPost[m.id];
    };

    async deleteCacheMessage(messageId: string) {
        if(!this.aggregateChannelID)
            return;
        let ch = client.bot.channels.resolve(this.aggregateChannelID);
        if (!(ch instanceof TextChannel))
            return;

        let msg = await this.tryFetchMessage(ch, messageId);

        if (msg)
            await msg.delete();
    }

    async tryFetchMessage(channel: Channel, messageId: string) {
        try {
            if (!(channel instanceof TextChannel))
                return null;
            return await channel.messages.fetch(messageId);
        } catch (error) {
            return null;
        }
    }

    initAggregators() {
        let aggregatorsPath = path.join(path.dirname(module.filename), "aggregators");
        let files = fs.readdirSync(aggregatorsPath);

        for (let file of files) {
            let ext = path.extname(file);
            let name = path.basename(file);

            if (name == "aggregator.js")
                continue;
            if (ext != ".js")
                continue;

            let obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator;

            if (obj)
                this.aggregators.push(obj);

            if (obj.init)
                obj.init();
        }
    }

    async initPendingReactors() {
        if(!this.aggregateChannelID)
            return;

        let verifyChannel = client.bot.channels.resolve(this.aggregateChannelID);
        if(!verifyChannel)
            throw new Error("Couldn't find verify channel!");

        let repo = getRepository(AggroNewsItem);

        let pendingVerifyMessages = await repo.find({
            where: { editMessageId: Not(IsNull()) }
        });

        for (let msg of pendingVerifyMessages) {
            let m = await this.tryFetchMessage(verifyChannel, msg.editMessageId!);

            if (!m) {
                await repo.update({ feedName: msg.feedName, newsId: msg.newsId }, { editMessageId: undefined });
                continue;
            }

            let collector = m.createReactionCollector(this.isVerifyReaction, { maxEmojis: 1 });
            collector.on("collect", this.collectReaction);
            this.reactionCollectors[m.id] = collector;
            this.verifyMessageIdToPost[m.id] = msg;
        }
    }

    async onStart() {
        let repo = getRepository(KnownChannel);

        let ch = await repo.findOne({
            where: { channelType: AGGREGATOR_MANAGER_CHANNEL }
        });

        if (!ch)
            return;

        this.aggregateChannelID = ch.channelId;

        await this.initPendingReactors();
        this.initAggregators();
        interval(this.checkFeeds, UPDATE_INTERVAL * 60 * 1000);
    }
}