noctbot/bot/src/commands/forums_news_checker.ts

import TurndownService, { Options } from "turndown";
import RSSParser from "rss-parser";
import interval from "interval-promise";
import { client, forumClient } from "../client";
import sha1 from "sha1";
import { ICommand } from "./command";
import { TextChannel, Message, ReactionCollector, MessageReaction, Collector, User, Channel } from "discord.js";
import { Dict } from "../util";
import { getRepository, Not, IsNull } from "typeorm";
import { PostedForumNewsItem } from "@db/entity/PostedForumsNewsItem";
import { KnownChannel } from "@db/entity/KnownChannel";
import { PostVerifyMessage } from "@db/entity/PostVerifyMessage";
import bbobHTML from '@bbob/html'
import presetHTML5 from '@bbob/preset-html5'

const PREVIEW_CHAR_LIMIT = 300;
const NEWS_POST_VERIFY_CHANNEL = "newsPostVerify";

let verifyChannelId: string = null;
const reactionCollectors: Dict<ReactionCollector> = {};
const verifyMessageIdToPost: Dict<PostedForumNewsItem> = {};
const NEWS_FEED_CHANNEL = "newsFeed";

let botUserId = 0;

const turndown = new TurndownService();
turndown.addRule("image", {
    filter: "img",
    replacement: () => ""
});
turndown.addRule("link", {
    filter: (node: HTMLElement, opts: Options) => node.nodeName === "A" && node.getAttribute("href") != null,
    replacement: (content: string, node: HTMLElement) => node.getAttribute("href")
});

const parser = new RSSParser();
const RSS_UPDATE_INTERVAL_MIN = 5;

function getThreadId(url: string) {
    let result = url.substring(url.lastIndexOf(".") + 1);
    if (result.endsWith("/"))
        result = result.substring(0, result.length - 1);
    return result;
}

const NEWS_FORUM_ID = 49;

const FEEDS = [
    {
        url: "http://custommaid3d2.com/index.php?forums/news.49/index.rss",
        contentElement: "content:encoded"
    }
];

function bbCodeToMarkdown(bbCode: string) {
    return turndown.turndown(bbobHTML(bbCode, presetHTML5())).replace(/( {2}\n|\n\n){2,}/gm, "\n");
}

async function checkFeeds() {
    console.log(`Checking feeds on ${new Date().toISOString()}`);
    let forumsNewsRepo = getRepository(PostedForumNewsItem);
    let postVerifyMessageRepo = getRepository(PostVerifyMessage);

    let forumThreads = await forumClient.getForumThreads(NEWS_FORUM_ID);

    for (let thread of forumThreads.threads) {
        let firstPost = await forumClient.getPost(thread.first_post_id);

        let contents = bbCodeToMarkdown(firstPost.message);
        let itemObj = forumsNewsRepo.create({
            id: thread.thread_id.toString(),
            hash: sha1(firstPost.message),
            verifyMessage: postVerifyMessageRepo.create({
                author: thread.username,
                link: `https://custommaid3d2.com/index.php?threads/${thread.thread_id}/`,
                title: thread.title,
                text: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
                isNew: true
            })
        });

        let postItem = await forumsNewsRepo.findOne({
            where: { id: itemObj.id },
            relations: ["verifyMessage"]
        });

        if (postItem) {

            if(process.env.INGORE_CHANGED_NEWS === "TRUE") {
                await forumsNewsRepo.update({
                    id: postItem.id
                }, {
                    hash: itemObj.hash
                });
                continue;
            }

            // Add message ID to mark for edit
            if (postItem.hash != itemObj.hash) {
                let newHash = itemObj.hash;
                if (!postItem.verifyMessage)
                    postItem.verifyMessage = itemObj.verifyMessage;

                itemObj = postItem;
                itemObj.verifyMessage.isNew = false;
                itemObj.hash = newHash;
            }
            else
                continue;
        }

        if (!shouldVerify() || firstPost.user_id == botUserId)
            await sendNews(itemObj);
        else
            await addVerifyMessage(itemObj);
    }
}

async function initPendingReactors() {
    let verifyChannel = client.channels.get(verifyChannelId);

    let repo = getRepository(PostedForumNewsItem);
    let verifyMessageRepo = getRepository(PostVerifyMessage);

    let pendingVerifyMessages = await repo.find({
        where: { verifyMessage: Not(IsNull()) },
        select: ["id"],
        relations: ["verifyMessage"]
    });

    for (let msg of pendingVerifyMessages) {
        let m = await tryFetchMessage(verifyChannel, msg.verifyMessage.messageId);

        if (!m) {
            await repo.update({ id: msg.id }, { verifyMessage: null });
            await verifyMessageRepo.delete(msg.verifyMessage);
            continue;
        }

        let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
        collector.on("collect", collectReaction);
        reactionCollectors[m.id] = collector;
        verifyMessageIdToPost[m.id] = msg;
    }
}

async function addVerifyMessage(item: PostedForumNewsItem) {
    let verifyChannel = client.channels.get(verifyChannelId) as TextChannel;
    let verifyMessageRepo = getRepository(PostVerifyMessage);
    let forumsNewsRepo = getRepository(PostedForumNewsItem);

    if (item.verifyMessage.messageId) {
        let oldMessage = await tryFetchMessage(verifyChannel, item.verifyMessage.messageId);
        if (oldMessage)
            await oldMessage.delete();
    }

    let newMessage = await verifyChannel.send(toVerifyString(item.id, item.verifyMessage)) as Message;
    item.verifyMessage.messageId = newMessage.id;

    await newMessage.react("✅");
    await newMessage.react("❌");

    let collector = newMessage.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
    collector.on("collect", collectReaction)
    reactionCollectors[newMessage.id] = collector;
    verifyMessageIdToPost[newMessage.id] = item;

    item.verifyMessage = await verifyMessageRepo.save(item.verifyMessage);
    await forumsNewsRepo.save(item);
}

async function collectReaction(reaction: MessageReaction, collector: Collector<string, MessageReaction>) {
    let verifyMessageRepo = getRepository(PostVerifyMessage);
    let postRepo = getRepository(PostedForumNewsItem);

    let m = reaction.message;
    collector.stop();
    delete reactionCollectors[m.id];
    let post = verifyMessageIdToPost[m.id];

    await postRepo.update({ id: post.id }, { verifyMessage: null });
    await verifyMessageRepo.delete({ id: post.verifyMessage.id });
    await reaction.message.delete();

    if (reaction.emoji.name == "✅")
        sendNews(post);

    delete verifyMessageIdToPost[m.id];
}

async function sendNews(item: PostedForumNewsItem) {
    let channelRepo = getRepository(KnownChannel);
    let newsPostRepo = getRepository(PostedForumNewsItem);

    let outChannel = await channelRepo.findOne({
        where: { channelType: NEWS_FEED_CHANNEL }
    });

    let sentMessage = await postNewsItem(outChannel.channelId, item);

    item.postedMessageId = sentMessage.id;
    item.verifyMessage = null;

    await newsPostRepo.save(item);
}

function isVerifyReaction(reaction: MessageReaction, user: User) {
    return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot;
}

async function tryFetchMessage(channel: Channel, messageId: string) {
    try {
        if (!(channel instanceof TextChannel))
            return null;
        return await channel.fetchMessage(messageId);
    } catch (error) {
        return null;
    }
}

function shouldVerify() {
    return verifyChannelId != null;
}

async function postNewsItem(channel: string, item: PostedForumNewsItem): Promise<Message | null> {
    let newsMessage = toNewsString(item.verifyMessage);
    let ch = client.channels.get(channel);

    if (!(ch instanceof TextChannel))
        return null;

    if (item.postedMessageId) {
        let message = await tryFetchMessage(ch, item.postedMessageId);
        if (message)
            return await message.edit(newsMessage);
        else
            return await ch.send(newsMessage) as Message;
    }
    else
        return await ch.send(newsMessage) as Message;
}

function markdownify(htmStr: string, link: string) {
    return turndown.turndown(htmStr).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "");
}

function toNewsString(item: PostVerifyMessage) {
    return `**${item.title}**
Posted by ${item.author}
${item.link}

${item.text}`;
}

function toVerifyString(postId: string, item: PostVerifyMessage) {
    return `[${item.isNew ? "🆕 ADD" : "✏️ EDIT"}]
Post ID: **${postId}**
    
${toNewsString(item)}
            
React with ✅ (approve) or ❌ (deny).`;
}

export default {
    commands: [
        {
            pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i,
            action: async (msg, s, match) => {
                if (msg.channel.id != verifyChannelId)
                    return;

                let id = match[1];
                let newContents = match[2].trim();

                let repo = getRepository(PostedForumNewsItem);
                let verifyRepo = getRepository(PostVerifyMessage);

                let post = await repo.findOne({
                    where: { id: id },
                    relations: ["verifyMessage"]
                });

                if (!post || !post.verifyMessage) {
                    msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`);
                    return;
                }

                let editMsg = await tryFetchMessage(client.channels.get(verifyChannelId), post.verifyMessage.messageId);

                if (!editMsg) {
                    msg.channel.send(`${msg.author.toString()} No verify message found for ${id}! This is a bug: report to horse.`);
                    return;
                }

                post.verifyMessage.text = newContents;

                await verifyRepo.save(post.verifyMessage);
                await editMsg.edit(toVerifyString(post.id, post.verifyMessage));
                await msg.delete();
            }
        }
    ],
    onStart: async () => {

        let repo = getRepository(KnownChannel);

        let verifyChannel = await repo.findOne({
            channelType: NEWS_POST_VERIFY_CHANNEL
        });

        if (!verifyChannel)
            return;
        
        verifyChannelId = verifyChannel.channelId;

        let user = await forumClient.getMe();
        botUserId = user.user_id;

        await initPendingReactors();
        interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);
    }
} as ICommand;