noctbot/bot/src/commands/news_aggregator.ts

import TurndownService, { Options } from "turndown";
import interval from "interval-promise";
import { client, forumClient, FORUMS_DOMAIN } from "../client";
import sha1 from "sha1";
import * as path from "path";
import * as fs from "fs";
import { HTML2BBCode } from "html2bbcode";
import { Dict } from "../util";

import { IAggregator, NewsPostItem, INewsPostData } from "./aggregators/aggregator";
import { ICommand } from "./command";
import { RichEmbed, TextChannel, Message, Channel, ReactionCollector, MessageReaction, User, Collector } from "discord.js";
import { getRepository, IsNull, Not } from "typeorm";
import { KnownChannel } from "@db/entity/KnownChannel";
import { AggroNewsItem } from "@db/entity/AggroNewsItem";

import { v3beta1 } from "@google-cloud/translate";

const { TranslationServiceClient } = v3beta1;
const tlClient = new TranslationServiceClient();

const UPDATE_INTERVAL = 5;
const MAX_PREVIEW_LENGTH = 300; 

const aggregators : IAggregator[] = [];
let aggregateChannelID : string = null;
const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager";

const FORUMS_STAGING_ID = 54;
const FORUMS_NEWS_ID = 49;
const bbCodeParser = new HTML2BBCode();

const reactionCollectors: Dict<ReactionCollector> = {};
const verifyMessageIdToPost: Dict<AggroNewsItem> = {};

// TODO: Run BBCode converter instead
const turndown = new TurndownService();
turndown.addRule("image", {
    filter: "img",
    replacement: () => ""
});
turndown.addRule("link", {
    filter: (node : HTMLElement, opts: Options) => node.nodeName === "A" &&node.getAttribute("href") != null,
    replacement: (content: string, node: HTMLElement) => node.getAttribute("href")
});

function markdownify(htmStr: string) {
    return turndown.turndown(htmStr)/*.replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "")*/;
}

async function checkFeeds() {
    console.log(`Aggregating feeds on ${new Date().toISOString()}`);

    let aggregatorJobs = [];

    for(let aggregator of aggregators) {
        aggregatorJobs.push(aggregator.aggregate());    
    }
    let aggregatedItems = await Promise.all(aggregatorJobs);

    for(let itemSet of aggregatedItems) {
        for(let item of itemSet) {
            let itemObj = {
                ...item,
                cacheMessageId: null,
                postedMessageId: null
            } as NewsPostItem;

            itemObj.hash = sha1(itemObj.contents);

            await addNewsItem(itemObj);
        }
    }
}

function clipText(text: string) {
    if(text.length <= MAX_PREVIEW_LENGTH)
        return text;

    return `${text.substring(0, MAX_PREVIEW_LENGTH)}...`;
}

// TODO: Replace with proper forum implementation
async function addNewsItem(item: NewsPostItem) {
    let repo = getRepository(AggroNewsItem);

    let ch = client.channels.get(aggregateChannelID);

    if(!(ch instanceof TextChannel))
        return;

    let isNew = true;
    let newsItem = await repo.findOne({
        where: { feedName: item.feedId, newsId: item.newsId }
    });

    if(newsItem) {
        // No changes, skip
        if(newsItem.hash == item.hash)
            return;
        else
            await deleteCacheMessage(newsItem.editMessageId);
        isNew = false;
    } else {
        newsItem = repo.create({
            newsId: item.newsId,
            feedName: item.feedId,
            hash: item.hash
        });
    }

    if(item.needsTranslation)
        try {
            let request = {
                parent: tlClient.locationPath(process.env.GOOGLE_APP_ID, "global"),
                contents: [ item.title, item.contents ],
                mimeType: "text/html",
                sourceLanguageCode: "ja",
                targetLanguageCode: "en"
            };

            let [ res ] = await tlClient.translateText(request);

            item.title = res.translations[0].translatedText
            item.contents = res.translations[1].translatedText;
        } catch(err) {
            console.log(`Failed to translate because ${err}`);
        }

    item.contents = bbCodeParser.feed(item.contents).toString();

    if(!newsItem.forumsEditPostId) {
        let createResponse = await forumClient.createThread(FORUMS_STAGING_ID, item.title, item.contents);
        newsItem.forumsEditPostId = createResponse.thread.thread_id;
    } else {
        await forumClient.postReply(newsItem.forumsNewsPostId, item.contents);
    }
    

    let msg = await ch.send(new RichEmbed({
        title: item.title,
        url: item.link,
        color: item.embedColor,
        timestamp: new Date(),
        description: `${(isNew ? "**[NEW]**" : "**[EDIT]**")}\n[**Edit on forums**](${FORUMS_DOMAIN}/index.php?threads/.${newsItem.forumsEditPostId}/)`,
        author: {
            name: item.author
        },
        footer: {
            text: "NoctBot News Aggregator"
        }
    })) as Message;

    newsItem.editMessageId = msg.id;

    await msg.react("✅");
    await msg.react("❌");

    let collector = msg.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
    collector.on("collect", collectReaction)
    reactionCollectors[msg.id] = collector;
    verifyMessageIdToPost[msg.id] = newsItem;

    await repo.save(newsItem);
}

function isVerifyReaction(reaction: MessageReaction, user: User) {
    return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot && user.id != client.user.id;
}

async function collectReaction(reaction: MessageReaction, collector: Collector<string, MessageReaction>) {
    let repo = getRepository(AggroNewsItem);

    let m = reaction.message;
    collector.stop();
    delete reactionCollectors[m.id];
    let post = verifyMessageIdToPost[m.id];
    
    if (reaction.emoji.name == "✅") {
        let res = await forumClient.getThread(post.forumsEditPostId);
        let forumPost = await forumClient.getPost(res.thread.first_post_id);

        if(!post.forumsNewsPostId) {
            let newThread = await forumClient.createThread(FORUMS_NEWS_ID, res.thread.title, forumPost.message);
            post.forumsNewsPostId = newThread.thread.thread_id;
        } else {
            let curThread = await forumClient.editThread(post.forumsNewsPostId, {
                title: res.thread.title
            });

            await forumClient.editPost(curThread.thread.first_post_id, {
                message: forumPost.message
            });
        }
    }
    
    await forumClient.deleteThread(post.forumsEditPostId);
    await repo.update({ newsId: post.newsId, feedName: post.feedName }, { editMessageId: null, forumsEditPostId: null, forumsNewsPostId: post.forumsNewsPostId });
    await reaction.message.delete();
    delete verifyMessageIdToPost[m.id];
}

async function deleteCacheMessage(messageId: string) {
    let ch = client.channels.get(aggregateChannelID);
    if(!(ch instanceof TextChannel))
        return;

    let msg = await tryFetchMessage(ch, messageId);

    if(msg)
        await msg.delete();
}

async function tryFetchMessage(channel: Channel, messageId: string) {
    try {
        if (!(channel instanceof TextChannel))
            return null;
        return await channel.fetchMessage(messageId);
    } catch (error) {
        return null;
    }
}

function initAggregators() {
    let aggregatorsPath = path.join(path.dirname(module.filename), "aggregators");
    let files = fs.readdirSync(aggregatorsPath);

    for(let file of files) {
        let ext  = path.extname(file);
        let name = path.basename(file);

        if(name == "aggregator.js")
            continue;
        if(ext != ".js")
            continue;

        let obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator;

        if(obj)
            aggregators.push(obj);

        if(obj.init)
            obj.init();
    }
}

async function initPendingReactors() {
    let verifyChannel = client.channels.get(aggregateChannelID);

    let repo = getRepository(AggroNewsItem);

    let pendingVerifyMessages = await repo.find({
        where: { editMessageId: Not(IsNull()) }
    });

    for (let msg of pendingVerifyMessages) {
        let m = await tryFetchMessage(verifyChannel, msg.editMessageId);

        if (!m) {
            await repo.update({ feedName: msg.feedName, newsId: msg.newsId }, { editMessageId: null });
            continue;
        }

        let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
        collector.on("collect", collectReaction);
        reactionCollectors[m.id] = collector;
        verifyMessageIdToPost[m.id] = msg;
    }
}

export default {
    onStart : async () => {
        let repo = getRepository(KnownChannel);

        let ch = await repo.findOne({
            where: { channelType: AGGREGATOR_MANAGER_CHANNEL }
        });

        if(!ch)
            return;
        
        aggregateChannelID = ch.channelId;

        await initPendingReactors();
        initAggregators();
        interval(checkFeeds, UPDATE_INTERVAL * 60 * 1000);
    }
} as ICommand;