123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288 |
- import TurndownService, { Options } from "turndown";
- import interval from "interval-promise";
- import { client, forumClient, FORUMS_DOMAIN } from "../client";
- import sha1 from "sha1";
- import * as path from "path";
- import * as fs from "fs";
- import { HTML2BBCode } from "html2bbcode";
- import { Dict } from "../util";
- import { IAggregator, NewsPostItem, INewsPostData } from "./aggregators/aggregator";
- import { ICommand } from "./command";
- import { RichEmbed, TextChannel, Message, Channel, ReactionCollector, MessageReaction, User, Collector } from "discord.js";
- import { getRepository, IsNull, Not } from "typeorm";
- import { KnownChannel } from "@db/entity/KnownChannel";
- import { AggroNewsItem } from "@db/entity/AggroNewsItem";
- import { v3beta1 } from "@google-cloud/translate";
- const { TranslationServiceClient } = v3beta1;
- const tlClient = new TranslationServiceClient();
- const UPDATE_INTERVAL = 5;
- const MAX_PREVIEW_LENGTH = 300;
- const aggregators : IAggregator[] = [];
- let aggregateChannelID : string = null;
- const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager";
- const FORUMS_STAGING_ID = 54;
- const FORUMS_NEWS_ID = 49;
- const bbCodeParser = new HTML2BBCode();
- const reactionCollectors: Dict<ReactionCollector> = {};
- const verifyMessageIdToPost: Dict<AggroNewsItem> = {};
- // TODO: Run BBCode converter instead
- const turndown = new TurndownService();
- turndown.addRule("image", {
- filter: "img",
- replacement: () => ""
- });
- turndown.addRule("link", {
- filter: (node : HTMLElement, opts: Options) => node.nodeName === "A" &&node.getAttribute("href") != null,
- replacement: (content: string, node: HTMLElement) => node.getAttribute("href")
- });
- function markdownify(htmStr: string) {
- return turndown.turndown(htmStr)/*.replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "")*/;
- }
- async function checkFeeds() {
- console.log(`Aggregating feeds on ${new Date().toISOString()}`);
- let aggregatorJobs = [];
- for(let aggregator of aggregators) {
- aggregatorJobs.push(aggregator.aggregate());
- }
- let aggregatedItems = await Promise.all(aggregatorJobs);
- for(let itemSet of aggregatedItems) {
- for(let item of itemSet) {
- let itemObj = {
- ...item,
- cacheMessageId: null,
- postedMessageId: null
- } as NewsPostItem;
- itemObj.hash = sha1(itemObj.contents);
- await addNewsItem(itemObj);
- }
- }
- }
- function clipText(text: string) {
- if(text.length <= MAX_PREVIEW_LENGTH)
- return text;
- return `${text.substring(0, MAX_PREVIEW_LENGTH)}...`;
- }
- // TODO: Replace with proper forum implementation
- async function addNewsItem(item: NewsPostItem) {
- let repo = getRepository(AggroNewsItem);
- let ch = client.channels.get(aggregateChannelID);
- if(!(ch instanceof TextChannel))
- return;
- let isNew = true;
- let newsItem = await repo.findOne({
- where: { feedName: item.feedId, newsId: item.newsId }
- });
- if(newsItem) {
- // No changes, skip
- if(newsItem.hash == item.hash)
- return;
- else
- await deleteCacheMessage(newsItem.editMessageId);
- isNew = false;
- } else {
- newsItem = repo.create({
- newsId: item.newsId,
- feedName: item.feedId,
- hash: item.hash
- });
- }
- if(item.needsTranslation)
- try {
- let request = {
- parent: tlClient.locationPath(process.env.GOOGLE_APP_ID, "global"),
- contents: [ item.title, item.contents ],
- mimeType: "text/html",
- sourceLanguageCode: "ja",
- targetLanguageCode: "en"
- };
- let [ res ] = await tlClient.translateText(request);
- item.title = res.translations[0].translatedText
- item.contents = res.translations[1].translatedText;
- } catch(err) {
- console.log(`Failed to translate because ${err}`);
- }
- item.contents = bbCodeParser.feed(item.contents).toString();
- if(!newsItem.forumsEditPostId) {
- let createResponse = await forumClient.createThread(FORUMS_STAGING_ID, item.title, item.contents);
- newsItem.forumsEditPostId = createResponse.thread.thread_id;
- } else {
- await forumClient.postReply(newsItem.forumsNewsPostId, item.contents);
- }
-
- let msg = await ch.send(new RichEmbed({
- title: item.title,
- url: item.link,
- color: item.embedColor,
- timestamp: new Date(),
- description: `${(isNew ? "**[NEW]**" : "**[EDIT]**")}\n[**Edit on forums**](${FORUMS_DOMAIN}/index.php?threads/.${newsItem.forumsEditPostId}/)`,
- author: {
- name: item.author
- },
- footer: {
- text: "NoctBot News Aggregator"
- }
- })) as Message;
- newsItem.editMessageId = msg.id;
- await msg.react("✅");
- await msg.react("❌");
- let collector = msg.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
- collector.on("collect", collectReaction)
- reactionCollectors[msg.id] = collector;
- verifyMessageIdToPost[msg.id] = newsItem;
- await repo.save(newsItem);
- }
- function isVerifyReaction(reaction: MessageReaction, user: User) {
- return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot && user.id != client.user.id;
- }
- async function collectReaction(reaction: MessageReaction, collector: Collector<string, MessageReaction>) {
- let repo = getRepository(AggroNewsItem);
- let m = reaction.message;
- collector.stop();
- delete reactionCollectors[m.id];
- let post = verifyMessageIdToPost[m.id];
-
- if (reaction.emoji.name == "✅") {
- let res = await forumClient.getThread(post.forumsEditPostId);
- let forumPost = await forumClient.getPost(res.thread.first_post_id);
- if(!post.forumsNewsPostId) {
- let newThread = await forumClient.createThread(FORUMS_NEWS_ID, res.thread.title, forumPost.message);
- post.forumsNewsPostId = newThread.thread.thread_id;
- } else {
- let curThread = await forumClient.editThread(post.forumsNewsPostId, {
- title: res.thread.title
- });
- await forumClient.editPost(curThread.thread.first_post_id, {
- message: forumPost.message
- });
- }
- }
-
- await forumClient.deleteThread(post.forumsEditPostId);
- await repo.update({ newsId: post.newsId, feedName: post.feedName }, { editMessageId: null, forumsEditPostId: null, forumsNewsPostId: post.forumsNewsPostId });
- await reaction.message.delete();
- delete verifyMessageIdToPost[m.id];
- }
- async function deleteCacheMessage(messageId: string) {
- let ch = client.channels.get(aggregateChannelID);
- if(!(ch instanceof TextChannel))
- return;
- let msg = await tryFetchMessage(ch, messageId);
- if(msg)
- await msg.delete();
- }
- async function tryFetchMessage(channel: Channel, messageId: string) {
- try {
- if (!(channel instanceof TextChannel))
- return null;
- return await channel.fetchMessage(messageId);
- } catch (error) {
- return null;
- }
- }
- function initAggregators() {
- let aggregatorsPath = path.join(path.dirname(module.filename), "aggregators");
- let files = fs.readdirSync(aggregatorsPath);
- for(let file of files) {
- let ext = path.extname(file);
- let name = path.basename(file);
- if(name == "aggregator.js")
- continue;
- if(ext != ".js")
- continue;
- let obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator;
- if(obj)
- aggregators.push(obj);
- if(obj.init)
- obj.init();
- }
- }
- async function initPendingReactors() {
- let verifyChannel = client.channels.get(aggregateChannelID);
- let repo = getRepository(AggroNewsItem);
- let pendingVerifyMessages = await repo.find({
- where: { editMessageId: Not(IsNull()) }
- });
- for (let msg of pendingVerifyMessages) {
- let m = await tryFetchMessage(verifyChannel, msg.editMessageId);
- if (!m) {
- await repo.update({ feedName: msg.feedName, newsId: msg.newsId }, { editMessageId: null });
- continue;
- }
- let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
- collector.on("collect", collectReaction);
- reactionCollectors[m.id] = collector;
- verifyMessageIdToPost[m.id] = msg;
- }
- }
- export default {
- onStart : async () => {
- let repo = getRepository(KnownChannel);
- let ch = await repo.findOne({
- where: { channelType: AGGREGATOR_MANAGER_CHANNEL }
- });
- if(!ch)
- return;
-
- aggregateChannelID = ch.channelId;
- await initPendingReactors();
- initAggregators();
- interval(checkFeeds, UPDATE_INTERVAL * 60 * 1000);
- }
- } as ICommand;
|