import TurndownService from "turndown"; import interval from "interval-promise"; import { client } from "../client"; import sha1 from "sha1"; import * as path from "path"; import * as fs from "fs"; import { HTML2BBCode } from "html2bbcode"; import { IAggregator, NewsPostItem } from "./aggregators/aggregator"; import { TextChannel, Message, MessageEmbed } from "discord.js"; import { getRepository, } from "typeorm"; import { KnownChannel } from "@shared/db/entity/KnownChannel"; import { AggroNewsItem } from "@shared/db/entity/AggroNewsItem"; import { logger } from "src/logging"; import { Plugin } from "src/model/plugin"; import { assertOk, tryDo } from "src/util"; const UPDATE_INTERVAL = process.env.NODE_ENV == "dev" ? 60 : 5; const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager"; @Plugin export class NewsAggregator { aggregators: IAggregator[] = []; aggregateChannelID?: string; bbCodeParser = new HTML2BBCode(); turndown = new TurndownService(); constructor() { this.turndown.addRule("image", { filter: "img", replacement: () => "" }); this.turndown.addRule("link", { filter: (node: HTMLElement) => node.nodeName === "A" && node.getAttribute("href") != null, replacement: (content: string, node: Node) => (node as HTMLElement).getAttribute("htef") ?? "" }); } checkFeeds = async (): Promise => { logger.info(`Aggregating feeds on ${new Date().toISOString()}`); const aggregatorJobs = []; for (const aggregator of this.aggregators) { aggregatorJobs.push(aggregator.aggregate()); } const aggregatedItems = await Promise.all(aggregatorJobs); for (const itemSet of aggregatedItems) { for (const item of itemSet) { const itemObj = { ...item, cacheMessageId: undefined, postedMessageId: undefined } as NewsPostItem; itemObj.hash = sha1(itemObj.contents); await this.addNewsItem(itemObj); } } } async addNewsItem(item: NewsPostItem): Promise { if (!this.aggregateChannelID) return; const repo = getRepository(AggroNewsItem); const ch = client.bot.channels.resolve(this.aggregateChannelID); if (!(ch instanceof TextChannel)) return; let isNew = true; let newsItem = await repo.findOne({ where: { feedName: item.feedId, newsId: item.newsId } }); if (newsItem) { if (process.env.IGNORE_CHANGED_NEWS === "TRUE") { newsItem.hash = item.hash ?? ""; await repo.save(newsItem); return; } // No changes, skip if (newsItem.hash == item.hash) return; else if(newsItem.editMessageId) await this.deleteCacheMessage(newsItem.editMessageId); isNew = false; } else { newsItem = repo.create({ newsId: item.newsId, feedName: item.feedId, hash: item.hash }); } const msg = await assertOk(ch.send(new MessageEmbed({ title: `${(isNew ? "**[NEW]**" : "**[EDIT]**")} ${item.title}`, url: item.link, color: item.embedColor, timestamp: new Date(), author: { name: item.author }, footer: { text: "NoctBot News Aggregator" } }))) as Message; newsItem.editMessageId = msg.id; await repo.save(newsItem); } async deleteCacheMessage(messageId: string): Promise { if(!this.aggregateChannelID) return; const ch = client.bot.channels.resolve(this.aggregateChannelID); if (!(ch instanceof TextChannel)) return; const msg = await tryDo(ch.messages.fetch(messageId)); if (msg.ok) { const deleteResult = await tryDo(msg.result.delete()); if (!deleteResult.ok) logger.error("NewsAggregator: failed to delete message %s: %s", messageId, deleteResult.error); } else logger.error("NewsAggregator: failed to fetch messate %s: %s", this.aggregateChannelID, msg.error); } initAggregators(): void { const aggregatorsPath = path.join(path.dirname(module.filename), "aggregators"); const files = fs.readdirSync(aggregatorsPath); for (const file of files) { const ext = path.extname(file); const name = path.basename(file); if (name == "aggregator.js") continue; if (ext != ".js") continue; // eslint-disable-next-line @typescript-eslint/no-var-requires const obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator; if (obj) this.aggregators.push(obj); if (obj.init) obj.init(); } } async start(): Promise { const repo = getRepository(KnownChannel); const ch = await repo.findOne({ where: { channelType: AGGREGATOR_MANAGER_CHANNEL } }); if (!ch) return; this.aggregateChannelID = ch.channelId; this.initAggregators(); interval(this.checkFeeds, UPDATE_INTERVAL * 60 * 1000); } }