123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176 |
- import TurndownService from "turndown";
- import interval from "interval-promise";
- import { client } from "../client";
- import sha1 from "sha1";
- import * as path from "path";
- import * as fs from "fs";
- import { HTML2BBCode } from "html2bbcode";
- import { IAggregator, NewsPostItem } from "./aggregators/aggregator";
- import { TextChannel, Message, MessageEmbed } from "discord.js";
- import { getRepository, } from "typeorm";
- import { KnownChannel } from "@shared/db/entity/KnownChannel";
- import { AggroNewsItem } from "@shared/db/entity/AggroNewsItem";
- import { logger } from "src/logging";
- import { Plugin } from "src/model/plugin";
- import { assertOk, tryDo } from "src/util";
- const UPDATE_INTERVAL = process.env.NODE_ENV == "dev" ? 60 : 5;
- const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager";
- @Plugin
- export class NewsAggregator {
- aggregators: IAggregator[] = [];
- aggregateChannelID?: string;
- bbCodeParser = new HTML2BBCode();
- turndown = new TurndownService();
- constructor() {
- this.turndown.addRule("image", {
- filter: "img",
- replacement: () => ""
- });
- this.turndown.addRule("link", {
- filter: (node: HTMLElement) => node.nodeName === "A" && node.getAttribute("href") != null,
- replacement: (content: string, node: Node) => (node as HTMLElement).getAttribute("htef") ?? ""
- });
- }
- checkFeeds = async (): Promise<void> => {
- logger.info(`Aggregating feeds on ${new Date().toISOString()}`);
- const aggregatorJobs = [];
- for (const aggregator of this.aggregators) {
- aggregatorJobs.push(aggregator.aggregate());
- }
- const aggregatedItems = await Promise.all(aggregatorJobs);
- for (const itemSet of aggregatedItems) {
- for (const item of itemSet) {
- const itemObj = {
- ...item,
- cacheMessageId: undefined,
- postedMessageId: undefined
- } as NewsPostItem;
- itemObj.hash = sha1(itemObj.contents);
- await this.addNewsItem(itemObj);
- }
- }
- }
- async addNewsItem(item: NewsPostItem): Promise<void> {
- if (!this.aggregateChannelID)
- return;
- const repo = getRepository(AggroNewsItem);
- const ch = client.bot.channels.resolve(this.aggregateChannelID);
- if (!(ch instanceof TextChannel))
- return;
- let isNew = true;
- let newsItem = await repo.findOne({
- where: { feedName: item.feedId, newsId: item.newsId }
- });
- if (newsItem) {
- if (process.env.IGNORE_CHANGED_NEWS === "TRUE") {
- newsItem.hash = item.hash ?? "";
- await repo.save(newsItem);
- return;
- }
- // No changes, skip
- if (newsItem.hash == item.hash)
- return;
- else if(newsItem.editMessageId)
- await this.deleteCacheMessage(newsItem.editMessageId);
- isNew = false;
- } else {
- newsItem = repo.create({
- newsId: item.newsId,
- feedName: item.feedId,
- hash: item.hash
- });
- }
- const msg = await assertOk(ch.send(new MessageEmbed({
- title: `${(isNew ? "**[NEW]**" : "**[EDIT]**")} ${item.title}`,
- url: item.link,
- color: item.embedColor,
- timestamp: new Date(),
- author: {
- name: item.author
- },
- footer: {
- text: "NoctBot News Aggregator"
- }
- }))) as Message;
- newsItem.editMessageId = msg.id;
- await repo.save(newsItem);
- }
- async deleteCacheMessage(messageId: string): Promise<void> {
- if(!this.aggregateChannelID)
- return;
- const ch = client.bot.channels.resolve(this.aggregateChannelID);
- if (!(ch instanceof TextChannel))
- return;
- const msg = await tryDo(ch.messages.fetch(messageId));
- if (msg.ok) {
- const deleteResult = await tryDo(msg.result.delete());
- if (!deleteResult.ok)
- logger.error("NewsAggregator: failed to delete message %s: %s", messageId, deleteResult.error);
- }
- else
- logger.error("NewsAggregator: failed to fetch messate %s: %s", this.aggregateChannelID, msg.error);
- }
- initAggregators(): void {
- const aggregatorsPath = path.join(path.dirname(module.filename), "aggregators");
- const files = fs.readdirSync(aggregatorsPath);
- for (const file of files) {
- const ext = path.extname(file);
- const name = path.basename(file);
- if (name == "aggregator.js")
- continue;
- if (ext != ".js")
- continue;
- // eslint-disable-next-line @typescript-eslint/no-var-requires
- const obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator;
- if (obj)
- this.aggregators.push(obj);
- if (obj.init)
- obj.init();
- }
- }
- async start(): Promise<void> {
- const repo = getRepository(KnownChannel);
- const ch = await repo.findOne({
- where: { channelType: AGGREGATOR_MANAGER_CHANNEL }
- });
- if (!ch)
- return;
- this.aggregateChannelID = ch.channelId;
- this.initAggregators();
- interval(this.checkFeeds, UPDATE_INTERVAL * 60 * 1000);
- }
- }
|