import TurndownService, { Options } from "turndown"; import interval from "interval-promise"; import { client } from "../client"; import sha1 from "sha1"; import * as path from "path"; import * as fs from "fs"; import { IAggregator, NewsPostItem, INewsPostData } from "./aggregators/aggregator"; import { ICommand } from "./command"; import { RichEmbed, TextChannel, Message, Channel } from "discord.js"; import { getRepository } from "typeorm"; import { KnownChannel } from "../entity/KnownChannel"; import { AggroNewsItem } from "../entity/AggroNewsItem"; const UPDATE_INTERVAL = 5; const MAX_PREVIEW_LENGTH = 300; const aggregators : IAggregator[] = []; let aggregateChannelID : string = null; const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager"; // TODO: Run BBCode converter instead const turndown = new TurndownService(); turndown.addRule("image", { filter: "img", replacement: () => "" }); turndown.addRule("link", { filter: (node : HTMLElement, opts: Options) => node.nodeName === "A" &&node.getAttribute("href") != null, replacement: (content: string, node: HTMLElement) => node.getAttribute("href") }); function markdownify(htmStr: string) { return turndown.turndown(htmStr)/*.replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "")*/; } async function checkFeeds() { console.log(`Aggregating feeds on ${new Date().toISOString()}`); let aggregatorJobs = []; for(let aggregator of aggregators) { aggregatorJobs.push(aggregator.aggregate()); } let aggregatedItems = await Promise.all(aggregatorJobs); for(let itemSet of aggregatedItems) { for(let item of itemSet) { let itemObj = { ...item, cacheMessageId: null, postedMessageId: null } as NewsPostItem; itemObj.contents = markdownify(item.contents); itemObj.hash = sha1(itemObj.contents); await addNewsItem(itemObj); } } } function clipText(text: string) { if(text.length <= MAX_PREVIEW_LENGTH) return text; return `${text.substring(0, MAX_PREVIEW_LENGTH)}...`; } // TODO: Replace with proper forum implementation async function addNewsItem(item: NewsPostItem) { let repo = getRepository(AggroNewsItem); let newsItem = await repo.findOne({ where: { feedName: item.feedId, newsId: item.newsId } }); if(newsItem) { // No changes, skip if(newsItem.hash == item.hash) return; else await deleteCacheMessage(newsItem.editMessageId); } else { newsItem = repo.create({ newsId: item.newsId, feedName: item.feedId, hash: item.hash }); } let ch = client.channels.get(aggregateChannelID); if(!(ch instanceof TextChannel)) return; let msg = await ch.send(new RichEmbed({ title: item.title, url: item.link, color: item.embedColor, timestamp: new Date(), description: clipText(item.contents), author: { name: item.author }, footer: { text: "NoctBot News Aggregator" } })) as Message; newsItem.editMessageId = msg.id; await repo.save(newsItem); } async function deleteCacheMessage(messageId: string) { let ch = client.channels.get(aggregateChannelID); if(!(ch instanceof TextChannel)) return; let msg = await tryFetchMessage(ch, messageId); if(msg) await msg.delete(); } async function tryFetchMessage(channel : TextChannel, messageId: string) { try { return await channel.fetchMessage(messageId); }catch(error){ return null; } } function initAggregators() { let aggregatorsPath = path.join(path.dirname(module.filename), "aggregators"); let files = fs.readdirSync(aggregatorsPath); for(let file of files) { let ext = path.extname(file); let name = path.basename(file); if(name == "aggregator.js") continue; if(ext != ".js") continue; let obj = require(path.resolve(aggregatorsPath, file)).default as IAggregator; if(obj) aggregators.push(obj); if(obj.init) obj.init(); } } export default { onStart : async () => { let repo = getRepository(KnownChannel); let ch = await repo.findOne({ where: { channelType: AGGREGATOR_MANAGER_CHANNEL } }); aggregateChannelID = ch.channelId; initAggregators(); interval(checkFeeds, UPDATE_INTERVAL * 60 * 1000); } } as ICommand;