|
@@ -1,17 +1,21 @@
|
|
-const TurndownService = require("turndown");
|
|
|
|
-const RSSParser = require("rss-parser");
|
|
|
|
-const db = require("../db.js");
|
|
|
|
-const interval = require("interval-promise");
|
|
|
|
-const client = require("../client.js");
|
|
|
|
-const sha1 = require("sha1");
|
|
|
|
-const html = require("node-html-parser");
|
|
|
|
-const axios = require("axios");
|
|
|
|
|
|
+import TurndownService, { Options } from "turndown";
|
|
|
|
+import RSSParser from "rss-parser";
|
|
|
|
+import { db } from "../db";
|
|
|
|
+import interval from "interval-promise";
|
|
|
|
+import client from "../client";
|
|
|
|
+import sha1 from "sha1";
|
|
|
|
+import html from "node-html-parser";
|
|
|
|
+import request from "request-promise-native";
|
|
|
|
+import { ICommand } from "./command";
|
|
|
|
+import { Response } from "request";
|
|
|
|
+import { TextChannel, Message, ReactionCollector, MessageReaction, Collector, User, Channel } from "discord.js";
|
|
|
|
+import { Dictionary, ObjectChain } from "lodash";
|
|
|
|
|
|
const PREVIEW_CHAR_LIMIT = 300;
|
|
const PREVIEW_CHAR_LIMIT = 300;
|
|
const verifyChannelID = db.get("newsPostVerifyChannel").value();
|
|
const verifyChannelID = db.get("newsPostVerifyChannel").value();
|
|
|
|
|
|
-const reactionCollectors = {};
|
|
|
|
-const verifyMessageIdToPostId = {};
|
|
|
|
|
|
+const reactionCollectors : Dictionary<ReactionCollector> = {};
|
|
|
|
+const verifyMessageIdToPostId : Dictionary<string> = {};
|
|
|
|
|
|
const turndown = new TurndownService();
|
|
const turndown = new TurndownService();
|
|
turndown.addRule("image", {
|
|
turndown.addRule("image", {
|
|
@@ -19,47 +23,62 @@ turndown.addRule("image", {
|
|
replacement: () => ""
|
|
replacement: () => ""
|
|
});
|
|
});
|
|
turndown.addRule("link", {
|
|
turndown.addRule("link", {
|
|
- filter: node => node.nodeName === "A" &&node.getAttribute("href"),
|
|
|
|
- replacement: (content, node) => node.getAttribute("href")
|
|
|
|
|
|
+ filter: (node : HTMLElement, opts: Options) => node.nodeName === "A" && node.getAttribute("href") != null,
|
|
|
|
+ replacement: (content: string, node: HTMLElement) => node.getAttribute("href")
|
|
});
|
|
});
|
|
|
|
|
|
const parser = new RSSParser();
|
|
const parser = new RSSParser();
|
|
const RSS_UPDATE_INTERVAL_MIN = 5;
|
|
const RSS_UPDATE_INTERVAL_MIN = 5;
|
|
|
|
|
|
-function getThreadId(url) {
|
|
|
|
|
|
+function getThreadId(url: string) {
|
|
let result = url.substring(url.lastIndexOf(".") + 1);
|
|
let result = url.substring(url.lastIndexOf(".") + 1);
|
|
if(result.endsWith("/"))
|
|
if(result.endsWith("/"))
|
|
result = result.substring(0, result.length - 1);
|
|
result = result.substring(0, result.length - 1);
|
|
return result;
|
|
return result;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+interface PostItem {
|
|
|
|
+ id: string,
|
|
|
|
+ title: string,
|
|
|
|
+ link: string,
|
|
|
|
+ creator: string,
|
|
|
|
+ contents: string,
|
|
|
|
+ hash: string,
|
|
|
|
+ messageId?: string,
|
|
|
|
+ verifyMessageId?: string,
|
|
|
|
+ type?: string
|
|
|
|
+}
|
|
|
|
+
|
|
async function checkFeeds() {
|
|
async function checkFeeds() {
|
|
console.log(`Checking feeds on ${new Date().toISOString()}`);
|
|
console.log(`Checking feeds on ${new Date().toISOString()}`);
|
|
let feeds = db.get("rssFeeds").value();
|
|
let feeds = db.get("rssFeeds").value();
|
|
- let oldNews = db.get("postedNewsGuids");
|
|
|
|
|
|
+ let oldNews = db.get("postedNewsGuids") as ObjectChain<any>;
|
|
for(let feedEntry of feeds) {
|
|
for(let feedEntry of feeds) {
|
|
let feed = await parser.parseURL(feedEntry.url);
|
|
let feed = await parser.parseURL(feedEntry.url);
|
|
if(feed.items.length == 0)
|
|
if(feed.items.length == 0)
|
|
continue;
|
|
continue;
|
|
- let printableItems = feed.items.sort((a, b) => a.isoDate.localeCompare(b.isoDate));
|
|
|
|
|
|
+ let printableItems = feed.items.sort((a : any, b: any) => a.isoDate.localeCompare(b.isoDate));
|
|
if(printableItems.length > 0) {
|
|
if(printableItems.length > 0) {
|
|
for(let item of printableItems) {
|
|
for(let item of printableItems) {
|
|
let itemID = getThreadId(item.guid);
|
|
let itemID = getThreadId(item.guid);
|
|
let contents = null;
|
|
let contents = null;
|
|
|
|
|
|
try {
|
|
try {
|
|
- let res = await axios.get(item.link);
|
|
|
|
- if(res.status != 200) {
|
|
|
|
- console.log(`Post ${itemID} could not be loaded because request returned status ${res.status}`);
|
|
|
|
|
|
+ let res = await request(item.link, {resolveWithFullResponse: true}) as Response;
|
|
|
|
+ if(res.statusCode != 200) {
|
|
|
|
+ console.log(`Post ${itemID} could not be loaded because request returned status ${res.statusCode}`);
|
|
continue;
|
|
continue;
|
|
}
|
|
}
|
|
|
|
|
|
- let rootNode = html.parse(res.data, {
|
|
|
|
|
|
+ let rootNode = html.parse(res.body, {
|
|
pre: true,
|
|
pre: true,
|
|
script: false,
|
|
script: false,
|
|
style: false
|
|
style: false
|
|
});
|
|
});
|
|
|
|
|
|
|
|
+ if(!(rootNode instanceof HTMLElement))
|
|
|
|
+ continue;
|
|
|
|
+
|
|
let opDiv = rootNode.querySelector("div.bbWrapper");
|
|
let opDiv = rootNode.querySelector("div.bbWrapper");
|
|
|
|
|
|
if (!opDiv) {
|
|
if (!opDiv) {
|
|
@@ -83,7 +102,7 @@ async function checkFeeds() {
|
|
messageId: null,
|
|
messageId: null,
|
|
verifyMessageId: null,
|
|
verifyMessageId: null,
|
|
type: null
|
|
type: null
|
|
- };
|
|
|
|
|
|
+ } as PostItem;
|
|
|
|
|
|
if(oldNews.has(itemObj.id).value()){
|
|
if(oldNews.has(itemObj.id).value()){
|
|
let data = oldNews.get(itemObj.id).value();
|
|
let data = oldNews.get(itemObj.id).value();
|
|
@@ -105,14 +124,16 @@ async function checkFeeds() {
|
|
}
|
|
}
|
|
let lastUpdateDate = printableItems[printableItems.length - 1].isoDate;
|
|
let lastUpdateDate = printableItems[printableItems.length - 1].isoDate;
|
|
console.log(`Setting last update marker on ${feedEntry.url} to ${lastUpdateDate}`);
|
|
console.log(`Setting last update marker on ${feedEntry.url} to ${lastUpdateDate}`);
|
|
- db.get("rssFeeds").find({ url: feedEntry.url}).assign({lastUpdate: lastUpdateDate}).write();
|
|
|
|
|
|
+ let rssFeeds = db.get("rssFeeds") as ObjectChain<any>;
|
|
|
|
+ let entry = rssFeeds.find({ url: feedEntry.url}) as ObjectChain<any>;
|
|
|
|
+ entry.assign({lastUpdate: lastUpdateDate}).write();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
function initPendingReactors() {
|
|
function initPendingReactors() {
|
|
let verifyChannel = client.channels.get(verifyChannelID);
|
|
let verifyChannel = client.channels.get(verifyChannelID);
|
|
- db.get("newsCache").forOwn(async i => {
|
|
|
|
|
|
+ db.get("newsCache").forOwn(async (i : any) => {
|
|
let m = await tryFetchMessage(verifyChannel, i.verifyMessageId);
|
|
let m = await tryFetchMessage(verifyChannel, i.verifyMessageId);
|
|
let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
|
|
let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
|
|
collector.on("collect", collectReaction)
|
|
collector.on("collect", collectReaction)
|
|
@@ -121,9 +142,9 @@ function initPendingReactors() {
|
|
}).value();
|
|
}).value();
|
|
}
|
|
}
|
|
|
|
|
|
-async function addVerifyMessage(item) {
|
|
|
|
- let verifyChannel = client.channels.get(verifyChannelID);
|
|
|
|
- let cache = db.get("newsCache");
|
|
|
|
|
|
+async function addVerifyMessage(item : PostItem) {
|
|
|
|
+ let verifyChannel = client.channels.get(verifyChannelID) as TextChannel;
|
|
|
|
+ let cache = db.get("newsCache") as ObjectChain<any>;
|
|
let oldNews = db.get("postedNewsGuids");
|
|
let oldNews = db.get("postedNewsGuids");
|
|
let postedNews = db.get("postedNewsGuids");
|
|
let postedNews = db.get("postedNewsGuids");
|
|
item.type = "🆕 ADD";
|
|
item.type = "🆕 ADD";
|
|
@@ -138,31 +159,32 @@ async function addVerifyMessage(item) {
|
|
await oldMessage.delete();
|
|
await oldMessage.delete();
|
|
}
|
|
}
|
|
|
|
|
|
- let newMessage = await verifyChannel.send(toVerifyString(item));
|
|
|
|
|
|
+ let newMessage = await verifyChannel.send(toVerifyString(item)) as Message;
|
|
|
|
|
|
await newMessage.react("✅");
|
|
await newMessage.react("✅");
|
|
await newMessage.react("❌");
|
|
await newMessage.react("❌");
|
|
|
|
|
|
- var collector = newMessage.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
|
|
|
|
|
|
+ let collector = newMessage.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
|
|
collector.on("collect", collectReaction)
|
|
collector.on("collect", collectReaction)
|
|
reactionCollectors[newMessage.id] = collector;
|
|
reactionCollectors[newMessage.id] = collector;
|
|
verifyMessageIdToPostId[newMessage.id] = item.id;
|
|
verifyMessageIdToPostId[newMessage.id] = item.id;
|
|
item.verifyMessageId = newMessage.id;
|
|
item.verifyMessageId = newMessage.id;
|
|
- cache.set(item.id, item).write();
|
|
|
|
|
|
+ cache.set(item.id, item);
|
|
|
|
|
|
oldNews.set(item.id, {
|
|
oldNews.set(item.id, {
|
|
hash: item.hash,
|
|
hash: item.hash,
|
|
messageId: null
|
|
messageId: null
|
|
- }).write();
|
|
|
|
|
|
+ });
|
|
|
|
+ db.write();
|
|
}
|
|
}
|
|
|
|
|
|
-function collectReaction(reaction, collector) {
|
|
|
|
- let cache = db.get("newsCache");
|
|
|
|
|
|
+function collectReaction(reaction : MessageReaction, collector: Collector<string, MessageReaction>) {
|
|
|
|
+ let cache = db.get("newsCache") as ObjectChain<any>;
|
|
let m = reaction.message;
|
|
let m = reaction.message;
|
|
collector.stop();
|
|
collector.stop();
|
|
delete reactionCollectors[m.id];
|
|
delete reactionCollectors[m.id];
|
|
let postId = verifyMessageIdToPostId[m.id];
|
|
let postId = verifyMessageIdToPostId[m.id];
|
|
- let item = cache.get(postId).value();
|
|
|
|
|
|
+ let item = cache.get(postId).value() as PostItem;
|
|
cache.unset(postId).write();
|
|
cache.unset(postId).write();
|
|
m.delete();
|
|
m.delete();
|
|
|
|
|
|
@@ -170,7 +192,7 @@ function collectReaction(reaction, collector) {
|
|
sendNews(item);
|
|
sendNews(item);
|
|
}
|
|
}
|
|
|
|
|
|
-async function sendNews(item) {
|
|
|
|
|
|
+async function sendNews(item : PostItem) {
|
|
let outChannel = db.get("feedOutputChannel").value();
|
|
let outChannel = db.get("feedOutputChannel").value();
|
|
let oldNews = db.get("postedNewsGuids");
|
|
let oldNews = db.get("postedNewsGuids");
|
|
|
|
|
|
@@ -178,15 +200,18 @@ async function sendNews(item) {
|
|
oldNews.set(item.id, {
|
|
oldNews.set(item.id, {
|
|
hash: item.hash,
|
|
hash: item.hash,
|
|
messageId: sentMessage.id
|
|
messageId: sentMessage.id
|
|
- }).write();
|
|
|
|
|
|
+ });
|
|
|
|
+ db.write();
|
|
}
|
|
}
|
|
|
|
|
|
-function isVerifyReaction(reaction, user) {
|
|
|
|
|
|
+function isVerifyReaction(reaction : MessageReaction, user: User) {
|
|
return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot;
|
|
return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot;
|
|
}
|
|
}
|
|
|
|
|
|
-async function tryFetchMessage(channel, messageId) {
|
|
|
|
|
|
+async function tryFetchMessage(channel: Channel, messageId: string) {
|
|
try {
|
|
try {
|
|
|
|
+ if(!(channel instanceof TextChannel))
|
|
|
|
+ return null;
|
|
return await channel.fetchMessage(messageId);
|
|
return await channel.fetchMessage(messageId);
|
|
}catch(error){
|
|
}catch(error){
|
|
return null;
|
|
return null;
|
|
@@ -197,26 +222,29 @@ function shouldVerify() {
|
|
return verifyChannelID != "";
|
|
return verifyChannelID != "";
|
|
}
|
|
}
|
|
|
|
|
|
-async function postNewsItem(channel, item) {
|
|
|
|
|
|
+async function postNewsItem(channel: string, item: PostItem) : Promise<Message | null> {
|
|
let newsMessage = toNewsString(item);
|
|
let newsMessage = toNewsString(item);
|
|
let ch = client.channels.get(channel);
|
|
let ch = client.channels.get(channel);
|
|
|
|
|
|
|
|
+ if(!(ch instanceof TextChannel))
|
|
|
|
+ return null;
|
|
|
|
+
|
|
if(item.messageId) {
|
|
if(item.messageId) {
|
|
let message = await tryFetchMessage(ch, item.messageId);
|
|
let message = await tryFetchMessage(ch, item.messageId);
|
|
if(message)
|
|
if(message)
|
|
return await message.edit(newsMessage);
|
|
return await message.edit(newsMessage);
|
|
else
|
|
else
|
|
- return await ch.send(newsMessage);
|
|
|
|
|
|
+ return await ch.send(newsMessage) as Message;
|
|
}
|
|
}
|
|
else
|
|
else
|
|
- return await ch.send(newsMessage);
|
|
|
|
|
|
+ return await ch.send(newsMessage) as Message;
|
|
}
|
|
}
|
|
|
|
|
|
-function markdownify(htmStr, link) {
|
|
|
|
|
|
+function markdownify(htmStr: string, link: string) {
|
|
return turndown.turndown(htmStr).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "");
|
|
return turndown.turndown(htmStr).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "");
|
|
}
|
|
}
|
|
|
|
|
|
-function toNewsString(item) {
|
|
|
|
|
|
+function toNewsString(item: PostItem) {
|
|
return `**${item.title}**
|
|
return `**${item.title}**
|
|
Posted by ${item.creator}
|
|
Posted by ${item.creator}
|
|
${item.link}
|
|
${item.link}
|
|
@@ -224,7 +252,7 @@ ${item.link}
|
|
${item.contents}`;
|
|
${item.contents}`;
|
|
}
|
|
}
|
|
|
|
|
|
-function toVerifyString(item) {
|
|
|
|
|
|
+function toVerifyString(item: PostItem) {
|
|
return `[${item.type}]
|
|
return `[${item.type}]
|
|
Post ID: **${item.id}**
|
|
Post ID: **${item.id}**
|
|
|
|
|
|
@@ -233,45 +261,43 @@ ${toNewsString(item)}
|
|
React with ✅ (approve) or ❌ (deny).`;
|
|
React with ✅ (approve) or ❌ (deny).`;
|
|
}
|
|
}
|
|
|
|
|
|
-const onStart = () => {
|
|
|
|
- initPendingReactors();
|
|
|
|
- interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);
|
|
|
|
-};
|
|
|
|
-
|
|
|
|
-const commands = [
|
|
|
|
- {
|
|
|
|
- pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i,
|
|
|
|
- action: async (msg, s, match) => {
|
|
|
|
- if(msg.channel.id != verifyChannelID)
|
|
|
|
- return;
|
|
|
|
-
|
|
|
|
- let id = match[1];
|
|
|
|
- let newContents = match[2].trim();
|
|
|
|
-
|
|
|
|
- if(!db.get("newsCache").has(id).value()) {
|
|
|
|
- msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`);
|
|
|
|
- return;
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- let item = db.get("newsCache").get(id).value();
|
|
|
|
-
|
|
|
|
- let editMsg = await tryFetchMessage(client.channels.get(verifyChannelID), item.verifyMessageId);
|
|
|
|
-
|
|
|
|
- if(!editMsg){
|
|
|
|
- msg.channel.send(`${msg.author.toString()} No verify messafe found for ${id}! This is a bug: report to horse.`);
|
|
|
|
- return;
|
|
|
|
|
|
+export default {
|
|
|
|
+ commands: [
|
|
|
|
+ {
|
|
|
|
+ pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i,
|
|
|
|
+ action: async (msg, s, match) => {
|
|
|
|
+ if(msg.channel.id != verifyChannelID)
|
|
|
|
+ return;
|
|
|
|
+
|
|
|
|
+ let id = match[1];
|
|
|
|
+ let newContents = match[2].trim();
|
|
|
|
+
|
|
|
|
+ if(!db.get("newsCache").has(id).value()) {
|
|
|
|
+ msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`);
|
|
|
|
+ return;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ let newsCache = db.get("newsCache") as ObjectChain<any>;
|
|
|
|
+ let item = newsCache.get(id).value() as PostItem;
|
|
|
|
+
|
|
|
|
+ let editMsg = await tryFetchMessage(client.channels.get(verifyChannelID), item.verifyMessageId);
|
|
|
|
+
|
|
|
|
+ if(!editMsg){
|
|
|
|
+ msg.channel.send(`${msg.author.toString()} No verify messafe found for ${id}! This is a bug: report to horse.`);
|
|
|
|
+ return;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ item.contents = newContents;
|
|
|
|
+
|
|
|
|
+ db.get("newsCache").set(id, item);
|
|
|
|
+ db.write();
|
|
|
|
+ await editMsg.edit(toVerifyString(item));
|
|
|
|
+ await msg.delete();
|
|
}
|
|
}
|
|
-
|
|
|
|
- item.contents = newContents;
|
|
|
|
-
|
|
|
|
- db.get("newsCache").set(id, item).write();
|
|
|
|
- await editMsg.edit(toVerifyString(item));
|
|
|
|
- await msg.delete();
|
|
|
|
}
|
|
}
|
|
|
|
+ ],
|
|
|
|
+ onStart: () => {
|
|
|
|
+ initPendingReactors();
|
|
|
|
+ interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);
|
|
}
|
|
}
|
|
-];
|
|
|
|
-
|
|
|
|
-module.exports = {
|
|
|
|
- onStart: onStart,
|
|
|
|
- commands: commands
|
|
|
|
-};
|
|
|
|
|
|
+} as ICommand;
|