|
@@ -1,25 +1,24 @@
|
|
|
import TurndownService, { Options } from "turndown";
|
|
|
import RSSParser from "rss-parser";
|
|
|
import interval from "interval-promise";
|
|
|
-import client from "../client";
|
|
|
+import { client, forumClient } from "../client";
|
|
|
import sha1 from "sha1";
|
|
|
-import * as html from "node-html-parser";
|
|
|
-import request from "request-promise-native";
|
|
|
import { ICommand } from "./command";
|
|
|
-import { Response } from "request";
|
|
|
import { TextChannel, Message, ReactionCollector, MessageReaction, Collector, User, Channel } from "discord.js";
|
|
|
import { Dict } from "../util";
|
|
|
import { getRepository, Not, IsNull } from "typeorm";
|
|
|
import { PostedForumNewsItem } from "../entity/PostedForumsNewsItem";
|
|
|
import { KnownChannel } from "../entity/KnownChannel";
|
|
|
import { PostVerifyMessage } from "../entity/PostVerifyMessage";
|
|
|
+import bbobHTML from '@bbob/html'
|
|
|
+import presetHTML5 from '@bbob/preset-html5'
|
|
|
|
|
|
const PREVIEW_CHAR_LIMIT = 300;
|
|
|
const NEWS_POST_VERIFY_CHANNEL = "newsPostVerify";
|
|
|
|
|
|
-let verifyChannelId : string = null;
|
|
|
-const reactionCollectors : Dict<ReactionCollector> = {};
|
|
|
-const verifyMessageIdToPost : Dict<PostedForumNewsItem> = {};
|
|
|
+let verifyChannelId: string = null;
|
|
|
+const reactionCollectors: Dict<ReactionCollector> = {};
|
|
|
+const verifyMessageIdToPost: Dict<PostedForumNewsItem> = {};
|
|
|
const NEWS_FEED_CHANNEL = "newsFeed";
|
|
|
|
|
|
const turndown = new TurndownService();
|
|
@@ -28,7 +27,7 @@ turndown.addRule("image", {
|
|
|
replacement: () => ""
|
|
|
});
|
|
|
turndown.addRule("link", {
|
|
|
- filter: (node : HTMLElement, opts: Options) => node.nodeName === "A" && node.getAttribute("href") != null,
|
|
|
+ filter: (node: HTMLElement, opts: Options) => node.nodeName === "A" && node.getAttribute("href") != null,
|
|
|
replacement: (content: string, node: HTMLElement) => node.getAttribute("href")
|
|
|
});
|
|
|
|
|
@@ -37,11 +36,13 @@ const RSS_UPDATE_INTERVAL_MIN = 5;
|
|
|
|
|
|
function getThreadId(url: string) {
|
|
|
let result = url.substring(url.lastIndexOf(".") + 1);
|
|
|
- if(result.endsWith("/"))
|
|
|
+ if (result.endsWith("/"))
|
|
|
result = result.substring(0, result.length - 1);
|
|
|
return result;
|
|
|
}
|
|
|
|
|
|
+const NEWS_FORUM_ID = 49;
|
|
|
+
|
|
|
const FEEDS = [
|
|
|
{
|
|
|
url: "http://custommaid3d2.com/index.php?forums/news.49/index.rss",
|
|
@@ -49,89 +50,112 @@ const FEEDS = [
|
|
|
}
|
|
|
];
|
|
|
|
|
|
+function bbCodeToMarkdown(bbCode: string) {
|
|
|
+ return turndown.turndown(bbobHTML(bbCode, presetHTML5())).replace(/( {2}\n|\n\n){2,}/gm, "\n");
|
|
|
+}
|
|
|
+
|
|
|
async function checkFeeds() {
|
|
|
console.log(`Checking feeds on ${new Date().toISOString()}`);
|
|
|
let forumsNewsRepo = getRepository(PostedForumNewsItem);
|
|
|
let postVerifyMessageRepo = getRepository(PostVerifyMessage);
|
|
|
|
|
|
- for(let feedEntry of FEEDS) {
|
|
|
- let feed = await parser.parseURL(feedEntry.url);
|
|
|
- if(feed.items.length == 0)
|
|
|
- continue;
|
|
|
- let printableItems = feed.items.sort((a : any, b: any) => a.isoDate.localeCompare(b.isoDate));
|
|
|
- if(printableItems.length > 0) {
|
|
|
- for(let item of printableItems) {
|
|
|
- let itemID = getThreadId(item.guid);
|
|
|
- let contents = null;
|
|
|
-
|
|
|
- try {
|
|
|
- let res = await request(item.link, {resolveWithFullResponse: true}) as Response;
|
|
|
- if(res.statusCode != 200) {
|
|
|
- console.log(`Post ${itemID} could not be loaded because request returned status ${res.statusCode}`);
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- let rootNode = html.parse(res.body, {
|
|
|
- pre: true,
|
|
|
- script: false,
|
|
|
- style: false
|
|
|
- });
|
|
|
-
|
|
|
- if(!(rootNode instanceof html.HTMLElement))
|
|
|
- continue;
|
|
|
-
|
|
|
- let opDiv = rootNode.querySelector("div.bbWrapper");
|
|
|
-
|
|
|
- if (!opDiv) {
|
|
|
- console.log(`No posts found for ${itemID}!`);
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- contents = markdownify(opDiv.outerHTML, item.link);
|
|
|
- } catch(err){
|
|
|
- console.log(`Failed to get html for item ${itemID} because ${err}`);
|
|
|
- continue;
|
|
|
- }
|
|
|
+ let forumThreads = await forumClient.getForumThreads(NEWS_FORUM_ID);
|
|
|
+
|
|
|
+ for (let thread of forumThreads.threads) {
|
|
|
+ let firstPost = await forumClient.getPost(thread.first_post_id);
|
|
|
+
|
|
|
+ let contents = bbCodeToMarkdown(firstPost.message);
|
|
|
+ let itemObj = forumsNewsRepo.create({
|
|
|
+ id: thread.thread_id.toString(),
|
|
|
+ hash: sha1(firstPost.message),
|
|
|
+ verifyMessage: postVerifyMessageRepo.create({
|
|
|
+ author: thread.username,
|
|
|
+ link: `https://custommaid3d2.com/index.php?threads/${thread.thread_id}/`,
|
|
|
+ title: thread.title,
|
|
|
+ text: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
|
|
|
+ isNew: true
|
|
|
+ })
|
|
|
+ });
|
|
|
+
|
|
|
+ let postItem = await forumsNewsRepo.findOne({
|
|
|
+ where: { id: itemObj.id },
|
|
|
+ relations: ["verifyMessage"]
|
|
|
+ });
|
|
|
+
|
|
|
+ if (postItem) {
|
|
|
|
|
|
- let itemObj = forumsNewsRepo.create({
|
|
|
- id: itemID,
|
|
|
- hash: sha1(contents),
|
|
|
- verifyMessage: postVerifyMessageRepo.create({
|
|
|
- author: item.creator,
|
|
|
- link: item.link,
|
|
|
- title: item.title,
|
|
|
- text: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
|
|
|
- isNew: true
|
|
|
- })
|
|
|
- });
|
|
|
-
|
|
|
- let postItem = await forumsNewsRepo.findOne({
|
|
|
- where: { id: itemObj.id },
|
|
|
- relations: [ "verifyMessage" ]
|
|
|
+ if(process.env.INGORE_CHANGED_NEWS === "TRUE") {
|
|
|
+ await forumsNewsRepo.update({
|
|
|
+ id: postItem.id
|
|
|
+ }, {
|
|
|
+ hash: itemObj.hash
|
|
|
});
|
|
|
+ continue;
|
|
|
+ }
|
|
|
|
|
|
- if(postItem){
|
|
|
- // Add message ID to mark for edit
|
|
|
- if(postItem.hash != itemObj.hash){
|
|
|
- let newHash = itemObj.hash;
|
|
|
- if(!postItem.verifyMessage)
|
|
|
- postItem.verifyMessage = itemObj.verifyMessage;
|
|
|
-
|
|
|
- itemObj = postItem;
|
|
|
- itemObj.verifyMessage.isNew = false;
|
|
|
- itemObj.hash = newHash;
|
|
|
- }
|
|
|
- else
|
|
|
- continue;
|
|
|
- }
|
|
|
+ // Add message ID to mark for edit
|
|
|
+ if (postItem.hash != itemObj.hash) {
|
|
|
+ let newHash = itemObj.hash;
|
|
|
+ if (!postItem.verifyMessage)
|
|
|
+ postItem.verifyMessage = itemObj.verifyMessage;
|
|
|
|
|
|
- if(!shouldVerify())
|
|
|
- await sendNews(itemObj);
|
|
|
- else
|
|
|
- await addVerifyMessage(itemObj);
|
|
|
+ itemObj = postItem;
|
|
|
+ itemObj.verifyMessage.isNew = false;
|
|
|
+ itemObj.hash = newHash;
|
|
|
}
|
|
|
+ else
|
|
|
+ continue;
|
|
|
}
|
|
|
+
|
|
|
+ if (!shouldVerify())
|
|
|
+ await sendNews(itemObj);
|
|
|
+ else
|
|
|
+ await addVerifyMessage(itemObj);
|
|
|
}
|
|
|
+
|
|
|
+ // for(let feedEntry of FEEDS) {
|
|
|
+ // let feed = await parser.parseURL(feedEntry.url);
|
|
|
+ // if(feed.items.length == 0)
|
|
|
+ // continue;
|
|
|
+ // let printableItems = feed.items.sort((a : any, b: any) => a.isoDate.localeCompare(b.isoDate));
|
|
|
+ // if(printableItems.length > 0) {
|
|
|
+ // for(let item of printableItems) {
|
|
|
+ // let itemID = getThreadId(item.guid);
|
|
|
+ // let contents = null;
|
|
|
+
|
|
|
+ // try {
|
|
|
+ // let res = await request(item.link, {resolveWithFullResponse: true}) as Response;
|
|
|
+ // if(res.statusCode != 200) {
|
|
|
+ // console.log(`Post ${itemID} could not be loaded because request returned status ${res.statusCode}`);
|
|
|
+ // continue;
|
|
|
+ // }
|
|
|
+
|
|
|
+ // let rootNode = html.parse(res.body, {
|
|
|
+ // pre: true,
|
|
|
+ // script: false,
|
|
|
+ // style: false
|
|
|
+ // });
|
|
|
+
|
|
|
+ // if(!(rootNode instanceof html.HTMLElement))
|
|
|
+ // continue;
|
|
|
+
|
|
|
+ // let opDiv = rootNode.querySelector("div.bbWrapper");
|
|
|
+
|
|
|
+ // if (!opDiv) {
|
|
|
+ // console.log(`No posts found for ${itemID}!`);
|
|
|
+ // continue;
|
|
|
+ // }
|
|
|
+
|
|
|
+ // contents = markdownify(opDiv.outerHTML, item.link);
|
|
|
+ // } catch(err){
|
|
|
+ // console.log(`Failed to get html for item ${itemID} because ${err}`);
|
|
|
+ // continue;
|
|
|
+ // }
|
|
|
+
|
|
|
+
|
|
|
+ // }
|
|
|
+ // }
|
|
|
+ // }
|
|
|
}
|
|
|
|
|
|
async function initPendingReactors() {
|
|
@@ -142,14 +166,14 @@ async function initPendingReactors() {
|
|
|
|
|
|
let pendingVerifyMessages = await repo.find({
|
|
|
where: { verifyMessage: Not(IsNull()) },
|
|
|
- select: [ "id" ],
|
|
|
- relations: [ "verifyMessage" ]
|
|
|
+ select: ["id"],
|
|
|
+ relations: ["verifyMessage"]
|
|
|
});
|
|
|
|
|
|
- for(let msg of pendingVerifyMessages) {
|
|
|
+ for (let msg of pendingVerifyMessages) {
|
|
|
let m = await tryFetchMessage(verifyChannel, msg.verifyMessage.messageId);
|
|
|
|
|
|
- if(!m) {
|
|
|
+ if (!m) {
|
|
|
await verifyMessageRepo.delete(msg.verifyMessage);
|
|
|
await repo.update({ id: m.id }, { verifyMessage: null })
|
|
|
continue;
|
|
@@ -162,20 +186,20 @@ async function initPendingReactors() {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-async function addVerifyMessage(item : PostedForumNewsItem) {
|
|
|
+async function addVerifyMessage(item: PostedForumNewsItem) {
|
|
|
let verifyChannel = client.channels.get(verifyChannelId) as TextChannel;
|
|
|
let verifyMessageRepo = getRepository(PostVerifyMessage);
|
|
|
let forumsNewsRepo = getRepository(PostedForumNewsItem);
|
|
|
|
|
|
- if(item.verifyMessage.messageId){
|
|
|
+ if (item.verifyMessage.messageId) {
|
|
|
let oldMessage = await tryFetchMessage(verifyChannel, item.verifyMessage.messageId);
|
|
|
- if(oldMessage)
|
|
|
+ if (oldMessage)
|
|
|
await oldMessage.delete();
|
|
|
}
|
|
|
|
|
|
let newMessage = await verifyChannel.send(toVerifyString(item.id, item.verifyMessage)) as Message;
|
|
|
item.verifyMessage.messageId = newMessage.id;
|
|
|
-
|
|
|
+
|
|
|
await newMessage.react("✅");
|
|
|
await newMessage.react("❌");
|
|
|
|
|
@@ -183,15 +207,15 @@ async function addVerifyMessage(item : PostedForumNewsItem) {
|
|
|
collector.on("collect", collectReaction)
|
|
|
reactionCollectors[newMessage.id] = collector;
|
|
|
verifyMessageIdToPost[newMessage.id] = item;
|
|
|
-
|
|
|
+
|
|
|
item.verifyMessage = await verifyMessageRepo.save(item.verifyMessage);
|
|
|
await forumsNewsRepo.save(item);
|
|
|
}
|
|
|
|
|
|
-async function collectReaction(reaction : MessageReaction, collector: Collector<string, MessageReaction>) {
|
|
|
+async function collectReaction(reaction: MessageReaction, collector: Collector<string, MessageReaction>) {
|
|
|
let verifyMessageRepo = getRepository(PostVerifyMessage);
|
|
|
let postRepo = getRepository(PostedForumNewsItem);
|
|
|
-
|
|
|
+
|
|
|
let m = reaction.message;
|
|
|
collector.stop();
|
|
|
delete reactionCollectors[m.id];
|
|
@@ -201,11 +225,11 @@ async function collectReaction(reaction : MessageReaction, collector: Collector<
|
|
|
await verifyMessageRepo.delete({ id: post.verifyMessage.id });
|
|
|
await reaction.message.delete();
|
|
|
|
|
|
- if(reaction.emoji.name == "✅")
|
|
|
+ if (reaction.emoji.name == "✅")
|
|
|
sendNews(post);
|
|
|
}
|
|
|
|
|
|
-async function sendNews(item : PostedForumNewsItem) {
|
|
|
+async function sendNews(item: PostedForumNewsItem) {
|
|
|
let channelRepo = getRepository(KnownChannel);
|
|
|
let newsPostRepo = getRepository(PostedForumNewsItem);
|
|
|
|
|
@@ -221,16 +245,16 @@ async function sendNews(item : PostedForumNewsItem) {
|
|
|
await newsPostRepo.save(item);
|
|
|
}
|
|
|
|
|
|
-function isVerifyReaction(reaction : MessageReaction, user: User) {
|
|
|
+function isVerifyReaction(reaction: MessageReaction, user: User) {
|
|
|
return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot;
|
|
|
}
|
|
|
|
|
|
-async function tryFetchMessage(channel: Channel, messageId: string) {
|
|
|
+async function tryFetchMessage(channel: Channel, messageId: string) {
|
|
|
try {
|
|
|
- if(!(channel instanceof TextChannel))
|
|
|
+ if (!(channel instanceof TextChannel))
|
|
|
return null;
|
|
|
return await channel.fetchMessage(messageId);
|
|
|
- }catch(error){
|
|
|
+ } catch (error) {
|
|
|
return null;
|
|
|
}
|
|
|
}
|
|
@@ -239,21 +263,21 @@ function shouldVerify() {
|
|
|
return verifyChannelId != null;
|
|
|
}
|
|
|
|
|
|
-async function postNewsItem(channel: string, item: PostedForumNewsItem) : Promise<Message | null> {
|
|
|
+async function postNewsItem(channel: string, item: PostedForumNewsItem): Promise<Message | null> {
|
|
|
let newsMessage = toNewsString(item.verifyMessage);
|
|
|
let ch = client.channels.get(channel);
|
|
|
|
|
|
- if(!(ch instanceof TextChannel))
|
|
|
+ if (!(ch instanceof TextChannel))
|
|
|
return null;
|
|
|
|
|
|
- if(item.postedMessageId) {
|
|
|
+ if (item.postedMessageId) {
|
|
|
let message = await tryFetchMessage(ch, item.postedMessageId);
|
|
|
- if(message)
|
|
|
+ if (message)
|
|
|
return await message.edit(newsMessage);
|
|
|
- else
|
|
|
+ else
|
|
|
return await ch.send(newsMessage) as Message;
|
|
|
- }
|
|
|
- else
|
|
|
+ }
|
|
|
+ else
|
|
|
return await ch.send(newsMessage) as Message;
|
|
|
}
|
|
|
|
|
@@ -270,7 +294,7 @@ ${item.text}`;
|
|
|
}
|
|
|
|
|
|
function toVerifyString(postId: string, item: PostVerifyMessage) {
|
|
|
- return `[${item.isNew ? "🆕 ADD": "✏️ EDIT"}]
|
|
|
+ return `[${item.isNew ? "🆕 ADD" : "✏️ EDIT"}]
|
|
|
Post ID: **${postId}**
|
|
|
|
|
|
${toNewsString(item)}
|
|
@@ -283,9 +307,9 @@ export default {
|
|
|
{
|
|
|
pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i,
|
|
|
action: async (msg, s, match) => {
|
|
|
- if(msg.channel.id != verifyChannelId)
|
|
|
+ if (msg.channel.id != verifyChannelId)
|
|
|
return;
|
|
|
-
|
|
|
+
|
|
|
let id = match[1];
|
|
|
let newContents = match[2].trim();
|
|
|
|
|
@@ -294,23 +318,23 @@ export default {
|
|
|
|
|
|
let post = await repo.findOne({
|
|
|
where: { id: id },
|
|
|
- relations: [ "verifyMessage" ]
|
|
|
+ relations: ["verifyMessage"]
|
|
|
});
|
|
|
|
|
|
- if(!post || !post.verifyMessage) {
|
|
|
+ if (!post || !post.verifyMessage) {
|
|
|
msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`);
|
|
|
return;
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
let editMsg = await tryFetchMessage(client.channels.get(verifyChannelId), post.verifyMessage.messageId);
|
|
|
-
|
|
|
- if(!editMsg){
|
|
|
+
|
|
|
+ if (!editMsg) {
|
|
|
msg.channel.send(`${msg.author.toString()} No verify message found for ${id}! This is a bug: report to horse.`);
|
|
|
return;
|
|
|
}
|
|
|
-
|
|
|
+
|
|
|
post.verifyMessage.text = newContents;
|
|
|
-
|
|
|
+
|
|
|
await verifyRepo.save(post.verifyMessage);
|
|
|
await editMsg.edit(toVerifyString(post.id, post.verifyMessage));
|
|
|
await msg.delete();
|
|
@@ -325,7 +349,7 @@ export default {
|
|
|
channelType: NEWS_POST_VERIFY_CHANNEL
|
|
|
});
|
|
|
|
|
|
- if(verifyChannel)
|
|
|
+ if (verifyChannel)
|
|
|
verifyChannelId = verifyChannel.channelId;
|
|
|
|
|
|
await initPendingReactors();
|