Add Xenforo client; use API to get news from forums

package.json

@@ -22,6 +22,8 @@
    "author": "Geoffrey Horsington <geoffrey.hoooooorse@gmail.com>",
    "license": "MIT",
    "dependencies": {
+      "@bbob/html": "^2.5.2",
+      "@bbob/preset-html5": "^2.5.2",
       "@types/dotenv": "^6.1.1",
       "@types/lowdb": "^1.0.9",
       "@types/request-promise-native": "^1.0.16",

src/client.ts

@@ -1,2 +1,7 @@
 import { Client } from "discord.js";
-export default new Client();
+import { XenforoClient } from "./xenforo";
+
+
+export const client = new Client();
+export const forumClient = new XenforoClient("https://custommaid3d2.com/api", process.env.FORUM_API_KEY);
+

src/commands/facemorph.ts

@@ -1,6 +1,6 @@
 import { isValidImage } from "../util";
 import Jimp from "jimp";
-import client from "../client";
+import { client } from "../client";
 import * as cv from "opencv4nodejs";
 import * as path from "path";
 import request from "request-promise-native";

src/commands/rss_checker.ts

@@ -1,25 +1,24 @@
 import TurndownService, { Options } from "turndown";
 import RSSParser from "rss-parser";
 import interval from "interval-promise";
-import client from "../client";
+import { client, forumClient } from "../client";
 import sha1 from "sha1";
-import * as html from "node-html-parser";
-import request from "request-promise-native";
 import { ICommand } from "./command";
-import { Response } from "request";
 import { TextChannel, Message, ReactionCollector, MessageReaction, Collector, User, Channel } from "discord.js";
 import { Dict } from "../util";
 import { getRepository, Not, IsNull } from "typeorm";
 import { PostedForumNewsItem } from "../entity/PostedForumsNewsItem";
 import { KnownChannel } from "../entity/KnownChannel";
 import { PostVerifyMessage } from "../entity/PostVerifyMessage";
+import bbobHTML from '@bbob/html'
+import presetHTML5 from '@bbob/preset-html5'
 
 const PREVIEW_CHAR_LIMIT = 300;
 const NEWS_POST_VERIFY_CHANNEL = "newsPostVerify";
 
-let verifyChannelId : string = null;
-const reactionCollectors : Dict<ReactionCollector> = {};
-const verifyMessageIdToPost : Dict<PostedForumNewsItem> = {};
+let verifyChannelId: string = null;
+const reactionCollectors: Dict<ReactionCollector> = {};
+const verifyMessageIdToPost: Dict<PostedForumNewsItem> = {};
 const NEWS_FEED_CHANNEL = "newsFeed";
 
 const turndown = new TurndownService();
@@ -28,7 +27,7 @@ turndown.addRule("image", {
     replacement: () => ""
 });
 turndown.addRule("link", {
-    filter: (node : HTMLElement, opts: Options) => node.nodeName === "A" && node.getAttribute("href") != null,
+    filter: (node: HTMLElement, opts: Options) => node.nodeName === "A" && node.getAttribute("href") != null,
     replacement: (content: string, node: HTMLElement) => node.getAttribute("href")
 });
 
@@ -37,11 +36,13 @@ const RSS_UPDATE_INTERVAL_MIN = 5;
 
 function getThreadId(url: string) {
     let result = url.substring(url.lastIndexOf(".") + 1);
-    if(result.endsWith("/"))
+    if (result.endsWith("/"))
         result = result.substring(0, result.length - 1);
     return result;
 }
 
+const NEWS_FORUM_ID = 49;
+
 const FEEDS = [
     {
         url: "http://custommaid3d2.com/index.php?forums/news.49/index.rss",
@@ -49,89 +50,112 @@ const FEEDS = [
     }
 ];
 
+function bbCodeToMarkdown(bbCode: string) {
+    return turndown.turndown(bbobHTML(bbCode, presetHTML5())).replace(/( {2}\n|\n\n){2,}/gm, "\n");
+}
+
 async function checkFeeds() {
     console.log(`Checking feeds on ${new Date().toISOString()}`);
     let forumsNewsRepo = getRepository(PostedForumNewsItem);
     let postVerifyMessageRepo = getRepository(PostVerifyMessage);
 
-    for(let feedEntry of FEEDS) {
-        let feed = await parser.parseURL(feedEntry.url);
-        if(feed.items.length == 0)
-            continue;
-        let printableItems = feed.items.sort((a : any, b: any) => a.isoDate.localeCompare(b.isoDate));
-        if(printableItems.length > 0) {
-            for(let item of printableItems) {
-                let itemID = getThreadId(item.guid);
-                let contents = null;
-                
-                try {
-                    let res = await request(item.link, {resolveWithFullResponse: true}) as Response;
-                    if(res.statusCode != 200) {
-                        console.log(`Post ${itemID} could not be loaded because request returned status ${res.statusCode}`);
-                        continue;
-                    }
-
-                    let rootNode = html.parse(res.body, {
-                        pre: true,
-                        script: false,
-                        style: false
-                    });
-
-                    if(!(rootNode instanceof html.HTMLElement))
-                        continue;
-
-                    let opDiv = rootNode.querySelector("div.bbWrapper");
-
-                    if (!opDiv) {
-                        console.log(`No posts found for ${itemID}!`);
-                        continue;
-                    }
-                    
-                    contents = markdownify(opDiv.outerHTML, item.link);
-                } catch(err){
-                    console.log(`Failed to get html for item ${itemID} because ${err}`);
-                    continue;
-                }
+    let forumThreads = await forumClient.getForumThreads(NEWS_FORUM_ID);
+
+    for (let thread of forumThreads.threads) {
+        let firstPost = await forumClient.getPost(thread.first_post_id);
+
+        let contents = bbCodeToMarkdown(firstPost.message);
+        let itemObj = forumsNewsRepo.create({
+            id: thread.thread_id.toString(),
+            hash: sha1(firstPost.message),
+            verifyMessage: postVerifyMessageRepo.create({
+                author: thread.username,
+                link: `https://custommaid3d2.com/index.php?threads/${thread.thread_id}/`,
+                title: thread.title,
+                text: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
+                isNew: true
+            })
+        });
+
+        let postItem = await forumsNewsRepo.findOne({
+            where: { id: itemObj.id },
+            relations: ["verifyMessage"]
+        });
+
+        if (postItem) {
 
-                let itemObj = forumsNewsRepo.create({
-                        id: itemID,
-                        hash: sha1(contents),
-                        verifyMessage: postVerifyMessageRepo.create({
-                            author: item.creator,
-                            link: item.link,
-                            title: item.title,
-                            text: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
-                            isNew: true
-                        })
-                    });
-
-                let postItem = await forumsNewsRepo.findOne({ 
-                    where: { id: itemObj.id },
-                    relations: [ "verifyMessage" ]
+            if(process.env.INGORE_CHANGED_NEWS === "TRUE") {
+                await forumsNewsRepo.update({
+                    id: postItem.id
+                }, {
+                    hash: itemObj.hash
                 });
+                continue;
+            }
 
-                if(postItem){
-                    // Add message ID to mark for edit
-                    if(postItem.hash != itemObj.hash){
-                        let newHash = itemObj.hash;
-                        if(!postItem.verifyMessage)
-                            postItem.verifyMessage = itemObj.verifyMessage;
-                        
-                        itemObj = postItem;
-                        itemObj.verifyMessage.isNew = false;
-                        itemObj.hash = newHash;
-                    }
-                    else 
-                        continue;
-                }
+            // Add message ID to mark for edit
+            if (postItem.hash != itemObj.hash) {
+                let newHash = itemObj.hash;
+                if (!postItem.verifyMessage)
+                    postItem.verifyMessage = itemObj.verifyMessage;
 
-                if(!shouldVerify())
-                    await sendNews(itemObj);
-                else
-                    await addVerifyMessage(itemObj);
+                itemObj = postItem;
+                itemObj.verifyMessage.isNew = false;
+                itemObj.hash = newHash;
             }
+            else
+                continue;
         }
+
+        if (!shouldVerify())
+            await sendNews(itemObj);
+        else
+            await addVerifyMessage(itemObj);
     }
+
+    // for(let feedEntry of FEEDS) {
+    //     let feed = await parser.parseURL(feedEntry.url);
+    //     if(feed.items.length == 0)
+    //         continue;
+    //     let printableItems = feed.items.sort((a : any, b: any) => a.isoDate.localeCompare(b.isoDate));
+    //     if(printableItems.length > 0) {
+    //         for(let item of printableItems) {
+    //             let itemID = getThreadId(item.guid);
+    //             let contents = null;
+
+    //             try {
+    //                 let res = await request(item.link, {resolveWithFullResponse: true}) as Response;
+    //                 if(res.statusCode != 200) {
+    //                     console.log(`Post ${itemID} could not be loaded because request returned status ${res.statusCode}`);
+    //                     continue;
+    //                 }
+
+    //                 let rootNode = html.parse(res.body, {
+    //                     pre: true,
+    //                     script: false,
+    //                     style: false
+    //                 });
+
+    //                 if(!(rootNode instanceof html.HTMLElement))
+    //                     continue;
+
+    //                 let opDiv = rootNode.querySelector("div.bbWrapper");
+
+    //                 if (!opDiv) {
+    //                     console.log(`No posts found for ${itemID}!`);
+    //                     continue;
+    //                 }
+
+    //                 contents = markdownify(opDiv.outerHTML, item.link);
+    //             } catch(err){
+    //                 console.log(`Failed to get html for item ${itemID} because ${err}`);
+    //                 continue;
+    //             }
+
+
+    //         }
+    //     }
+    // }
 }
 
 async function initPendingReactors() {
@@ -142,14 +166,14 @@ async function initPendingReactors() {
 
     let pendingVerifyMessages = await repo.find({
         where: { verifyMessage: Not(IsNull()) },
-        select: [ "id" ],
-        relations: [ "verifyMessage" ]
+        select: ["id"],
+        relations: ["verifyMessage"]
     });
 
-    for(let msg of pendingVerifyMessages) {
+    for (let msg of pendingVerifyMessages) {
         let m = await tryFetchMessage(verifyChannel, msg.verifyMessage.messageId);
 
-        if(!m) {
+        if (!m) {
             await verifyMessageRepo.delete(msg.verifyMessage);
             await repo.update({ id: m.id }, { verifyMessage: null })
             continue;
@@ -162,20 +186,20 @@ async function initPendingReactors() {
     }
 }
 
-async function addVerifyMessage(item : PostedForumNewsItem) {
+async function addVerifyMessage(item: PostedForumNewsItem) {
     let verifyChannel = client.channels.get(verifyChannelId) as TextChannel;
     let verifyMessageRepo = getRepository(PostVerifyMessage);
     let forumsNewsRepo = getRepository(PostedForumNewsItem);
 
-    if(item.verifyMessage.messageId){
+    if (item.verifyMessage.messageId) {
         let oldMessage = await tryFetchMessage(verifyChannel, item.verifyMessage.messageId);
-        if(oldMessage)
+        if (oldMessage)
             await oldMessage.delete();
     }
 
     let newMessage = await verifyChannel.send(toVerifyString(item.id, item.verifyMessage)) as Message;
     item.verifyMessage.messageId = newMessage.id;
-    
+
     await newMessage.react("✅");
     await newMessage.react("❌");
 
@@ -183,15 +207,15 @@ async function addVerifyMessage(item : PostedForumNewsItem) {
     collector.on("collect", collectReaction)
     reactionCollectors[newMessage.id] = collector;
     verifyMessageIdToPost[newMessage.id] = item;
-    
+
     item.verifyMessage = await verifyMessageRepo.save(item.verifyMessage);
     await forumsNewsRepo.save(item);
 }
 
-async function collectReaction(reaction : MessageReaction, collector: Collector<string, MessageReaction>) {
+async function collectReaction(reaction: MessageReaction, collector: Collector<string, MessageReaction>) {
     let verifyMessageRepo = getRepository(PostVerifyMessage);
     let postRepo = getRepository(PostedForumNewsItem);
-    
+
     let m = reaction.message;
     collector.stop();
     delete reactionCollectors[m.id];
@@ -201,11 +225,11 @@ async function collectReaction(reaction : MessageReaction, collector: Collector<
     await verifyMessageRepo.delete({ id: post.verifyMessage.id });
     await reaction.message.delete();
 
-    if(reaction.emoji.name == "✅")
+    if (reaction.emoji.name == "✅")
         sendNews(post);
 }
 
-async function sendNews(item : PostedForumNewsItem) {
+async function sendNews(item: PostedForumNewsItem) {
     let channelRepo = getRepository(KnownChannel);
     let newsPostRepo = getRepository(PostedForumNewsItem);
 
@@ -221,16 +245,16 @@ async function sendNews(item : PostedForumNewsItem) {
     await newsPostRepo.save(item);
 }
 
-function isVerifyReaction(reaction : MessageReaction, user: User) {
+function isVerifyReaction(reaction: MessageReaction, user: User) {
     return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot;
 }
 
-async function tryFetchMessage(channel: Channel, messageId: string)  {
+async function tryFetchMessage(channel: Channel, messageId: string) {
     try {
-        if(!(channel instanceof TextChannel))
+        if (!(channel instanceof TextChannel))
             return null;
         return await channel.fetchMessage(messageId);
-    }catch(error){
+    } catch (error) {
         return null;
     }
 }
@@ -239,21 +263,21 @@ function shouldVerify() {
     return verifyChannelId != null;
 }
 
-async function postNewsItem(channel: string, item: PostedForumNewsItem) : Promise<Message | null> {
+async function postNewsItem(channel: string, item: PostedForumNewsItem): Promise<Message | null> {
     let newsMessage = toNewsString(item.verifyMessage);
     let ch = client.channels.get(channel);
 
-    if(!(ch instanceof TextChannel))
+    if (!(ch instanceof TextChannel))
         return null;
 
-    if(item.postedMessageId) {
+    if (item.postedMessageId) {
         let message = await tryFetchMessage(ch, item.postedMessageId);
-        if(message)
+        if (message)
             return await message.edit(newsMessage);
-        else 
+        else
             return await ch.send(newsMessage) as Message;
-    } 
-    else 
+    }
+    else
         return await ch.send(newsMessage) as Message;
 }
 
@@ -270,7 +294,7 @@ ${item.text}`;
 }
 
 function toVerifyString(postId: string, item: PostVerifyMessage) {
-    return `[${item.isNew ? "🆕 ADD": "✏️ EDIT"}]
+    return `[${item.isNew ? "🆕 ADD" : "✏️ EDIT"}]
 Post ID: **${postId}**
     
 ${toNewsString(item)}
@@ -283,9 +307,9 @@ export default {
         {
             pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i,
             action: async (msg, s, match) => {
-                if(msg.channel.id != verifyChannelId)
+                if (msg.channel.id != verifyChannelId)
                     return;
-    
+
                 let id = match[1];
                 let newContents = match[2].trim();
 
@@ -294,23 +318,23 @@ export default {
 
                 let post = await repo.findOne({
                     where: { id: id },
-                    relations: [ "verifyMessage" ]
+                    relations: ["verifyMessage"]
                 });
 
-                if(!post || !post.verifyMessage) {
+                if (!post || !post.verifyMessage) {
                     msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`);
                     return;
                 }
-                
+
                 let editMsg = await tryFetchMessage(client.channels.get(verifyChannelId), post.verifyMessage.messageId);
-    
-                if(!editMsg){
+
+                if (!editMsg) {
                     msg.channel.send(`${msg.author.toString()} No verify message found for ${id}! This is a bug: report to horse.`);
                     return;
                 }
-    
+
                 post.verifyMessage.text = newContents;
-                
+
                 await verifyRepo.save(post.verifyMessage);
                 await editMsg.edit(toVerifyString(post.id, post.verifyMessage));
                 await msg.delete();
@@ -325,7 +349,7 @@ export default {
             channelType: NEWS_POST_VERIFY_CHANNEL
         });
 
-        if(verifyChannel)
+        if (verifyChannel)
             verifyChannelId = verifyChannel.channelId;
 
         await initPendingReactors();

src/commands/news_aggregator.ts

@@ -1,6 +1,6 @@
 import TurndownService, { Options } from "turndown";
 import interval from "interval-promise";
-import client from "../client";
+import { client } from "../client";
 import sha1 from "sha1";
 import * as path from "path";
 import * as fs from "fs";

src/commands/react.ts

@@ -1,4 +1,4 @@
-import client from "../client";
+import { client } from "../client";
 import { ICommand } from "./command";
 import { getRepository } from "typeorm";
 import { MessageReaction } from "../entity/MessageReaction";

src/main.ts

@@ -1,14 +1,15 @@
 import * as dotenv from "dotenv";
+dotenv.config();
+
 import * as fs from "fs";
 import * as path from "path";
-import client from "./client";
+import { client } from "./client";
 import { ICommand, BotEvent, IBotCommand } from "./commands/command"
 import "reflect-metadata";
 import {createConnection} from "typeorm";
 import { migrate } from "./lowdb_migrator";
 import { documentation } from "./util";
 
-dotenv.config();
 const REACT_PROBABILITY = 0.3;
 
 async function trigger(actions : BotEvent[], ...params: any[]) {

src/typedefs/bobb.d.ts

@@ -0,0 +1,2 @@
+declare module '@bbob/html';
+declare module '@bbob/preset-html5';

src/util.ts

@@ -1,4 +1,3 @@
-import { CollectionChain } from "lodash";
 import { GuildMember } from "discord.js";
 import { DocumentationSet } from "./commands/command";
 import { getRepository } from "typeorm";

src/xenforo.ts

@@ -1,4 +1,5 @@
 import request from "request-promise-native";
+import { Response } from "request";
 import { Dict } from "./util";
 
 enum ReqMethod {
@@ -16,11 +17,11 @@ export interface RequestError {
 export type RequestErrorSet = { errors: RequestError[] };
 
 export class XenforoClient {
-    
-    constructor(private endpoint: string, private userKey: string) {  
+
+    constructor(private endpoint: string, private userKey: string) {
     }
 
-    private async makeRequest<T>(uri :string, method: ReqMethod, data?: any) {
+    private async makeRequest<T>(uri: string, method: ReqMethod, data?: any) {
         let result = await request(`${this.endpoint}/${uri}`, {
             method: method,
             headers: {
@@ -30,22 +31,22 @@ export class XenforoClient {
             resolveWithFullResponse: true
         }) as Response;
 
-        if(result.status != 200){
-            throw await result.json() as RequestErrorSet;
+        if (result.statusCode != 200) {
+            throw await JSON.parse(result.body) as RequestErrorSet;
         } else {
-            return await result.json() as T;
+            return await JSON.parse(result.body) as T;
         }
     }
 
-    async getThread(id: string, opts?: GetThreadOptions) {
+    async getThread(id: number, opts?: GetThreadOptions) {
         return await this.makeRequest<GetThreadResponse>(`threads/${id}`, ReqMethod.GET, opts);
     }
 
-    async deleteThread(id: string, opts?: DeleteThreadOptions) {
+    async deleteThread(id: number, opts?: DeleteThreadOptions) {
         return await this.makeRequest<SuccessResponse>(`threads/${id}`, ReqMethod.DELETE, opts);
     }
 
-    async createThread(forumId: string, title: string, message: string, opts?: CreateThreadOptions) {
+    async createThread(forumId: number, title: string, message: string, opts?: CreateThreadOptions) {
         return await this.makeRequest<CreateThreadResponse>(`threads/`, ReqMethod.POST, {
             node_id: forumId,
             title: title,
@@ -53,8 +54,16 @@ export class XenforoClient {
             ...opts
         });
     }
-}
 
+    async getPost(id: number) {
+        let post = await this.makeRequest<{post: Post}>(`posts/${id}`, ReqMethod.GET);
+        return post.post;
+    }
+
+    async getForumThreads(id: number) {
+        return await this.makeRequest<GetForumThreadsResponse>(`forums/${id}/threads`, ReqMethod.GET);
+    }
+}
 
 //#region Request types
 interface DeleteThreadOptions {
@@ -91,6 +100,12 @@ type SuccessResponse = {
 }
 
 type CreateThreadResponse = SuccessResponse & { thread: Thread; };
+
+type GetForumThreadsResponse = {
+    threads: Thread[];
+    pagination: object;
+    sticky: Thread[];
+};
 //#endregion
 
 //#region Data types