Add message editor for rss checker

commands/rss_checker.js

@@ -4,7 +4,10 @@ const db = require("../db.js");
 const interval = require("interval-promise");
 const client = require("../client.js");
 const sha1 = require("sha1");
+const html = require("node-html-parser"); 
+const axios = require("axios");
 
+const PREVIEW_CHAR_LIMIT = 300;
 const verifyChannelID = db.get("newsPostVerifyChannel").value();
 
 const reactionCollectors = {};
@@ -42,22 +45,46 @@ async function checkFeeds() {
         if(printableItems.length > 0) {
             for(let item of printableItems) {
                 let itemID = getThreadId(item.guid);
+                let contents = null;
+                
+                try {
+                    let res = await axios.get(item.link);
+                    if(res.status != 200) {
+                        console.log(`Post ${itemID} could not be loaded because request returned status ${res.status}`);
+                        continue;
+                    }
+
+                    let rootNode = html.parse(res.data, {
+                        pre: true,
+                        script: false,
+                        style: false
+                    });
+
+                    let opDiv = rootNode.querySelector("div.bbWrapper");
+
+                    if (!opDiv) {
+                        console.log(`No posts found for ${itemID}!`);
+                        continue;
+                    }
+                    
+                    contents = markdownify(opDiv.outerHTML, item.link);
+                } catch(err){
+                    console.log(`Failed to get html for item ${itemID} because ${err}`);
+                    continue;
+                }
 
-                let contents = item[feedEntry.contentElement];
                 let itemObj = {
                     id: itemID,
                     title: item.title,
                     link: item.link,
                     creator: item.creator,
-                    contents: contents,
-                    hash: null,
+                    contents: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
+                    hash: sha1(contents),
                     messageId: null,
-                    verifyMessageId: null
+                    verifyMessageId: null,
+                    type: null
                 };
 
-                itemObj.contents = newsToString(itemObj);
-                itemObj.hash = sha1(itemObj.contents);
-
                 if(oldNews.has(itemObj.id).value()){
                     let data = oldNews.get(itemObj.id).value();
                     // Old type, don't care
@@ -99,10 +126,10 @@ async function addVerifyMessage(item) {
     let cache = db.get("newsCache");
     let oldNews = db.get("postedNewsGuids");
     let postedNews = db.get("postedNewsGuids");
-    let process = "🆕 ADD";
+    item.type = "🆕 ADD";
 
     if(postedNews.has(item.id).value())
-        process = "✏️ EDIT";
+        item.type = "✏️ EDIT";
 
     if(cache.has(item.id).value()) {
         let oldItem = cache.get(item.id).value();
@@ -111,14 +138,7 @@ async function addVerifyMessage(item) {
             await oldMessage.delete();
     }
 
-    let newMessage = await verifyChannel.send(`[${process}]
-Post ID: **${item.id}**
-
-${item.contents}
-        
-React with ✅ (approve) or ❌ (deny).`
-);
-
+    let newMessage = await verifyChannel.send(toVerifyString(item));
     
     await newMessage.react("✅");
     await newMessage.react("❌");
@@ -178,25 +198,39 @@ function shouldVerify() {
 }
 
 async function postNewsItem(channel, item) {
+    let newsMessage = toNewsString(item);
     let ch = client.channels.get(channel);
 
     if(item.messageId) {
         let message = await tryFetchMessage(ch, item.messageId);
         if(message)
-            return await message.edit(item.contents);
+            return await message.edit(newsMessage);
         else 
-            return await ch.send(item.contents);
+            return await ch.send(newsMessage);
     } 
     else 
-        return await ch.send(item.contents);
+        return await ch.send(newsMessage);
 }
 
-function newsToString(item) {
+function markdownify(htmStr, link) {
+    return turndown.turndown(htmStr).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "");
+}
+
+function toNewsString(item) {
     return `**${item.title}**
 Posted by ${item.creator}
 ${item.link}
 
-${turndown.turndown(item.contents).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(item.link, "")}`;
+${item.contents}`;
+}
+
+function toVerifyString(item) {
+    return `[${item.type}]
+Post ID: **${item.id}**
+    
+${toNewsString(item)}
+            
+React with ✅ (approve) or ❌ (deny).`;
 }
 
 const onStart = () => {
@@ -204,6 +238,40 @@ const onStart = () => {
     interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);
 };
 
+const commands = [
+    {
+        pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i,
+        action: async (msg, s, match) => {
+            if(msg.channel.id != verifyChannelID)
+                return;
+
+            let id = match[1];
+            let newContents = match[2].trim();
+            
+            if(!db.get("newsCache").has(id).value()) {
+                msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`);
+                return;
+            }
+
+            let item = db.get("newsCache").get(id).value();
+
+            let editMsg = await tryFetchMessage(client.channels.get(verifyChannelID), item.verifyMessageId);
+
+            if(!editMsg){
+                msg.channel.send(`${msg.author.toString()} No verify messafe found for ${id}! This is a bug: report to horse.`);
+                return;
+            }
+
+            item.contents = newContents;
+
+            db.get("newsCache").set(id, item).write();
+            await editMsg.edit(toVerifyString(item));
+            await msg.delete();
+        }
+    }
+];
+
 module.exports = {
-    onStart: onStart
+    onStart: onStart,
+    commands: commands
 };

package.json

@@ -20,11 +20,13 @@
   "author": "Geoffrey Horsington <geoffrey.hoooooorse@gmail.com>",
   "license": "MIT",
   "dependencies": {
+    "axios": "^0.19.0",
     "discord.js": "^11.4.2",
     "dotenv": "^8.0.0",
     "interval-promise": "^1.2.0",
     "jimp": "^0.5.4",
     "lowdb": "^1.0.0",
+    "node-html-parser": "^1.1.16",
     "opencv4nodejs": "^4.9.0",
     "request": "^2.88.0",
     "request-promise-native": "^1.0.5",