Update rss checker to use sqlite

src/commands/command.ts

@@ -23,5 +23,5 @@ export interface ICommand {
     onIndirectMention?(actionsDone: boolean, m: Message) : boolean | Promise<boolean>;
     onDirectMention?(actionsDone: boolean, m: Message, content: string) : boolean | Promise<boolean>;
     postMessage?(actionsDone: boolean, m: Message) : boolean | Promise<boolean>;
-    onStart?(): void;
+    onStart?(): void | Promise<void>;
 };

src/commands/react.ts

@@ -1,10 +1,10 @@
-import { isAuthorised } from "../util";
 import client from "../client";
 import { ICommand } from "./command";
 import { getRepository } from "typeorm";
 import { MessageReaction } from "../entity/MessageReaction";
 import { KnownUser } from "../entity/KnownUser";
 import { ReactionType, ReactionEmote } from "../entity/ReactionEmote";
+import { isAuthorisedAsync } from "../util";
 
 const pattern = /^react to\s+"([^"]+)"\s+with\s+\<:[^:]+:([^\>]+)\>$/i;
 
@@ -25,7 +25,7 @@ export default {
         {
             pattern: "react to",
             action: async (msg, s) => {
-                if (!isAuthorised(msg.member))
+                if (!await isAuthorisedAsync(msg.member))
                     return;
                 let contents = pattern.exec(s);
 
@@ -53,7 +53,7 @@ export default {
         {
             pattern: "remove reaction to",
             action: async (msg, s) => {
-                if (!isAuthorised(msg.member))
+                if (!await isAuthorisedAsync(msg.member))
                     return;
 
                 let content = s.substring("remove reaction to ".length).trim().toLowerCase();

src/commands/rss_checker.ts

@@ -1,6 +1,5 @@
 import TurndownService, { Options } from "turndown";
 import RSSParser from "rss-parser";
-import { db } from "../db";
 import interval from "interval-promise";
 import client from "../client";
 import sha1 from "sha1";
@@ -9,13 +8,19 @@ import request from "request-promise-native";
 import { ICommand } from "./command";
 import { Response } from "request";
 import { TextChannel, Message, ReactionCollector, MessageReaction, Collector, User, Channel } from "discord.js";
-import { Dictionary, ObjectChain } from "lodash";
+import { Dict } from "../util";
+import { getRepository, Not } from "typeorm";
+import { PostedForumNewsItem } from "../entity/PostedForumsNewsItem";
+import { KnownChannel } from "../entity/KnownChannel";
+import { PostVerifyMessage } from "../entity/PostVerifyMessage";
 
 const PREVIEW_CHAR_LIMIT = 300;
-const verifyChannelID = db.get("newsPostVerifyChannel").value();
+const NEWS_POST_VERIFY_CHANNEL = "newsPostVerify";
 
-const reactionCollectors : Dictionary<ReactionCollector> = {};
-const verifyMessageIdToPostId : Dictionary<string> = {};
+let verifyChannelId : string = null;
+const reactionCollectors : Dict<ReactionCollector> = {};
+const verifyMessageIdToPost : Dict<PostedForumNewsItem> = {};
+const NEWS_FEED_CHANNEL = "newsFeed";
 
 const turndown = new TurndownService();
 turndown.addRule("image", {
@@ -37,23 +42,19 @@ function getThreadId(url: string) {
     return result;
 }
 
-interface PostItem {
-    id: string,
-    title: string,
-    link: string,
-    creator: string,
-    contents: string,
-    hash: string,
-    messageId?: string,
-    verifyMessageId?: string,
-    type?: string
-}
+const FEEDS = [
+    {
+        url: "http://custommaid3d2.com/index.php?forums/news.49/index.rss",
+        contentElement: "content:encoded"
+    }
+];
 
 async function checkFeeds() {
     console.log(`Checking feeds on ${new Date().toISOString()}`);
-    let feeds = db.get("rssFeeds").value();
-    let oldNews = db.get("postedNewsGuids") as ObjectChain<any>;
-    for(let feedEntry of feeds) {
+    let forumsNewsRepo = getRepository(PostedForumNewsItem);
+    let postVerifyMessageRepo = getRepository(PostVerifyMessage);
+
+    for(let feedEntry of FEEDS) {
         let feed = await parser.parseURL(feedEntry.url);
         if(feed.items.length == 0)
             continue;
@@ -92,27 +93,32 @@ async function checkFeeds() {
                     continue;
                 }
 
-                let itemObj = {
-                    id: itemID,
-                    title: item.title,
-                    link: item.link,
-                    creator: item.creator,
-                    contents: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
-                    hash: sha1(contents),
-                    messageId: null,
-                    verifyMessageId: null,
-                    type: null
-                } as PostItem;
-
-                if(oldNews.has(itemObj.id).value()){
-                    let data = oldNews.get(itemObj.id).value();
-                    // Old type, don't care
-                    if(data === true)
-                        continue;
+                let itemObj = forumsNewsRepo.create({
+                        id: itemID,
+                        hash: sha1(contents),
+                        verifyMessage: postVerifyMessageRepo.create({
+                            author: item.creator,
+                            link: item.link,
+                            title: item.title,
+                            text: `${contents.substr(0, Math.min(contents.length, PREVIEW_CHAR_LIMIT))}...`,
+                            isNew: true
+                        })
+                    });
 
+                let postItem = await forumsNewsRepo.findOne({ 
+                    where: { id: itemObj.id },
+                    relations: [ "verifyMessage" ]
+                });
+
+                if(postItem){
                     // Add message ID to mark for edit
-                    if(data.hash != itemObj.hash)
-                        itemObj.messageId = data.messageId;
+                    if(postItem.hash != itemObj.hash){
+                        if(!postItem.verifyMessage)
+                            postItem.verifyMessage = itemObj.verifyMessage;
+                        
+                        itemObj = postItem;
+                        itemObj.verifyMessage.isNew = false;
+                    }
                     else 
                         continue;
                 }
@@ -122,44 +128,51 @@ async function checkFeeds() {
                 else
                     await addVerifyMessage(itemObj);
             }
-            let lastUpdateDate = printableItems[printableItems.length - 1].isoDate;
-            console.log(`Setting last update marker on ${feedEntry.url} to ${lastUpdateDate}`);
-            let rssFeeds = db.get("rssFeeds") as ObjectChain<any>;
-            let entry = rssFeeds.find({ url: feedEntry.url}) as ObjectChain<any>;
-            entry.assign({lastUpdate: lastUpdateDate}).write();
         }
     }
 }
 
-function initPendingReactors() {
-    let verifyChannel = client.channels.get(verifyChannelID);
-    db.get("newsCache").forOwn(async (i : any) => {
-        let m = await tryFetchMessage(verifyChannel, i.verifyMessageId);
+async function initPendingReactors() {
+    let verifyChannel = client.channels.get(verifyChannelId);
+
+    let repo = getRepository(PostedForumNewsItem);
+    let verifyMessageRepo = getRepository(PostVerifyMessage);
+
+    let pendingVerifyMessages = await repo.find({
+        where: { verifyMessage: Not(null) },
+        select: [ "id" ],
+        relations: [ "verifyMessage" ]
+    });
+
+    for(let msg of pendingVerifyMessages) {
+        let m = await tryFetchMessage(verifyChannel, msg.verifyMessage.messageId);
+
+        if(!m) {
+            await verifyMessageRepo.delete(msg.verifyMessage);
+            await repo.update({ id: m.id }, { verifyMessage: null })
+            continue;
+        }
+
         let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
-        collector.on("collect", collectReaction)
+        collector.on("collect", collectReaction);
         reactionCollectors[m.id] = collector;
-        verifyMessageIdToPostId[m.id] = i.id;
-    }).value();
+        verifyMessageIdToPost[m.id] = msg;
+    }
 }
 
-async function addVerifyMessage(item : PostItem) {
-    let verifyChannel = client.channels.get(verifyChannelID) as TextChannel;
-    let cache = db.get("newsCache") as ObjectChain<any>;
-    let oldNews = db.get("postedNewsGuids");
-    let postedNews = db.get("postedNewsGuids");
-    item.type = "🆕 ADD";
+async function addVerifyMessage(item : PostedForumNewsItem) {
+    let verifyChannel = client.channels.get(verifyChannelId) as TextChannel;
+    let verifyMessageRepo = getRepository(PostVerifyMessage);
+    let forumsNewsRepo = getRepository(PostedForumNewsItem);
 
-    if(postedNews.has(item.id).value())
-        item.type = "✏️ EDIT";
-
-    if(cache.has(item.id).value()) {
-        let oldItem = cache.get(item.id).value();
-        let oldMessage = await tryFetchMessage(verifyChannel, oldItem.verifyMessageId);
+    if(item.verifyMessage.messageId){
+        let oldMessage = await tryFetchMessage(verifyChannel, item.verifyMessage.messageId);
         if(oldMessage)
             await oldMessage.delete();
     }
 
-    let newMessage = await verifyChannel.send(toVerifyString(item)) as Message;
+    let newMessage = await verifyChannel.send(toVerifyString(item.id, item.verifyMessage)) as Message;
+    item.verifyMessage.messageId = newMessage.id;
     
     await newMessage.react("✅");
     await newMessage.react("❌");
@@ -167,41 +180,41 @@ async function addVerifyMessage(item : PostItem) {
     let collector = newMessage.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
     collector.on("collect", collectReaction)
     reactionCollectors[newMessage.id] = collector;
-    verifyMessageIdToPostId[newMessage.id] = item.id;
-    item.verifyMessageId = newMessage.id;
-    cache.set(item.id, item);
-
-    oldNews.set(item.id, {
-        hash: item.hash,
-        messageId: null
-    });
-    db.write();
+    verifyMessageIdToPost[newMessage.id] = item;
+    
+    item.verifyMessage = await verifyMessageRepo.save(item.verifyMessage);
+    await forumsNewsRepo.save(item);
 }
 
-function collectReaction(reaction : MessageReaction, collector: Collector<string, MessageReaction>) {
-    let cache = db.get("newsCache") as ObjectChain<any>;
+async function collectReaction(reaction : MessageReaction, collector: Collector<string, MessageReaction>) {
+    let verifyMessageRepo = getRepository(PostedForumNewsItem);
+    
     let m = reaction.message;
     collector.stop();
     delete reactionCollectors[m.id];
-    let postId = verifyMessageIdToPostId[m.id];
-    let item = cache.get(postId).value() as PostItem;
-    cache.unset(postId).write();
-    m.delete();
+    let post = verifyMessageIdToPost[m.id];
+
+    await verifyMessageRepo.delete({ id: post.id });
+    await reaction.message.delete();
 
     if(reaction.emoji.name == "✅")
-        sendNews(item);
+        sendNews(post);
 }
 
-async function sendNews(item : PostItem) {
-    let outChannel = db.get("feedOutputChannel").value();
-    let oldNews = db.get("postedNewsGuids");
+async function sendNews(item : PostedForumNewsItem) {
+    let channelRepo = getRepository(KnownChannel);
+    let newsPostRepo = getRepository(PostedForumNewsItem);
 
-    let sentMessage = await postNewsItem(outChannel, item);
-    oldNews.set(item.id, {
-                    hash: item.hash,
-                    messageId: sentMessage.id
-                });
-    db.write();
+    let outChannel = await channelRepo.findOne({
+        where: { channelType: NEWS_FEED_CHANNEL }
+    });
+
+    let sentMessage = await postNewsItem(outChannel.channelId, item);
+
+    item.postedMessageId = sentMessage.id;
+    item.verifyMessage = null;
+
+    await newsPostRepo.save(item);
 }
 
 function isVerifyReaction(reaction : MessageReaction, user: User) {
@@ -219,18 +232,18 @@ async function tryFetchMessage(channel: Channel, messageId: string)  {
 }
 
 function shouldVerify() {
-    return verifyChannelID != "";
+    return verifyChannelId != null;
 }
 
-async function postNewsItem(channel: string, item: PostItem) : Promise<Message | null> {
-    let newsMessage = toNewsString(item);
+async function postNewsItem(channel: string, item: PostedForumNewsItem) : Promise<Message | null> {
+    let newsMessage = toNewsString(item.verifyMessage);
     let ch = client.channels.get(channel);
 
     if(!(ch instanceof TextChannel))
         return null;
 
-    if(item.messageId) {
-        let message = await tryFetchMessage(ch, item.messageId);
+    if(item.postedMessageId) {
+        let message = await tryFetchMessage(ch, item.postedMessageId);
         if(message)
             return await message.edit(newsMessage);
         else 
@@ -244,17 +257,17 @@ function markdownify(htmStr: string, link: string) {
     return turndown.turndown(htmStr).replace(/( {2}\n|\n\n){2,}/gm, "\n").replace(link, "");
 }
 
-function toNewsString(item: PostItem) {
+function toNewsString(item: PostVerifyMessage) {
     return `**${item.title}**
-Posted by ${item.creator}
+Posted by ${item.author}
 ${item.link}
 
-${item.contents}`;
+${item.text}`;
 }
 
-function toVerifyString(item: PostItem) {
-    return `[${item.type}]
-Post ID: **${item.id}**
+function toVerifyString(postId: string, item: PostVerifyMessage) {
+    return `[${item.isNew ? "🆕 ADD": "✏️ EDIT"}]
+Post ID: **${postId}**
     
 ${toNewsString(item)}
             
@@ -266,38 +279,52 @@ export default {
         {
             pattern: /^edit (\d+)\s+((.*[\n\r]*)+)$/i,
             action: async (msg, s, match) => {
-                if(msg.channel.id != verifyChannelID)
+                if(msg.channel.id != verifyChannelId)
                     return;
     
                 let id = match[1];
                 let newContents = match[2].trim();
-                
-                if(!db.get("newsCache").has(id).value()) {
+
+                let repo = getRepository(PostedForumNewsItem);
+                let verifyRepo = getRepository(PostVerifyMessage);
+
+                let post = await repo.findOne({
+                    where: { id: id },
+                    relations: [ "verifyMessage" ]
+                });
+
+                if(!post || !post.verifyMessage) {
                     msg.channel.send(`${msg.author.toString()} No unapproved news items with id ${id}!`);
                     return;
                 }
                 
-                let newsCache = db.get("newsCache") as ObjectChain<any>;
-                let item = newsCache.get(id).value() as PostItem;
-    
-                let editMsg = await tryFetchMessage(client.channels.get(verifyChannelID), item.verifyMessageId);
+                let editMsg = await tryFetchMessage(client.channels.get(verifyChannelId), post.verifyMessage.messageId);
     
                 if(!editMsg){
-                    msg.channel.send(`${msg.author.toString()} No verify messafe found for ${id}! This is a bug: report to horse.`);
+                    msg.channel.send(`${msg.author.toString()} No verify message found for ${id}! This is a bug: report to horse.`);
                     return;
                 }
     
-                item.contents = newContents;
+                post.verifyMessage.text = newContents;
                 
-                db.get("newsCache").set(id, item);
-                db.write();
-                await editMsg.edit(toVerifyString(item));
+                await verifyRepo.save(post.verifyMessage);
+                await editMsg.edit(toVerifyString(post.verifyMessage.isNew, post.id, post.verifyMessage));
                 await msg.delete();
             }
         }
     ],
-    onStart: () => {
-        initPendingReactors();
+    onStart: async () => {
+
+        let repo = getRepository(KnownChannel);
+
+        let verifyChannel = await repo.findOne({
+            channelType: NEWS_POST_VERIFY_CHANNEL
+        });
+
+        if(verifyChannel)
+            verifyChannelId = verifyChannel.channelId;
+
+        await initPendingReactors();
         interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);
     }
 } as ICommand;

src/entity/PostVerifyMessage.ts

@@ -0,0 +1,26 @@
+import {Entity, PrimaryGeneratedColumn, Column} from "typeorm";
+
+@Entity()
+export class PostVerifyMessage {
+
+    @PrimaryGeneratedColumn()
+    id: number;
+
+    @Column()
+    messageId: string;
+
+    @Column()
+    author: string;
+
+    @Column()
+    title: string;
+
+    @Column()
+    link: string;
+
+    @Column()
+    text: string;
+
+    @Column()
+    isNew: boolean;
+}

src/entity/PostedForumsNewsItem.ts

@@ -1,4 +1,5 @@
-import { Entity, PrimaryColumn, Column } from "typeorm";
+import { Entity, PrimaryColumn, Column, OneToOne, JoinColumn } from "typeorm";
+import { PostVerifyMessage } from "./PostVerifyMessage";
 
 @Entity()
 export class PostedForumNewsItem {
@@ -12,6 +13,7 @@ export class PostedForumNewsItem {
     @Column({ nullable: true })
     postedMessageId?: string;
 
-    @Column({ nullable: true })
-    verifyMessageId?: string;
+    @OneToOne(type => PostVerifyMessage, { nullable: true })
+    @JoinColumn()
+    verifyMessage?: PostVerifyMessage;
 };

src/main.ts

@@ -27,18 +27,20 @@ async function trigger(actions : BotEvent[], ...params: any[]) {
 let commands : IBotCommand[] = [];
 let msgActions : BotEvent[] = [];
 let indirectMentionActions : BotEvent[] = [];
-let startActions : Array<() => void> = [];
+let startActions : Array<() => void | Promise<void>> = [];
 let directMessageActions : BotEvent[] = [];
 let postActions : BotEvent[] = [];
 
-client.on("ready", () => {
+client.on("ready", async () => {
     console.log("Starting up NoctBot!");
     client.user.setActivity(process.env.NODE_ENV == "dev" ? "Maintenance" : "@NoctBot help", {
         type: "PLAYING"
     });
     for (let i = 0; i < startActions.length; i++) {
         const action = startActions[i];
-        action();
+        let val = action();
+        if(val instanceof Promise)
+            await val;
     }
     console.log("NoctBot is ready!");
 });

src/util.ts

@@ -63,4 +63,6 @@ export async function isAuthorisedAsync(member : GuildMember) {
     if (role)
         return true;
     return false;
-}
+}
+
+export type Dict<TVal> = { [key: string]: TVal };