Implement forums news posting

.env.template

@@ -2,6 +2,8 @@ BOT_TOKEN=
 FORUM_PASS=
 FORUM_API_KEY=
 IGNORE_CHANGED_NEWS=
+GOOGLE_APPLICATION_CREDENTIALS=gcloud_key.json
+GOOGLE_APP_ID=
 
 DB_USERNAME=
 DB_PASSWORD=

.gitignore

@@ -2,6 +2,8 @@ build/
 data/
 lib/
 
+gcloud_key.json
+
 __sapper__/
 .rpt2_cache/
 

bot/package.json

@@ -27,6 +27,7 @@
    "dependencies": {
       "@bbob/html": "^2.5.2",
       "@bbob/preset-html5": "^2.5.2",
+      "@google-cloud/translate": "^4.1.1",
       "@types/dotenv": "^6.1.1",
       "@types/lowdb": "^1.0.9",
       "@types/request-promise-native": "^1.0.16",
@@ -36,6 +37,7 @@
       "axios": "^0.19.0",
       "discord.js": "^11.4.2",
       "dotenv": "^8.0.0",
+      "html2bbcode": "^1.2.6",
       "interval-promise": "^1.2.0",
       "jimp": "^0.5.4",
       "lowdb": "^1.0.0",
@@ -49,7 +51,6 @@
       "rimraf": "^2.6.3",
       "rss-parser": "^3.4.3",
       "sha1": "^1.1.1",
-      "translate-google": "^1.3.5",
       "tsconfig-paths": "^3.8.0",
       "turndown": "^5.0.1",
       "typeorm": "0.2.18",

bot/src/client.ts

@@ -1,7 +1,7 @@
 import { Client } from "discord.js";
 import { XenforoClient } from "./xenforo";
 
-
+export const FORUMS_DOMAIN = "https://custommaid3d2.com";
 export const client = new Client();
-export const forumClient = new XenforoClient("https://custommaid3d2.com/api", process.env.FORUM_API_KEY);
+export const forumClient = new XenforoClient(`${FORUMS_DOMAIN}/api`, process.env.FORUM_API_KEY);
 

bot/src/commands/aggregators/aggregator.ts

@@ -6,7 +6,8 @@ export interface INewsItem {
     title: string | "",
     author: string,
     contents: string,
-    embedColor: number | 0xffffff
+    embedColor: number | 0xffffff,
+    needsTranslation?: boolean;
 }
 
 export interface INewsPostData {

bot/src/commands/aggregators/com3d2_updates.ts

@@ -69,7 +69,8 @@ async function aggregate() {
             title: latestVersionChangelog[1],
             author: "COM3D2 UPDATE",
             contents: text,
-            embedColor: 0xcccccc
+            embedColor: 0xcccccc,
+            needsTranslation: true
         }] as INewsItem[];
     } catch(err) {
         return [];

bot/src/commands/aggregators/kiss_diary.ts

@@ -90,7 +90,8 @@ async function aggregate() {
                 title: title.text,
                 author: "KISS BLOG",
                 contents: contents.innerHTML,
-                embedColor: 0xf4c100
+                embedColor: 0xf4c100,
+                needsTranslation: true
             });
         }
 

bot/src/commands/forums_news_checker.ts

@@ -21,6 +21,8 @@ const reactionCollectors: Dict<ReactionCollector> = {};
 const verifyMessageIdToPost: Dict<PostedForumNewsItem> = {};
 const NEWS_FEED_CHANNEL = "newsFeed";
 
+let botUserId = 0;
+
 const turndown = new TurndownService();
 turndown.addRule("image", {
     filter: "img",
@@ -107,55 +109,11 @@ async function checkFeeds() {
                 continue;
         }
 
-        if (!shouldVerify())
+        if (!shouldVerify() || firstPost.user_id == botUserId)
             await sendNews(itemObj);
         else
             await addVerifyMessage(itemObj);
     }
-
-    // for(let feedEntry of FEEDS) {
-    //     let feed = await parser.parseURL(feedEntry.url);
-    //     if(feed.items.length == 0)
-    //         continue;
-    //     let printableItems = feed.items.sort((a : any, b: any) => a.isoDate.localeCompare(b.isoDate));
-    //     if(printableItems.length > 0) {
-    //         for(let item of printableItems) {
-    //             let itemID = getThreadId(item.guid);
-    //             let contents = null;
-
-    //             try {
-    //                 let res = await request(item.link, {resolveWithFullResponse: true}) as Response;
-    //                 if(res.statusCode != 200) {
-    //                     console.log(`Post ${itemID} could not be loaded because request returned status ${res.statusCode}`);
-    //                     continue;
-    //                 }
-
-    //                 let rootNode = html.parse(res.body, {
-    //                     pre: true,
-    //                     script: false,
-    //                     style: false
-    //                 });
-
-    //                 if(!(rootNode instanceof html.HTMLElement))
-    //                     continue;
-
-    //                 let opDiv = rootNode.querySelector("div.bbWrapper");
-
-    //                 if (!opDiv) {
-    //                     console.log(`No posts found for ${itemID}!`);
-    //                     continue;
-    //                 }
-
-    //                 contents = markdownify(opDiv.outerHTML, item.link);
-    //             } catch(err){
-    //                 console.log(`Failed to get html for item ${itemID} because ${err}`);
-    //                 continue;
-    //             }
-
-
-    //         }
-    //     }
-    // }
 }
 
 async function initPendingReactors() {
@@ -227,6 +185,8 @@ async function collectReaction(reaction: MessageReaction, collector: Collector<s
 
     if (reaction.emoji.name == "✅")
         sendNews(post);
+
+    delete verifyMessageIdToPost[m.id];
 }
 
 async function sendNews(item: PostedForumNewsItem) {
@@ -349,8 +309,13 @@ export default {
             channelType: NEWS_POST_VERIFY_CHANNEL
         });
 
-        if (verifyChannel)
-            verifyChannelId = verifyChannel.channelId;
+        if (!verifyChannel)
+            return;
+        
+        verifyChannelId = verifyChannel.channelId;
+
+        let user = await forumClient.getMe();
+        botUserId = user.user_id;
 
         await initPendingReactors();
         interval(checkFeeds, RSS_UPDATE_INTERVAL_MIN * 60 * 1000);

bot/src/commands/news_aggregator.ts

@@ -1,18 +1,24 @@
 import TurndownService, { Options } from "turndown";
 import interval from "interval-promise";
-import { client, forumClient } from "../client";
+import { client, forumClient, FORUMS_DOMAIN } from "../client";
 import sha1 from "sha1";
 import * as path from "path";
 import * as fs from "fs";
-import translate from "translate-google";
+import { HTML2BBCode } from "html2bbcode";
+import { Dict } from "../util";
 
 import { IAggregator, NewsPostItem, INewsPostData } from "./aggregators/aggregator";
 import { ICommand } from "./command";
-import { RichEmbed, TextChannel, Message, Channel } from "discord.js";
-import { getRepository } from "typeorm";
+import { RichEmbed, TextChannel, Message, Channel, ReactionCollector, MessageReaction, User, Collector } from "discord.js";
+import { getRepository, IsNull, Not } from "typeorm";
 import { KnownChannel } from "@db/entity/KnownChannel";
 import { AggroNewsItem } from "@db/entity/AggroNewsItem";
 
+import { v3beta1 } from "@google-cloud/translate";
+
+const { TranslationServiceClient } = v3beta1;
+const tlClient = new TranslationServiceClient();
+
 const UPDATE_INTERVAL = 5;
 const MAX_PREVIEW_LENGTH = 300; 
 
@@ -22,6 +28,10 @@ const AGGREGATOR_MANAGER_CHANNEL = "aggregatorManager";
 
 const FORUMS_STAGING_ID = 54;
 const FORUMS_NEWS_ID = 49;
+const bbCodeParser = new HTML2BBCode();
+
+const reactionCollectors: Dict<ReactionCollector> = {};
+const verifyMessageIdToPost: Dict<AggroNewsItem> = {};
 
 // TODO: Run BBCode converter instead
 const turndown = new TurndownService();
@@ -55,7 +65,7 @@ async function checkFeeds() {
                 cacheMessageId: null,
                 postedMessageId: null
             } as NewsPostItem;
-            itemObj.contents = markdownify(item.contents);
+
             itemObj.hash = sha1(itemObj.contents);
 
             await addNewsItem(itemObj);
@@ -74,6 +84,12 @@ function clipText(text: string) {
 async function addNewsItem(item: NewsPostItem) {
     let repo = getRepository(AggroNewsItem);
 
+    let ch = client.channels.get(aggregateChannelID);
+
+    if(!(ch instanceof TextChannel))
+        return;
+
+    let isNew = true;
     let newsItem = await repo.findOne({
         where: { feedName: item.feedId, newsId: item.newsId }
     });
@@ -84,6 +100,7 @@ async function addNewsItem(item: NewsPostItem) {
             return;
         else
             await deleteCacheMessage(newsItem.editMessageId);
+        isNew = false;
     } else {
         newsItem = repo.create({
             newsId: item.newsId,
@@ -92,17 +109,40 @@ async function addNewsItem(item: NewsPostItem) {
         });
     }
 
-    let ch = client.channels.get(aggregateChannelID);
+    if(item.needsTranslation)
+        try {
+            let request = {
+                parent: tlClient.locationPath(process.env.GOOGLE_APP_ID, "global"),
+                contents: [ item.title, item.contents ],
+                mimeType: "text/html",
+                sourceLanguageCode: "ja",
+                targetLanguageCode: "en"
+            };
+
+            let [ res ] = await tlClient.translateText(request);
+
+            item.title = res.translations[0].translatedText
+            item.contents = res.translations[1].translatedText;
+        } catch(err) {
+            console.log(`Failed to translate because ${err}`);
+        }
+
+    item.contents = bbCodeParser.feed(item.contents).toString();
+
+    if(!newsItem.forumsEditPostId) {
+        let createResponse = await forumClient.createThread(FORUMS_STAGING_ID, item.title, item.contents);
+        newsItem.forumsEditPostId = createResponse.thread.thread_id;
+    } else {
+        await forumClient.postReply(newsItem.forumsNewsPostId, item.contents);
+    }
     
-    if(!(ch instanceof TextChannel))
-        return;
 
     let msg = await ch.send(new RichEmbed({
         title: item.title,
         url: item.link,
         color: item.embedColor,
         timestamp: new Date(),
-        description: clipText(item.contents),
+        description: `${(isNew ? "**[NEW]**" : "**[EDIT]**")}\n[**Edit on forums**](${FORUMS_DOMAIN}/index.php?threads/.${newsItem.forumsEditPostId}/)`,
         author: {
             name: item.author
         },
@@ -113,9 +153,53 @@ async function addNewsItem(item: NewsPostItem) {
 
     newsItem.editMessageId = msg.id;
 
+    await msg.react("✅");
+    await msg.react("❌");
+
+    let collector = msg.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
+    collector.on("collect", collectReaction)
+    reactionCollectors[msg.id] = collector;
+    verifyMessageIdToPost[msg.id] = newsItem;
+
     await repo.save(newsItem);
 }
 
+function isVerifyReaction(reaction: MessageReaction, user: User) {
+    return (reaction.emoji.name == "✅" || reaction.emoji.name == "❌") && !user.bot && user.id != client.user.id;
+}
+
+async function collectReaction(reaction: MessageReaction, collector: Collector<string, MessageReaction>) {
+    let repo = getRepository(AggroNewsItem);
+
+    let m = reaction.message;
+    collector.stop();
+    delete reactionCollectors[m.id];
+    let post = verifyMessageIdToPost[m.id];
+    
+    if (reaction.emoji.name == "✅") {
+        let res = await forumClient.getThread(post.forumsEditPostId);
+        let forumPost = await forumClient.getPost(res.thread.first_post_id);
+
+        if(!post.forumsNewsPostId) {
+            let newThread = await forumClient.createThread(FORUMS_NEWS_ID, res.thread.title, forumPost.message);
+            post.forumsNewsPostId = newThread.thread.thread_id;
+        } else {
+            let curThread = await forumClient.editThread(post.forumsNewsPostId, {
+                title: res.thread.title
+            });
+
+            await forumClient.editPost(curThread.thread.first_post_id, {
+                message: forumPost.message
+            });
+        }
+    }
+    
+    await forumClient.deleteThread(post.forumsEditPostId);
+    await repo.update({ newsId: post.newsId, feedName: post.feedName }, { editMessageId: null, forumsEditPostId: null, forumsNewsPostId: post.forumsNewsPostId });
+    await reaction.message.delete();
+    delete verifyMessageIdToPost[m.id];
+}
+
 async function deleteCacheMessage(messageId: string) {
     let ch = client.channels.get(aggregateChannelID);
     if(!(ch instanceof TextChannel))
@@ -127,10 +211,12 @@ async function deleteCacheMessage(messageId: string) {
         await msg.delete();
 }
 
-async function tryFetchMessage(channel : TextChannel, messageId: string) {
+async function tryFetchMessage(channel: Channel, messageId: string) {
     try {
+        if (!(channel instanceof TextChannel))
+            return null;
         return await channel.fetchMessage(messageId);
-    }catch(error){
+    } catch (error) {
         return null;
     }
 }
@@ -158,6 +244,30 @@ function initAggregators() {
     }
 }
 
+async function initPendingReactors() {
+    let verifyChannel = client.channels.get(aggregateChannelID);
+
+    let repo = getRepository(AggroNewsItem);
+
+    let pendingVerifyMessages = await repo.find({
+        where: { editMessageId: Not(IsNull()) }
+    });
+
+    for (let msg of pendingVerifyMessages) {
+        let m = await tryFetchMessage(verifyChannel, msg.editMessageId);
+
+        if (!m) {
+            await repo.update({ feedName: msg.feedName, newsId: msg.newsId }, { editMessageId: null });
+            continue;
+        }
+
+        let collector = m.createReactionCollector(isVerifyReaction, { maxEmojis: 1 });
+        collector.on("collect", collectReaction);
+        reactionCollectors[m.id] = collector;
+        verifyMessageIdToPost[m.id] = msg;
+    }
+}
+
 export default {
     onStart : async () => {
         let repo = getRepository(KnownChannel);
@@ -166,9 +276,12 @@ export default {
             where: { channelType: AGGREGATOR_MANAGER_CHANNEL }
         });
 
-        if(ch)
-            aggregateChannelID = ch.channelId;
+        if(!ch)
+            return;
+        
+        aggregateChannelID = ch.channelId;
 
+        await initPendingReactors();
         initAggregators();
         interval(checkFeeds, UPDATE_INTERVAL * 60 * 1000);
     }

bot/src/main.ts

@@ -1,17 +1,7 @@
 // We need some kind of module resolver for @db. We use module-alias.
 require("module-alias/register");
 
-import * as fs from "fs";
-import * as path from "path";
-import { client } from "./client";
-import { ICommand, BotEvent, IBotCommand } from "./commands/command"
-import "reflect-metadata";
-import {createConnection, getConnectionOptions} from "typeorm";
-import { migrate } from "./lowdb_migrator";
-import { documentation } from "./util";
 import dotenv from "dotenv";
-import { DB_ENTITIES } from "@db/entities";
-
 if(process.env.NODE_ENV == "dev") {
     dotenv.config({
         path: "../.env"
@@ -25,6 +15,16 @@ if(process.env.NODE_ENV == "dev") {
     process.env.TYPEORM_DATABASE = process.env.DB_NAME;
 }
 
+import * as fs from "fs";
+import * as path from "path";
+import { client } from "./client";
+import { ICommand, BotEvent, IBotCommand } from "./commands/command"
+import "reflect-metadata";
+import {createConnection, getConnectionOptions} from "typeorm";
+import { documentation } from "./util";
+import { DB_ENTITIES } from "@db/entities";
+
+
 const REACT_PROBABILITY = 0.3;
 
 async function trigger(actions : BotEvent[], ...params: any[]) {

bot/src/typedefs/html2bbcode.d.ts

@@ -0,0 +1,32 @@
+declare module "html2bbcode" {
+
+    export interface BBCodeMapping {
+        section: string;
+        attr?: string;
+        data?: string;
+        newline?: number;
+        extend?: string[];
+        empty?: boolean;
+        ignore?: boolean;
+    }
+
+    export class BBCode {
+        static maps: { [htmlNode: string] : BBCodeMapping };
+        constructor();
+        toString() : string;
+    }
+
+    export interface HTML2BBCodeOptions {
+        imagescale?: boolean;
+        transsize?: boolean;
+        nolist?: boolean;
+        noalign?: boolean;
+        noheadings?: boolean;
+        debug?: boolean;
+    }
+
+    export class HTML2BBCode {
+        constructor(opts?: HTML2BBCodeOptions);
+        feed(html: string) : BBCode;
+    }
+}

bot/src/typedefs/translate-google.d.ts

@@ -1,21 +0,0 @@
-declare module "translate-google" {
-
-    export interface TranslationOptions {
-        from?: string;
-        to?: string;
-    }
-
-    export interface LanguageService {
-        isSupported(desiredLang: string) : boolean;
-        getCode(language: string) : string;
-    }
-
-    export type LanguageData = { [langCode: string] : string } & LanguageService;
-
-    export type TranslationInput = object | string | string[];
-
-    export type TranslateFunction = <T extends TranslationInput>(input: T, opts?: TranslationOptions, domain?: string) => Promise<T>;
-    
-    export const translateFunction : TranslateFunction & { languages: LanguageData };
-    export default translateFunction;
-}

bot/src/xenforo.ts

@@ -38,12 +38,33 @@ export class XenforoClient {
         }
     }
 
+    async getMe() {
+        let { me } = await this.makeRequest<{me: User}>(`me/`, ReqMethod.GET);
+        return me;
+    }
+
+    async postReply(thread_id: number, message: string, attachment_key?: string) {
+        return await this.makeRequest<void>(`posts/`, ReqMethod.POST, {
+            thread_id,
+            message,
+            attachment_key
+        });
+    }
+
+    async editThread(id: number, opts?: EditThreadOptions) {
+        return await this.makeRequest<CreateThreadResponse>(`threads/${id}`, ReqMethod.POST, opts || {});
+    }
+
+    async editPost(id: number, opts?: EditPostOptions) {
+        return await this.makeRequest<EditPostResponse>(`posts/${id}`, ReqMethod.POST, opts || {});
+    }
+
     async getThread(id: number, opts?: GetThreadOptions) {
-        return await this.makeRequest<GetThreadResponse>(`threads/${id}`, ReqMethod.GET, opts);
+        return await this.makeRequest<GetThreadResponse>(`threads/${id}`, ReqMethod.GET, opts || {});
     }
 
     async deleteThread(id: number, opts?: DeleteThreadOptions) {
-        return await this.makeRequest<SuccessResponse>(`threads/${id}`, ReqMethod.DELETE, opts);
+        return await this.makeRequest<SuccessResponse>(`threads/${id}`, ReqMethod.DELETE, opts || {});
     }
 
     async createThread(forumId: number, title: string, message: string, opts?: CreateThreadOptions) {
@@ -86,6 +107,25 @@ interface CreateThreadOptions {
     sticky?: boolean;
     attachment_key?: boolean;
 }
+
+interface EditThreadOptions {
+    prefix_id?: number;
+    title?: string;
+    discussion_open?: boolean;
+    sticky?: boolean;
+    custom_fields?: Dict<string>;
+    add_tags?: any[];
+    remove_tags?: any[];
+}
+
+interface EditPostOptions {
+    message?: string;
+    silent?: boolean;
+    clear_edit?: boolean;
+    author_alert?: boolean;
+    author_alert_reason?: string;
+    attachment_key?: string;
+}
 //#endregion
 
 //#region Response types 
@@ -99,6 +139,8 @@ type SuccessResponse = {
     success: boolean;
 }
 
+type EditPostResponse = SuccessResponse & { post: Post };
+
 type CreateThreadResponse = SuccessResponse & { thread: Thread; };
 
 type GetForumThreadsResponse = {

db/src/entity/AggroNewsItem.ts

@@ -16,8 +16,8 @@ export class AggroNewsItem {
     editMessageId?: string;
 
     @Column({ unique: true, nullable: true })
-    forumsNewsPostId?: string;
+    forumsNewsPostId?: number;
 
     @Column({ unique: true, nullable: true })
-    forumsEditPostId?: string;
+    forumsEditPostId?: number;
 }