Implement migrations for lowdb

.gitignore

@@ -38,3 +38,4 @@ db.json
 imagestats.csv
 clarifai_keys.js
 .env
+db_old.json

package.json

@@ -5,9 +5,9 @@
    "main": "main.js",
    "scripts": {
       "test": "echo \"Error: no test specified\" && exit 1",
-      "build": "tsc",
-      "start": "ts-node src/index.ts",
-      "typeorm": "ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js"
+      "build": "rimraf build && tsc",
+      "start": "rimraf build && ts-node src/index.ts",
+      "typeorm": "rimraf build && ts-node -r tsconfig-paths/register ./node_modules/typeorm/cli.js"
    },
    "repository": {
       "type": "git",
@@ -38,6 +38,7 @@
       "reflect-metadata": "^0.1.10",
       "request": "^2.88.0",
       "request-promise-native": "^1.0.5",
+      "rimraf": "^2.6.3",
       "rss-parser": "^3.4.3",
       "sha1": "^1.1.1",
       "sqlite3": "^4.0.3",

src/entity/FaceMorphProbability.ts

@@ -1,11 +0,0 @@
-import {Entity, PrimaryColumn, Column} from "typeorm";
-
-@Entity()
-export class FaceMorphProbability {
-
-    @PrimaryColumn()
-    channelID: string;
-
-    @Column({ type: "decimal" })
-    probability: number;
-}

src/entity/Guide.ts

@@ -1,4 +1,4 @@
-import {Entity, PrimaryColumn, Column} from "typeorm";
+import {Entity, Column, PrimaryGeneratedColumn, ManyToMany, JoinTable} from "typeorm";
 
 export enum GuideType {
     GUIDE = "guide",
@@ -9,12 +9,32 @@ export enum GuideType {
 @Entity()
 export class Guide {
     
-    @PrimaryColumn()
-    keywords: string;
+    @PrimaryGeneratedColumn()
+    id: number;
+
+    @ManyToMany(type => GuideKeyword, keyword => keyword.relatedGuides)
+    @JoinTable()
+    keywords: GuideKeyword[];
 
     @Column()
     displayName: string;
 
     @Column()
     content: string;
+
+    @Column({ type: "varchar" })
+    type: GuideType;
+}
+
+@Entity()
+export class GuideKeyword {
+
+    @PrimaryGeneratedColumn()
+    id: number;
+
+    @Column()
+    keyword: string;
+
+    @ManyToMany(type => Guide, guide => guide.keywords)
+    relatedGuides: Guide[];
 }

src/entity/KnownChannel.ts

@@ -6,9 +6,12 @@ export class KnownChannel {
     @PrimaryGeneratedColumn()
     id: number;
 
-    @Column()
-    channelName: string;
+    @Column({ nullable: true })
+    channelType?: string;
 
     @Column()
     channelId: string;
+
+    @Column({ type: "decimal", default: 0.0 })
+    faceMorphProbability: number;
 }

src/entity/KnownUser.ts

@@ -8,7 +8,7 @@ export class KnownUser {
     @PrimaryColumn()
     userID: string;
 
-    @Column()
+    @Column({ default: false })
     canModerate: boolean;
 
     @Column({ type: "varchar", default: ReactionType.NONE })

src/lowdb_migrator.ts

@@ -0,0 +1,257 @@
+import { existsSync, readFileSync, renameSync } from "fs";
+import { ReactionType, ReactionEmote } from "./entity/ReactionEmote";
+import { getRepository, Repository } from "typeorm";
+import { KnownUser, User, UserRole } from "./entity/KnownUser";
+import { Dictionary } from "lodash";
+import { Guide, GuideKeyword, GuideType } from "./entity/Guide";
+import { Quote } from "./entity/Quote";
+import { MessageReaction } from "./entity/MessageReaction";
+import { FaceCaptionMessage, FaceCaptionType } from "./entity/FaceCaptionMessage";
+import { PostedForumNewsItem } from "./entity/PostedForumsNewsItem";
+import { KnownChannel } from "./entity/KnownChannel";
+import { DeadChatReply } from "./entity/DeadChatReply";
+
+type EmoteTable = { [reactionType: string]: string[] };
+
+interface IGuide {
+    name: string;
+    displayName: string;
+    content: string;
+}
+
+interface IEditors {
+    roles: string[];
+    users: string[];
+}
+
+interface INewsItem {
+    hash: string;
+    messageId: string;
+}
+
+interface IQuote {
+    author: string;
+    message: string;
+}
+
+interface IFaceCaptionTable {
+    pre: string[];
+    post: string[];
+}
+
+type NewsTable = { [id: string] : INewsItem | boolean};
+type MessageReactionsTable = { [message: string] : string };
+type FaceEditProbabilityTable = { [channel: string] : number };
+
+interface IOldDatabase {
+    emotes: EmoteTable;
+    reactableMentionedUsers: string[];
+    specialUsers: string[];
+    bigUsers: string[];
+    dedUsers: string[];
+    editors: IEditors;
+    memes: IGuide[];
+    miscs: IGuide[];
+    guides: IGuide[];
+    quotes: IQuote[];
+    messageReactions: MessageReactionsTable;
+    faceCaptions: IFaceCaptionTable;
+    postedNewsGuids: NewsTable;
+    faceEditChannels: FaceEditProbabilityTable;
+    deadChatReplies: string[];
+    newsPostVerifyChannel: string;
+    feedOutputChannel: string;
+    aggregateChannel: string;
+    latestKissDiaryEntry: number;
+    latestCom3D2WorldDiaryEntry: number;
+    lastCOMJPVersion: number;
+}
+
+async function migrateEmotes(db: IOldDatabase) {
+    let repo = getRepository(ReactionEmote);
+
+    for (const emoteType in db.emotes) {
+        if(!Object.values(ReactionType).includes(emoteType)) {
+            console.log(`WARN: emote type ${emoteType} is not a predefined emote type!`)
+            continue;
+        }
+
+        await repo.save(db.emotes[emoteType].map(id => repo.create({
+            reactionId: id,
+            type: emoteType as ReactionType
+        })));
+    }
+}
+
+async function migrateUsers(db: IOldDatabase) {
+    let userRepo = getRepository(User);
+    let roleRepo = getRepository(UserRole);
+
+    let users : Dictionary<User> = {};
+    let roles : Dictionary<UserRole> = {};
+
+    let iterator = <T extends KnownUser>(targetDict : Dictionary<T>, repo: Repository<T>, ids: string[], action: (u: T) => void) => {
+        for(let userId of ids) {
+            let u : T;
+
+            if(userId in users)
+                u = targetDict[userId];
+            else{
+                u = targetDict[userId] = repo.create();
+                u.userID = userId;
+            }
+
+            action(u);
+        }
+    };
+
+    iterator(users, userRepo, db.reactableMentionedUsers, u => u.mentionReactionType = ReactionType.ANGERY);
+    iterator(users, userRepo, db.specialUsers, u => u.replyReactionType = ReactionType.HUG);
+    iterator(users, userRepo, db.bigUsers, u => u.replyReactionType = ReactionType.BIG);
+    iterator(users, userRepo, db.dedUsers, u => u.replyReactionType = ReactionType.DED);
+
+    iterator(users, userRepo, db.editors.users, u => u.canModerate = true);
+    iterator(roles, roleRepo, db.editors.roles, r => r.canModerate = true);
+
+    await userRepo.save(Object.values(users));
+    await roleRepo.save(Object.values(roles));
+}
+
+async function migrateGuides(db : IOldDatabase) {
+    let guideRepo = getRepository(Guide);
+    let keywordsRepo = getRepository(GuideKeyword);
+
+    let keywords : Dictionary<GuideKeyword> = {};
+    let dbGuides : Guide[] = [];
+
+    let process = async (guides: IGuide[], type: GuideType) => {
+        for(let guide of guides){
+            if(!guide.displayName)
+                continue;
+
+            let guideKeywords = guide.name.split(" ").map(s => s.trim().toLowerCase());
+            let keywordsObjects : GuideKeyword[] = [];
+            
+            for(let keyword of guideKeywords) {
+                let keywordObj : GuideKeyword;
+    
+                if(keyword in keywords)
+                    keywordObj = keywords[keyword];
+                else
+                    keywordObj = keywords[keyword] = await keywordsRepo.save(keywordsRepo.create({ keyword: keyword }));
+                keywordsObjects.push(keywordObj);
+            }
+    
+            let guideObj = guideRepo.create({
+                keywords: keywordsObjects,
+                type: type,
+                displayName: guide.displayName,
+                content: guide.content
+            });
+    
+            dbGuides.push(guideObj);
+        }
+    };
+
+    await process(db.memes, GuideType.MEME);
+    await process(db.guides, GuideType.GUIDE);
+    await process(db.miscs, GuideType.MISC);
+
+    // await keywordsRepo.save(Object.values(keywords));
+    await guideRepo.save(dbGuides);
+}
+
+async function migrateQuotes(db: IOldDatabase) {
+    let repo = getRepository(Quote);
+
+    await repo.save(db.quotes.map(q => repo.create({
+        author: q.author,
+        message: q.message
+    })));
+}
+
+async function migrateMessageReactions(db: IOldDatabase) {
+    let repo = getRepository(MessageReaction);
+
+    await repo.save(Object.entries(db.messageReactions).map(([message, emoteId]) => repo.create({
+        message: message,
+        reactionEmoteId: emoteId
+    })));
+}
+
+async function migrateFaceCaptions(db: IOldDatabase) {
+    let repo = getRepository(FaceCaptionMessage);
+
+    await repo.save([
+        ...db.faceCaptions.pre.map(s => repo.create( {
+            message: s,
+            type: FaceCaptionType.PREFIX
+        })),
+        ...db.faceCaptions.post.map(s => repo.create( {
+            message: s,
+            type: FaceCaptionType.POSTFIX
+        }))
+    ]);
+}
+
+async function migrateNews(db: IOldDatabase) {
+    let repo = getRepository(PostedForumNewsItem);
+
+    await repo.save(Object.entries(db.postedNewsGuids).filter(([id, item]) => typeof item != "boolean").map(([id, item]) => repo.create({
+        hash: (item as INewsItem).hash,
+        postedMessageId: (item as INewsItem).messageId,
+        id: id
+    })));
+}
+
+async function migrateChannels(db: IOldDatabase) {
+    let repo = getRepository(KnownChannel);
+
+    await repo.save([
+        repo.create({
+            channelId: db.newsPostVerifyChannel,
+            channelType: "newsPostVerify"            
+        }),
+        repo.create({
+            channelId: db.aggregateChannel,
+            channelType: "aggregatorManager"
+        }),
+        repo.create({
+            channelId: db.feedOutputChannel,
+            channelType: "newsFeed"
+        }),
+        ...Object.entries(db.faceEditChannels).map(([channelId, prob]) => repo.create({
+            channelId: channelId,
+            faceMorphProbability: prob
+        }))
+    ]);
+}
+
+async function migrateDeadMessages(db: IOldDatabase) {
+    let repo = getRepository(DeadChatReply);
+
+    await repo.save(db.deadChatReplies.map(r => repo.create({
+        message: r
+    })));
+}
+
+export async function migrate() {
+    if(!existsSync("./db.json"))
+        return;
+
+    let json = readFileSync("./db.json", { encoding: "utf-8" });
+
+    let db = JSON.parse(json) as IOldDatabase;
+
+    await migrateEmotes(db);
+    await migrateUsers(db);
+    await migrateGuides(db);
+    await migrateQuotes(db);
+    await migrateMessageReactions(db);
+    await migrateFaceCaptions(db);
+    await migrateNews(db);
+    await migrateChannels(db);
+    await migrateDeadMessages(db);
+
+    renameSync("./db.json", "./db_migrated.json");
+}

src/main.ts

@@ -6,6 +6,7 @@ import { shouldShowMaintenanceMessage, documentation } from "./util";
 import { ICommand, BotEvent, IBotCommand } from "./commands/command"
 import "reflect-metadata";
 import {createConnection} from "typeorm";
+import { migrate } from "./lowdb_migrator";
 
 dotenv.config();
 const REACT_PROBABILITY = 0.3;
@@ -97,6 +98,7 @@ client.on("messageReactionAdd", (r, u) => {
 
 async function main() {
     await createConnection();
+    await migrate();
 
     let commandsPath = path.resolve(path.dirname(module.filename), "commands");
     let files = fs.readdirSync(commandsPath);