Update aggregators to use sqlite

src/commands/aggregators/com3d2_updates.ts

@@ -1,11 +1,13 @@
 import html, { HTMLElement } from "node-html-parser";
 import request from "request-promise-native";
-import { db } from "../../db";
 import { Response } from "request";
 import { IAggregator, INewsItem } from "./aggregator";
+import { getRepository } from "typeorm";
+import { AggroNewsItem } from "../../entity/AggroNewsItem";
 
 const updatePage = "http://com3d2.jp/update/";
 const changeLogPattern = /\[\s*([^\s\]]+)\s*\]\s*((・.*)\s+)+/gim;
+const FEED_NAME = "com3d2-jp-updates";
 
 function getVersionNumber(verStr: string) {
     let verPart = verStr.replace(/[\.\s]/g, "");
@@ -15,7 +17,13 @@ function getVersionNumber(verStr: string) {
 }
 
 async function aggregate() {
-    let lastVersion = db.get("lastCOMJPVersion").value() as number;
+    let repo = getRepository(AggroNewsItem);
+    
+    let lastPost = await repo.findOne({
+        select: [ "newsId" ],
+        where: { feedName: FEED_NAME },
+        order: { newsId: "DESC" }
+    });
     
     try {
         let mainPageRes = await request(updatePage, {resolveWithFullResponse: true}) as Response;
@@ -46,13 +54,12 @@ async function aggregate() {
         let version = getVersionNumber(latestVersionChangelog[1]);
         let text = latestVersionChangelog[0];
 
-        if(version <= lastVersion)
+        if(version <= lastPost.newsId)
             return [];
         
-        db.set("lastCOMJPVersion", version).write();
-
         return [{
-            id: `com3d2-jp-updates-${version}`,
+            feedId: FEED_NAME,
+            newsId: version,
             link: updatePage,
             title: latestVersionChangelog[1],
             author: "COM3D2 UPDATE",

src/commands/aggregators/com3d2_world.ts

@@ -1,14 +1,22 @@
 import html, { HTMLElement } from "node-html-parser";
 import request from "request-promise-native";
-import { db } from "../../db";
 import { Response } from "request";
 import { INewsItem, IAggregator } from "./aggregator";
+import { getRepository } from "typeorm";
+import { AggroNewsItem } from "../../entity/AggroNewsItem";
 
 const kissDiaryRoot = "https://com3d2.world/r18/notices.php";
+const FEED_NAME = "com3d2-world-notices";
 
 async function aggregate() {
-    let lastDiary = db.get("latestCom3D2WorldDiaryEntry").value() as number;
+    let repo = getRepository(AggroNewsItem);
     
+    let lastPost = await repo.findOne({
+        select: [ "newsId" ],
+        where: { feedName: FEED_NAME },
+        order: { newsId: "DESC" }
+    });
+
     try {
         let mainPageRes = await request(kissDiaryRoot, {resolveWithFullResponse: true}) as Response;
         
@@ -31,7 +39,7 @@ async function aggregate() {
         }
 
         let result : INewsItem[] = [];
-        let latestEntry = lastDiary;
+        let latestEntry = lastPost.newsId;
 
         for(let a of diaryEntries) {
             if(!a.rawAttributes.id)
@@ -39,7 +47,7 @@ async function aggregate() {
             
             let id = +a.rawAttributes.id;
 
-            if(id <= lastDiary)
+            if(id <= lastPost.newsId)
                 continue;
 
             if(id > latestEntry)
@@ -63,7 +71,8 @@ async function aggregate() {
             let contents = node.querySelectorAll("div.frame div")[1];
 
             result.push({
-                id: `com3d2-world-notices-${id}`,
+                newsId: id,
+                feedId: FEED_NAME,
                 link: diaryLink,
                 title: title.text,
                 author: "com3d2.world",
@@ -71,8 +80,6 @@ async function aggregate() {
                 embedColor: 0xa39869
             });
         }
-
-        db.set("latestCom3D2WorldDiaryEntry", latestEntry).write();
         return result;
     } catch(err) {
         return [];

src/commands/aggregators/kiss_diary.ts

@@ -1,14 +1,22 @@
 import html, { HTMLElement } from "node-html-parser";
 import request from "request-promise-native";
-import { db } from "../../db";
 import { Response } from "request";
 import { INewsItem, IAggregator } from "./aggregator";
+import { getRepository } from "typeorm";
+import { AggroNewsItem } from "../../entity/AggroNewsItem";
 
 const urlPattern = /diary\.php\?no=(\d+)/i;
 const kissDiaryRoot = "http://www.kisskiss.tv/kiss";
+const FEED_NAME = "kisskisstv-diary";
 
 async function aggregate() {
-    let lastDiary = db.get("latestKissDiaryEntry").value() as number;
+    let repo = getRepository(AggroNewsItem);
+
+    let lastPost = await repo.findOne({
+        select: [ "newsId" ],
+        where: { feedName: FEED_NAME },
+        order: { newsId: "DESC" }
+    });
     
     try {
         let mainPageRes = await request(`${kissDiaryRoot}/diary.php`, {resolveWithFullResponse: true}) as Response;
@@ -32,7 +40,7 @@ async function aggregate() {
         }
 
         let result : INewsItem[] = [];
-        let latestEntry = lastDiary;
+        let latestEntry = lastPost.newsId;
 
         for(let a of diaryEntries) {
             let matches = urlPattern.exec(a.rawAttributes.href);
@@ -41,7 +49,7 @@ async function aggregate() {
             
             let id = +matches[1];
 
-            if(id <= lastDiary)
+            if(id <= lastPost.newsId)
                 continue;
 
             if(id > latestEntry)
@@ -71,7 +79,8 @@ async function aggregate() {
             }
 
             result.push({
-                id: `kisskisstv-diary-${id}`,
+                newsId: id,
+                feedId: FEED_NAME,
                 link: diaryLink,
                 title: title.text,
                 author: "KISS BLOG",
@@ -80,7 +89,6 @@ async function aggregate() {
             });
         }
 
-        db.set("latestKissDiaryEntry", latestEntry).write();
         return result;
     } catch(err) {
         return [];