import * as html from "node-html-parser";
import request from "request-promise-native";
import { Response } from "request";
import { INewsItem, IAggregator } from "./aggregator";
import { getRepository } from "typeorm";
import { AggroNewsItem } from "@db/entity/AggroNewsItem";
const urlPattern = /diary\.php\?no=(\d+)/i;
const kissDiaryRoot = "http://www.kisskiss.tv/kiss";
const FEED_NAME = "kisskisstv-diary";
async function aggregate() {
let repo = getRepository(AggroNewsItem);
let lastPost = await repo.findOne({
select: [ "newsId" ],
where: { feedName: FEED_NAME },
order: { newsId: "DESC" }
});
if(!lastPost)
lastPost = repo.create({
newsId: 0
});
try {
let mainPageRes = await request(`${kissDiaryRoot}/diary.php`, {resolveWithFullResponse: true}) as Response;
if(mainPageRes.statusCode != 200)
return [];
let rootNode = html.parse(mainPageRes.body, {
pre: true,
script: false,
style: false
});
if(!(rootNode instanceof html.HTMLElement))
return;
let diaryEntries = rootNode.querySelectorAll("div.blog_frame_middle ul.disc li a");
if(!diaryEntries) {
console.log("[KISS DIARY] Failed to find listing!");
}
let result : INewsItem[] = [];
let latestEntry = lastPost.newsId;
for(let a of diaryEntries) {
let matches = urlPattern.exec(a.rawAttributes.href);
if(!matches)
continue;
let id = +matches[1];
if(id <= lastPost.newsId)
continue;
if(id > latestEntry)
latestEntry = id;
let diaryLink = `${kissDiaryRoot}/${a.rawAttributes.href}`;
let res = await request(diaryLink, {resolveWithFullResponse: true}) as Response;
if(res.statusCode != 200)
continue;
let node = html.parse(res.body, {
pre: true,
script: false,
style: false
});
if(!(node instanceof html.HTMLElement))
continue;
let title = node.querySelector("table.blog_frame_top tr td a");
let contents = node.querySelector("div.blog_frame_middle");
let bottomFrame = contents.querySelector("div.blog_data");
if(bottomFrame) {
let child = contents.childNodes[0];
if(child instanceof html.HTMLElement)
child.removeChild(bottomFrame);
}
result.push({
newsId: id,
feedId: FEED_NAME,
link: diaryLink,
title: title.text,
author: "KISS BLOG",
contents: contents.innerHTML,
embedColor: 0xf4c100,
needsTranslation: true
});
}
return result;
} catch(err) {
return [];
}
}
export default {
aggregate: aggregate
} as IAggregator;