noctbot/bot/src/commands/aggregators/kiss_diary.ts

import * as html from "node-html-parser";
import request from "request-promise-native";
import { Response } from "request";
import { INewsItem, IAggregator } from "./aggregator";
import { getRepository } from "typeorm";
import { AggroNewsItem } from "@db/entity/AggroNewsItem";

const urlPattern = /diary\.php\?no=(\d+)/i;
const kissDiaryRoot = "http://www.kisskiss.tv/kiss";
const FEED_NAME = "kisskisstv-diary";

async function aggregate() {
    let repo = getRepository(AggroNewsItem);

    let lastPost = await repo.findOne({
        select: [ "newsId" ],
        where: { feedName: FEED_NAME },
        order: { newsId: "DESC" }
    });

    if(!lastPost)
        lastPost = repo.create({
            newsId: 0
        });
    
    try {
        let mainPageRes = await request(`${kissDiaryRoot}/diary.php`, {resolveWithFullResponse: true}) as Response;
        
        if(mainPageRes.statusCode != 200)
            return [];

        let rootNode = html.parse(mainPageRes.body, {
                pre: true,
                script: false,
                style: false
        });

        if(!(rootNode instanceof html.HTMLElement))
            return;

        let diaryEntries = rootNode.querySelectorAll("div.blog_frame_middle ul.disc li a");

        if(!diaryEntries) {
            console.log("[KISS DIARY] Failed to find listing!");
        }

        let result : INewsItem[] = [];
        let latestEntry = lastPost.newsId;

        for(let a of diaryEntries) {
            let matches = urlPattern.exec(a.rawAttributes.href);
            if(!matches)
                continue;
            
            let id = +matches[1];

            if(id <= lastPost.newsId)
                continue;

            if(id > latestEntry)
                latestEntry = id;

            let diaryLink = `${kissDiaryRoot}/${a.rawAttributes.href}`;
            let res = await request(diaryLink, {resolveWithFullResponse: true}) as Response;
            if(res.statusCode != 200)
                continue;

            let node = html.parse(res.body, {
                pre: true,
                script: false,
                style: false
            });

            if(!(node instanceof html.HTMLElement))
                continue;

            let title = node.querySelector("table.blog_frame_top tr td a");
            let contents = node.querySelector("div.blog_frame_middle");
            let bottomFrame = contents.querySelector("div.blog_data");
            if(bottomFrame) {
                let child = contents.childNodes[0];
                if(child instanceof html.HTMLElement)
                    child.removeChild(bottomFrame);
            }

            result.push({
                newsId: id,
                feedId: FEED_NAME,
                link: diaryLink,
                title: title.text,
                author: "KISS BLOG",
                contents: contents.innerHTML,
                embedColor: 0xf4c100,
                needsTranslation: true
            });
        }

        return result;
    } catch(err) {
        return [];
    }
}

export default {
    aggregate: aggregate
} as IAggregator;