Move face detection to a separate container

.gitignore

@@ -1,51 +1,162 @@
-build/
-data/
-lib/
-
-gcloud_key.json
-
-__sapper__/
-.rpt2_cache/
-
-*.sqlite
-*.env
-
-# ---> Node
-# Logs
-logs
-*.log
-npm-debug.log*
-
-# Runtime data
-pids
-*.pid
-*.seed
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-
-# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (http://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directory
-# https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git
-node_modules
-
-token.js
-package-lock.json
-db.json
-imagestats.csv
-clarifai_keys.js
-.env
-db_old.json
-db_migrated.json
-*.old
+build/
+data/
+lib/
+venv/
+
+gcloud_key.json
+
+__sapper__/
+.rpt2_cache/
+
+*.sqlite
+*.env
+
+# ---> Node
+# Logs
+logs
+*.log
+npm-debug.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directory
+# https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git
+node_modules
+
+token.js
+package-lock.json
+db.json
+imagestats.csv
+clarifai_keys.js
+.env
+db_old.json
+db_migrated.json
+*.old
+
+# Created by https://www.gitignore.io/api/python
+# Edit at https://www.gitignore.io/?templates=python
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# End of https://www.gitignore.io/api/python

.vscode/launch.json

@@ -5,6 +5,13 @@
     "version": "0.2.0",
     "configurations": [
         {
+            "name": "Python: Facedetect",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/facedetect/app/main.py",
+            "console": "integratedTerminal"
+        },
+        {
             "type": "node",
             "request": "launch",
             "name": "Launch NoctBot",
@@ -34,7 +41,17 @@
     "compounds": [
         {
             "name": "Bot+WebServer",
-            "configurations": ["Launch NoctBot", "Launch WebServer"]
+            "configurations": [
+                "Launch NoctBot",
+                "Launch WebServer"
+            ]
+        },
+        {
+            "name": "Bot+Facedetect",
+            "configurations": [
+                "Launch NoctBot",
+                "Python: Facedetect"
+            ]
         }
     ]
 }

.vscode/settings.json

@@ -1,5 +1,9 @@
 {
     "editor.codeActionsOnSave": {
         "source.fixAll.eslint": true
-    }
+    },
+    "python.pythonPath": "/usr/bin/python3",
+    "python.linting.pylintArgs": [
+        "--generate-members"
+    ]
 }

Makefile

@@ -18,13 +18,13 @@ build:
 	$(dc) build
 
 start_env: build
-	$(dc) up db adminer
+	$(dc) up db adminer facedetect
 
 start: build
-	$(dc) up db adminer noctbot web
+	$(dc) up db adminer noctbot web facedetect
 
 start_bot: build
-	$(dc) up db adminer noctbot
+	$(dc) up db adminer noctbot facedetect
 
 start_web: build
 	$(dc) up db adminer web

bot/Dockerfile

@@ -1,86 +1,17 @@
-FROM alpine AS opencv-builder
-
-ARG OPENCV_VERSION
-
-RUN apk --no-cache add python make g++ cmake linux-headers
-
-RUN mkdir opencv && \
-    cd opencv && \
-    wget https://github.com/opencv/opencv/archive/${OPENCV_VERSION}.zip --no-check-certificate -O opencv-${OPENCV_VERSION}.zip && \
-    unzip opencv-${OPENCV_VERSION}.zip && \
-    mkdir opencv-${OPENCV_VERSION}/build && \
-    cd opencv-${OPENCV_VERSION}/build && \
-    cmake_flags="-D CMAKE_BUILD_TYPE=RELEASE \
-    -D BUILD_EXAMPLES=OFF \
-	-D BUILD_DOCS=OFF \
-	-D BUILD_TESTS=OFF \
-	-D BUILD_PERF_TESTS=OFF \
-	-D BUILD_JAVA=OFF \
-	-D BUILD_opencv_apps=OFF \
-	-D BUILD_opencv_aruco=OFF \
-	-D BUILD_opencv_bgsegm=OFF \
-	-D BUILD_opencv_bioinspired=OFF \
-	-D BUILD_opencv_ccalib=OFF \
-	-D BUILD_opencv_datasets=OFF \
-	-D BUILD_opencv_dnn_objdetect=OFF \
-	-D BUILD_opencv_dpm=OFF \
-	-D BUILD_opencv_fuzzy=OFF \
-	-D BUILD_opencv_hfs=OFF \
-	-D BUILD_opencv_java_bindings_generator=OFF \
-	-D BUILD_opencv_js=OFF \
-    -D BUILD_opencv_img_hash=OFF \
-    -D BUILD_opencv_line_descriptor=OFF \
-    -D BUILD_opencv_optflow=OFF \
-    -D BUILD_opencv_phase_unwrapping=OFF \
-	-D BUILD_opencv_python3=OFF \
-	-D BUILD_opencv_python_bindings_generator=OFF \
-	-D BUILD_opencv_reg=OFF \
-	-D BUILD_opencv_rgbd=OFF \
-	-D BUILD_opencv_saliency=OFF \
-	-D BUILD_opencv_shape=OFF \
-	-D BUILD_opencv_stereo=OFF \
-	-D BUILD_opencv_stitching=OFF \
-	-D BUILD_opencv_structured_light=OFF \
-	-D BUILD_opencv_superres=OFF \
-	-D BUILD_opencv_surface_matching=OFF \
-	-D BUILD_opencv_ts=OFF \
-	-D BUILD_opencv_xobjdetect=OFF \
-	-D BUILD_opencv_xphoto=OFF" && \
-    echo $cmake_flags && \
-    cmake $cmake_flags .. && \
-    make -j $(nproc) && \
-    make install
-
-FROM node:10-alpine AS builder
-
-ARG OPENCV_VERSION
-ENV OPENCV4NODEJS_DISABLE_AUTOBUILD=1
+FROM node:14-alpine
 
-RUN apk --no-cache add python make g++
+RUN apk --no-cache add make
 
-COPY --from=opencv-builder /opencv/opencv-${OPENCV_VERSION}/build/lib/libopencv* /usr/local/lib/
-COPY --from=opencv-builder /usr/local/include/opencv2 /usr/local/include/opencv2
-COPY --from=opencv-builder /usr/local/share/OpenCV /usr/local/share/OpenCV
-COPY ./bot/package.json ./
+WORKDIR /app/bot
+COPY ./bot/package.json .
 RUN npm install
 
-WORKDIR /shared/
+WORKDIR /app/shared
 COPY ./shared/package.json .
 RUN npm install
 
-FROM node:14-alpine
-
-ARG OPENCV_VERSION
 WORKDIR /app
 
-RUN apk --no-cache add make
-
-COPY --from=builder node_modules bot/node_modules
-COPY --from=builder /shared/node_modules shared/node_modules
-COPY --from=opencv-builder /opencv/opencv-${OPENCV_VERSION}/build/lib/libopencv* /usr/local/lib/
-COPY --from=opencv-builder /usr/local/include/opencv2 /usr/local/include/opencv2
-COPY --from=opencv-builder /usr/local/share/OpenCV /usr/local/share/OpenCV
-
 COPY ./bot bot
 COPY ./shared shared
 COPY ./Makefile Makefile

bot/package.json

@@ -37,7 +37,6 @@
       "@types/turndown": "^5.0.0",
       "@types/ws": "^7.2.4",
       "@types/xml2js": "^0.4.5",
-      "axios": "^0.19.2",
       "cheerio": "^1.0.0-rc.3",
       "discord.js": "^12.2.0",
       "dotenv": "^8.2.0",
@@ -50,9 +49,9 @@
       "koa-router": "^8.0.8",
       "lowdb": "^1.0.0",
       "module-alias": "^2.2.2",
+      "needle": "^2.5.0",
       "node-schedule": "^1.3.2",
       "nodemailer": "^6.4.8",
-      "opencv4nodejs": "^5.6.0",
       "pg": "^8.2.1",
       "reflect-metadata": "^0.1.13",
       "request": "^2.88.2",
@@ -71,6 +70,7 @@
       "yaml": "^1.10.0"
    },
    "devDependencies": {
+      "@types/needle": "^2.0.4",
       "@types/node": "^14.0.5",
       "@types/nodemailer": "^6.4.0",
       "@typescript-eslint/eslint-plugin": "^3.0.2",

bot/src/commands/facemorph.ts

@@ -1,8 +1,6 @@
 import { isValidImage } from "../util";
 import Jimp from "jimp";
 import { client } from "../client";
-import * as cv from "opencv4nodejs";
-import * as path from "path";
 import request from "request-promise-native";
 import { Message, MessageAttachment } from "discord.js";
 import { getRepository } from "typeorm";
@@ -10,30 +8,58 @@ import { FaceCaptionMessage, FaceCaptionType } from "@shared/db/entity/FaceCapti
 import { KnownChannel } from "@shared/db/entity/KnownChannel";
 import { CommandSet, Action, ActionType, Command } from "src/model/command";
 import { logger } from "src/logging";
+import { Response } from "request";
+import needle from "needle";
 
 const EMOTE_GUILD = "505333548694241281";
 
-const animeCascade = new cv.CascadeClassifier(path.resolve(process.cwd(), "animu.xml"));
-const faceCascade = new cv.CascadeClassifier(cv.HAAR_FRONTALFACE_ALT2);
-
 const CAPTION_IMG_SIZE = 300;
 const CAPTION_PROBABILITY = 0.33;
 
-type ImageProcessor = (faces: cv.Rect[], data: Buffer) => Promise<Jimp>;
+interface Rect {
+    x: number,
+    y: number,
+    w: number,
+    h: number
+}
+
+interface ErrorInfo {
+    ok: boolean,
+    error: string;
+}
+
+interface FaceData {
+    ok: boolean,
+    animeFaces: Rect[],
+    normalFaces: Rect[]
+}
+
+type FaceDetectionResponse = FaceData | ErrorInfo;
+
+function isError(resp: FaceDetectionResponse): resp is ErrorInfo {
+    return !resp.ok;
+}
+
+type ImageProcessor = (faces: Rect[], data: Buffer) => Promise<Jimp>;
 const CAPTION_OFFSET = 5;
 
 @CommandSet
 export class Facemorph {
 
-    intersects(r1: cv.Rect, r2: cv.Rect): boolean {
+    squareFace(rect: Rect): Rect {
+        const s = Math.min(rect.w, rect.h);
+        return {...rect, w: s, h: s};
+    }
+
+    intersects(r1: Rect, r2: Rect): boolean {
         return (
-            r1.x <= r2.x + r2.width &&
-            r1.x + r1.width >= r2.x &&
-            (r1.y <= r2.y + r2.height && r1.y + r1.height >= r2.y)
+            r1.x <= r2.x + r2.w &&
+            r1.x + r1.w >= r2.x &&
+            (r1.y <= r2.y + r2.h && r1.y + r1.h >= r2.y)
         );
     }
 
-    morphFaces = async (faces: cv.Rect[], data: Buffer): Promise<Jimp> => {
+    morphFaces = async (faces: Rect[], data: Buffer): Promise<Jimp> => {
         const padoru = Math.random() <= this.getPadoruChance();
         let jimpImage = await Jimp.read(data);
         const emoteGuild = client.bot.guilds.resolve(EMOTE_GUILD);
@@ -54,8 +80,8 @@ export class Facemorph {
             ];
 
         for (const rect of faces) {
-            const dx = rect.x + rect.width / 2;
-            const dy = rect.y + rect.height / 2;
+            const dx = rect.x + rect.w / 2;
+            const dy = rect.y + rect.h / 2;
             const emojiKey = emojiKeys[Math.floor(Math.random() * emojiKeys.length)];
             const emoji = client.bot.emojis.resolve(emojiKey);
             if (!emoji)
@@ -65,7 +91,7 @@ export class Facemorph {
             let eh = emojiImage.getHeight();
 
             const CONSTANT_SCALE = 1.1;
-            const scaleFactor = (Math.max(rect.width, rect.height) / Math.min(ew, eh)) * CONSTANT_SCALE;
+            const scaleFactor = (Math.max(rect.w, rect.h) / Math.min(ew, eh)) * CONSTANT_SCALE;
             ew *= scaleFactor;
             eh *= scaleFactor;
 
@@ -89,24 +115,24 @@ export class Facemorph {
         return caption[0];
     }
 
-    captionFace = async (faces: cv.Rect[], data: Buffer): Promise<Jimp> => {
+    captionFace = async (faces: Rect[], data: Buffer): Promise<Jimp> => {
         const padoru = Math.random() <= this.getPadoruChance();
         const face = faces[Math.floor(Math.random() * faces.length)];
-        const squaredFace = await face.toSquareAsync();
+        const squaredFace = this.squareFace(face);
         const targetSize = CAPTION_IMG_SIZE;
         const img = await Jimp.read(data);
 
-        let tempImg = await Jimp.create(squaredFace.width, squaredFace.height);
+        let tempImg = await Jimp.create(squaredFace.w, squaredFace.h);
         tempImg = await tempImg.blit(
             img,
             0,
             0,
             squaredFace.x,
             squaredFace.y,
-            squaredFace.width,
-            squaredFace.height
+            squaredFace.w,
+            squaredFace.h
         );
-        tempImg = await tempImg.scale(targetSize / squaredFace.width);
+        tempImg = await tempImg.scale(targetSize / squaredFace.w);
 
         const font = await Jimp.loadFont(padoru ? Jimp.FONT_SANS_16_WHITE : Jimp.FONT_SANS_16_BLACK);
         let text = "";
@@ -145,27 +171,36 @@ export class Facemorph {
 
     async processFaceSwap(message: Message, attachmentUrl: string, processor?: ImageProcessor, failMessage?: string, successMessage?: string): Promise<void> {
         const data = await request(attachmentUrl, { encoding: null }) as Buffer;
-        const im = await cv.imdecodeAsync(data, cv.IMREAD_COLOR);
-        const gray = await im.cvtColorAsync(cv.COLOR_BGR2GRAY);
-        const normGray = await gray.equalizeHistAsync();
-        const animeFaces = await animeCascade.detectMultiScaleAsync(
-            normGray,
-            1.1,
-            5,
-            0,
-            new cv.Size(24, 24)
-        );
-        const normalFaces = await faceCascade.detectMultiScaleAsync(gray);
 
-        if (animeFaces.objects.length == 0 && normalFaces.objects.length == 0) {
+        const result = await needle("post", `http://${process.env.FACEDETECT_URL}/process`, {
+            img_data: {
+                buffer: data,
+                filename: "image.png",
+                content_type: "application/octet-stream"
+            }
+        }, { multipart: true });
+
+        if(result.statusCode != 200) {
+            logger.error("Face detection failed! Got response %s", result.statusCode);
+            return;
+        }
+
+        const faceRects = result.body as FaceDetectionResponse;
+
+        if (isError(faceRects)) {
+            logger.error("Face detection failed! Got response %s", result.statusCode);
+            return;
+        }
+
+        if (faceRects.animeFaces.length == 0 && faceRects.normalFaces.length == 0) {
             if (failMessage) message.channel.send(failMessage);
             return;
         }
 
-        const faces = [...normalFaces.objects, ...animeFaces.objects];
+        const faces = [...faceRects.normalFaces, ...faceRects.animeFaces];
 
-        let normalCount = normalFaces.objects.length;
-        let animeCount = animeFaces.objects.length;
+        let normalCount = faceRects.normalFaces.length;
+        let animeCount = faceRects.animeFaces.length;
 
         for (let i = 0; i < normalCount; i++) {
             const rNormal = faces[i];
@@ -176,8 +211,8 @@ export class Facemorph {
                 const rAnime = faces[j];
 
                 if (this.intersects(rAnime, rNormal)) {
-                    const animeA = rAnime.width * rAnime.height;
-                    const faceA = rNormal.width * rNormal.height;
+                    const animeA = rAnime.w * rAnime.h;
+                    const faceA = rNormal.w * rNormal.h;
 
                     if (animeA > faceA) {
                         faces.splice(i, 1);

bot/src/main.ts

@@ -10,6 +10,7 @@ if (process.env.NODE_ENV == "dev") {
         path: "../db.env"
     });
     process.env.TYPEORM_HOST = "localhost";
+    process.env.FACEDETECT_URL = "localhost:8081";
     process.env.TYPEORM_USERNAME = process.env.DB_USERNAME;
     process.env.TYPEORM_PASSWORD = process.env.DB_PASSWORD;
     process.env.TYPEORM_DATABASE = process.env.DB_NAME;

docker-compose.dev.yml

@@ -3,4 +3,8 @@ version: '3.7'
 services:
   db:
     ports:
-      - 5432:5432
+      - 5432:5432
+
+  facedetect:
+    ports:
+      - 8081:80

docker-compose.yml

@@ -6,11 +6,10 @@ services:
     build:
       context: ./
       dockerfile: ./bot/Dockerfile
-      args:
-        OPENCV_VERSION: 3.4
     restart: always
     depends_on:
       - db
+      - facedetect
     env_file: 
       - .env
       - db.env
@@ -24,6 +23,14 @@ services:
       TYPEORM_USERNAME: ${DB_USERNAME}
       TYPEORM_PASSWORD: ${DB_PASSWORD}
       TYPEORM_DATABASE: ${DB_NAME}
+      FACEDETECT_URL: facedetect
+
+  facedetect:
+    image: facedetect
+    build:
+      context: ./
+      dockerfile: ./facedetect/Dockerfile
+    restart: always
 
   web:
     image: noctbot_web

facedetect/.dockerignore

@@ -0,0 +1,2 @@
+venv
+__pycache__

facedetect/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.8-slim
+
+RUN apt-get update && \
+    apt-get -y install libglib2.0; \
+    apt-get clean
+
+WORKDIR /app
+COPY ./facedetect/requirements.txt .
+
+RUN pip install -r requirements.txt
+
+COPY ./facedetect .
+
+EXPOSE 80
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] 

facedetect/app/main.py

@@ -0,0 +1,37 @@
+import uvicorn
+from fastapi import FastAPI, File, UploadFile
+import cv2
+import numpy as np
+import os
+
+app = FastAPI()
+
+face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + "haarcascade_frontalface_alt2.xml")
+anime_cascade = cv2.CascadeClassifier(os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "animu.xml"))
+
+@app.post("/process")
+def process(img_data: bytes = File(None)):
+    try:
+        im = cv2.imdecode(np.frombuffer(img_data, np.uint8), cv2.IMREAD_COLOR)
+        gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
+        normGray = cv2.equalizeHist(gray)
+
+        animeFaces = anime_cascade.detectMultiScale(normGray,
+                                                1.1,
+                                                5,
+                                                0,
+                                                (24, 24))
+        normalFaces = face_cascade.detectMultiScale(normGray)
+        return {
+            "ok": True,
+            "animeFaces": [{"x": x.item(), "y": y.item(), "w": w.item(), "h": h.item()} for (x, y, w, h) in animeFaces],
+            "normalFaces": [{"x": x.item(), "y": y.item(), "w": w.item(), "h": h.item()} for (x, y, w, h) in normalFaces]
+        }
+    except Exception as err:
+        return {
+            "ok": False,
+            "error": str(err)
+        }
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8081)

facedetect/requirements.txt

@@ -0,0 +1,4 @@
+opencv-python-headless==4.2.0.34
+python-multipart==0.0.5
+fastapi==0.55.1
+uvicorn==0.11.5