Merge branch 'cdn'

.env.example

@@ -0,0 +1,3 @@
+STORAGE_LOCATION=files/
+STORAGE_PROVIDER=file
+PORT=3003

.gitignore

@@ -0,0 +1,6 @@
+.vscode/
+node_modules/
+.DS_Store
+.env
+dist/
+files/

Dockerfile

@@ -0,0 +1,7 @@
+FROM node:lts-alpine
+WORKDIR /usr/src/fosscord-cdn
+COPY package.json .
+RUN npm install
+COPY . .
+EXPOSE 3003
+CMD ["node", "dist/"]

README.md

@@ -0,0 +1,89 @@
+# Fosscord-CDN
+CDN for Fosscord
+
+## Run localy:
+```
+npm i
+node dist/
+```
+
+## Endpoints:
+### POST `/attachments/<filename>`
+```
+Content-Type: form-data
+
+attachment: File (binary-data)
+```
+##### Returns:
+```
+{
+    "success": boolean,             // true
+    "message": string,              // "attachment uploaded"
+    "id": snowflake,                // "794183329158135808"
+    "filename": string              // "lakdoiauej.png"
+}
+```
+### GET `/attachments/<id>/<filename>`
+```
+requests image from database with given <id> and <filename>
+```
+##### Returns:
+```
+Content-Type: image/<imageType(png,img,gif)>
+Image
+```
+### DELETE `/attachments/<id>/<filename>`
+```
+deletes database entry
+```
+##### Returns:
+```
+Content-Type: application/json
+
+{
+    "success": true,
+    "message": "attachment deleted"
+}
+```
+
+<hr>
+
+_(endpoints for crawler):_
+### POST `/external` 
+
+```
+requests crawling of `og:`metadata and the download of the `og:image` property
+--------
+Content-Type: application/json
+
+body:
+{"url": URL}                  // "https://discord.com"
+```
+##### Returns:
+```
+Content-Type: application/json
+
+{
+    "id": string,             // "aHR0cHM6Ly9kaXNjb3JkLmNvbQ=="
+    "ogTitle": string,        // "Discord | Your Place to Talk and Hang Out"
+    "ogDescription": string,  // "Discord is the easiest way to talk over voice, video, and text. Talk, chat, hang out, and stay close with your friends and communities."
+    "cachedImage": string,    // "/external/aHR0cHM6Ly9kaXNjb3JkLmNvbQ==/discord.png"
+    "ogUrl": string,          // "https://discord.com/"
+    "ogType": string          // "website"
+}
+```
+### GET `/external/<id>/<filename>`
+- requests cached crawled image 
+```
+url-params:
+    :id                       // aHR0cHM6Ly9kaXNjb3JkLmNvbQ==
+    :filename                 // discord.png
+```
+```
+/external/aHR0cHM6Ly9kaXNjb3JkLmNvbQ==/discord.png
+```
+##### Returns:
+```
+Content-Type: image/<imageType(png,img,gif)>
+Image
+```

package.json

@@ -0,0 +1,53 @@
+{
+	"name": "@fosscord/cdn",
+	"version": "1.0.0",
+	"description": "cdn for discord clone",
+	"main": "dist/index.js",
+	"types": "dist/index.d.ts",
+	"scripts": {
+		"test": "echo \"Error: no test specified\" && exit 1",
+		"build": "tsc -b .",
+		"start": "npm run build && node dist/start.js"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/discord-open-source/discord-cdn.git"
+	},
+	"keywords": [],
+	"author": "",
+	"license": "ISC",
+	"bugs": {
+		"url": "https://github.com/discord-open-source/discord-cdn/issues"
+	},
+	"homepage": "https://github.com/discord-open-source/discord-cdn#readme",
+	"dependencies": {
+		"@fosscord/server-util": "^1.3.42",
+		"body-parser": "^1.19.0",
+		"btoa": "^1.2.1",
+		"cheerio": "^1.0.0-rc.5",
+		"dotenv": "^10.0.0",
+		"exif-be-gone": "^1.2.0",
+		"express": "^4.17.1",
+		"express-async-errors": "^3.1.1",
+		"file-type": "^16.5.0",
+		"fs-extra": "^10.0.0",
+		"image-size": "^1.0.0",
+		"lambert-db": "^1.2.3",
+		"lambert-server": "^1.2.8",
+		"missing-native-js-functions": "^1.0.8",
+		"multer": "^1.4.2",
+		"node-fetch": "^2.6.1",
+		"uuid": "^8.3.2"
+	},
+	"devDependencies": {
+		"@types/body-parser": "^1.19.0",
+		"@types/btoa": "^1.2.3",
+		"@types/dotenv": "^8.2.0",
+		"@types/express": "^4.17.12",
+		"@types/fs-extra": "^9.0.12",
+		"@types/multer": "^1.4.7",
+		"@types/node": "^14.17.0",
+		"@types/node-fetch": "^2.5.7",
+		"@types/uuid": "^8.3.0"
+	}
+}

src/Server.ts

@@ -0,0 +1,68 @@
+import { Server, ServerOptions } from "lambert-server";
+import { Config, db } from "@fosscord/server-util";
+import path from "path";
+import avatarsRoute from "./routes/avatars";
+
+export interface CDNServerOptions extends ServerOptions {}
+
+export class CDNServer extends Server {
+	public options: CDNServerOptions;
+
+	constructor(options?: Partial<CDNServerOptions>) {
+		super(options);
+	}
+
+	async start() {
+		console.log("[Database] connecting ...");
+		// @ts-ignore
+		await (db as Promise<Connection>);
+		await Config.init();
+		console.log("[Database] connected");
+		this.app.use((req, res, next) => {
+			res.set("Access-Control-Allow-Origin", "*");
+			// TODO: use better CSP policy
+			res.set(
+				"Content-security-policy",
+				"default-src *  data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';"
+			);
+			res.set("Access-Control-Allow-Headers", req.header("Access-Control-Request-Headers") || "*");
+			res.set("Access-Control-Allow-Methods", req.header("Access-Control-Request-Methods") || "*");
+			next();
+		});
+
+		await this.registerRoutes(path.join(__dirname, "routes/"));
+
+		this.app.use("/icons/", avatarsRoute);
+		this.log("info", "[Server] Route /icons registered");
+
+		this.app.use("/emojis/", avatarsRoute);
+		this.log("info", "[Server] Route /emojis registered");
+
+		this.app.use("/stickers/", avatarsRoute);
+		this.log("info", "[Server] Route /stickers registered");
+
+		this.app.use("/banners/", avatarsRoute);
+		this.log("info", "[Server] Route /banners registered");
+		
+		this.app.use("/splashes/", avatarsRoute);
+		this.log("info", "[Server] Route /splashes registered");
+
+		this.app.use("/app-icons/", avatarsRoute);
+		this.log("info", "[Server] Route /app-icons registered");
+
+		this.app.use("/app-assets/", avatarsRoute);
+		this.log("info", "[Server] Route /app-assets registered");
+
+		this.app.use("/discover-splashes/", avatarsRoute);
+		this.log("info", "[Server] Route /discover-splashes registered");
+
+		this.app.use("/team-icons/", avatarsRoute);
+		this.log("info", "[Server] Route /team-icons registered");
+
+		return super.start();
+	}
+
+	async stop() {
+		return super.stop();
+	}
+}

src/index.ts

@@ -0,0 +1 @@
+export * from "./Server";

src/routes/attachments.ts

@@ -0,0 +1,73 @@
+import { Router, Response, Request } from "express";
+import { Config, Snowflake } from "@fosscord/server-util";
+import { storage } from "../util/Storage";
+import FileType from "file-type";
+import { HTTPError } from "lambert-server";
+import { multer } from "../util/multer";
+import imageSize from "image-size";
+
+const router = Router();
+
+router.post("/:channel_id", multer.single("file"), async (req: Request, res: Response) => {
+	if (req.headers.signature !== Config.get().security.requestSignature)
+		throw new HTTPError("Invalid request signature");
+	if (!req.file) return;
+
+	const { buffer, mimetype, size, originalname, fieldname } = req.file;
+	const { channel_id } = req.params;
+	const filename = originalname.replaceAll(" ", "_").replace(/[^a-zA-Z0-9._]+/g, "");
+	const id = Snowflake.generate();
+	const path = `attachments/${channel_id}/${id}/${filename}`;
+
+	const endpoint = Config.get()?.cdn.endpoint || "http://localhost:3003";
+
+	await storage.set(path, buffer);
+	var width;
+	var height;
+	if (mimetype.includes("image")) {
+		const dimensions = imageSize(buffer);
+		if (dimensions) {
+			width = dimensions.width;
+			height = dimensions.height;
+		}
+	}
+
+	const file = {
+		id,
+		content_type: mimetype,
+		filename: filename,
+		size,
+		url: `${endpoint}/${path}`,
+		width,
+		height,
+	};
+
+	return res.json(file);
+});
+
+router.get("/:channel_id/:id/:filename", async (req: Request, res: Response) => {
+	const { channel_id, id, filename } = req.params;
+
+	const file = await storage.get(`attachments/${channel_id}/${id}/${filename}`);
+	if (!file) throw new HTTPError("File not found");
+	const type = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", type?.mime);
+	res.set("Cache-Control", "public, max-age=31536000");
+
+	return res.send(file);
+});
+
+router.delete("/:channel_id/:id/:filename", async (req: Request, res: Response) => {
+	if (req.headers.signature !== Config.get().security.requestSignature)
+		throw new HTTPError("Invalid request signature");
+
+	const { channel_id, id, filename } = req.params;
+	const path = `attachments/${channel_id}/${id}/${filename}`;
+
+	await storage.delete(path);
+
+	return res.send({ success: true });
+});
+
+export default router;

src/routes/avatars.ts

@@ -0,0 +1,72 @@
+import { Router, Response, Request } from "express";
+import { Config, Snowflake } from "@fosscord/server-util";
+import { storage } from "../util/Storage";
+import FileType from "file-type";
+import { HTTPError } from "lambert-server";
+import { multer } from "../util/multer";
+import crypto from "crypto";
+
+// TODO: check premium and animated pfp are allowed in the config
+// TODO: generate different sizes of icon
+// TODO: generate different image types of icon
+// TODO: delete old icons
+
+const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"];
+const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"];
+const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES];
+
+const router = Router();
+
+router.post("/:user_id", multer.single("file"), async (req: Request, res: Response) => {
+	if (req.headers.signature !== Config.get().security.requestSignature)
+		throw new HTTPError("Invalid request signature");
+	if (!req.file) throw new HTTPError("Missing file");
+	const { buffer, mimetype, size, originalname, fieldname } = req.file;
+	const { user_id } = req.params;
+
+	var hash = crypto.createHash("md5").update(Snowflake.generate()).digest("hex");
+
+	const type = await FileType.fromBuffer(buffer);
+	if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) throw new HTTPError("Invalid file type");
+	if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash
+
+	const path = `avatars/${user_id}/${hash}`;
+	const endpoint = Config.get().cdn.endpoint || "http://localhost:3003";
+
+	await storage.set(path, buffer);
+
+	return res.json({
+		id: hash,
+		content_type: type.mime,
+		size,
+		url: `${endpoint}${req.baseUrl}/${user_id}/${hash}`,
+	});
+});
+
+router.get("/:user_id/:hash", async (req: Request, res: Response) => {
+	var { user_id, hash } = req.params;
+	hash = hash.split(".")[0]; // remove .file extension
+	const path = `avatars/${user_id}/${hash}`;
+
+	const file = await storage.get(path);
+	if (!file) throw new HTTPError("not found", 404);
+	const type = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", type?.mime);
+	res.set("Cache-Control", "public, max-age=31536000");
+
+	return res.send(file);
+});
+
+router.delete("/:user_id/:id", async (req: Request, res: Response) => {
+	if (req.headers.signature !== Config.get().security.requestSignature)
+		throw new HTTPError("Invalid request signature");
+	const { user_id, id } = req.params;
+	const path = `avatars/${user_id}/${id}`;
+
+	await storage.delete(path);
+
+	return res.send({ success: true });
+});
+
+export default router;

src/routes/external.ts

@@ -0,0 +1,65 @@
+// @ts-nocheck
+import bodyParser from "body-parser";
+import { Router, Response, Request } from "express";
+import fetch from "node-fetch";
+import crypto from "crypto";
+import { HTTPError } from "lambert-server";
+import { Snowflake } from "@fosscord/server-util";
+import { storage } from "../util/Storage";
+
+const router = Router();
+
+type crawled = {
+	id: string;
+	ogTitle: string;
+	ogType: string;
+	ogDescription: string;
+	ogUrl: string;
+	cachedImage: string;
+};
+
+const DEFAULT_FETCH_OPTIONS: any = {
+	redirect: "follow",
+	follow: 1,
+	headers: {
+		"user-agent": "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)",
+	},
+	size: 1024 * 1024 * 8,
+	compress: true,
+	method: "GET",
+};
+
+router.post("/", bodyParser.json(), async (req: Request, res: Response) => {
+	if (req.headers.signature !== Config.get().security.requestSignature)
+		throw new HTTPError("Invalid request signature");
+	if (!req.body) throw new HTTPError("Invalid Body");
+	const { url } = req.body;
+	if (!url || typeof url !== "string") throw new HTTPError("Invalid url");
+
+	const id = Snowflake.generate();
+
+	try {
+		const response = await fetch(ogImage, DEFAULT_FETCH_OPTIONS);
+		const buffer = await response.buffer();
+
+		await storage.set(`/external/${id}`, buffer);
+
+		res.send({ id });
+	} catch (error) {
+		throw new HTTPError("Couldn't fetch website");
+	}
+});
+
+router.get("/:id/", async (req: Request, res: Response) => {
+	const { id } = req.params;
+
+	const file = await storage.get(`/external/${id}`);
+	if (!file) throw new HTTPError("File not found");
+	const result = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", result?.mime);
+
+	return res.send(file);
+});
+
+export default router;

src/routes/ping.ts

@@ -0,0 +1,9 @@
+import { Router, Response, Request } from "express";
+
+const router = Router();
+
+router.get("/", (req: Request, res: Response) => {
+	res.send("pong");
+});
+
+export default router;

src/start.ts

@@ -0,0 +1,27 @@
+import path from "path";
+import dotenv from "dotenv";
+import fse from "fs-extra";
+dotenv.config();
+
+if (!process.env.STORAGE_PROVIDER) process.env.STORAGE_PROVIDER = "file";
+// TODO:nodejs path.join trailing slash windows compatible
+if (process.env.STORAGE_PROVIDER === "file") {
+	if (process.env.STORAGE_LOCATION) {
+		if (!process.env.STORAGE_LOCATION.startsWith("/")) {
+			process.env.STORAGE_LOCATION = path.join(__dirname, "..", process.env.STORAGE_LOCATION, "/");
+		}
+	} else {
+		process.env.STORAGE_LOCATION = path.join(__dirname, "..", "files", "/");
+	}
+	fse.ensureDirSync(process.env.STORAGE_LOCATION);
+}
+
+import { CDNServer } from "./Server";
+
+const server = new CDNServer({ port: Number(process.env.PORT) || 3003 });
+server
+	.start()
+	.then(() => {
+		console.log("[Server] started on :" + server.options.port);
+	})
+	.catch((e) => console.error("[Server] Error starting: ", e));

src/util/FileStorage.ts

@@ -0,0 +1,40 @@
+import { Storage } from "./Storage";
+import fs from "fs";
+import fse from "fs-extra";
+import { join, relative, dirname } from "path";
+import "missing-native-js-functions";
+import { Readable } from "stream";
+import ExifTransformer = require("exif-be-gone");
+
+function getPath(path: string) {
+	// STORAGE_LOCATION has a default value in start.ts
+	const root = process.env.STORAGE_LOCATION || "../";
+	var filename = join(root, path);
+
+	if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) throw new Error("invalid path");
+	return filename;
+}
+
+export class FileStorage implements Storage {
+	async get(path: string): Promise<Buffer | null> {
+		try {
+			return fs.readFileSync(getPath(path));
+		} catch (error) {
+			return null;
+		}
+	}
+
+	async set(path: string, value: any) {
+		path = getPath(path);
+		fse.ensureDirSync(dirname(path));
+
+		value = Readable.from(value);
+		const cleaned_file = fs.createWriteStream(path);
+
+		return value.pipe(new ExifTransformer()).pipe(cleaned_file);
+	}
+
+	async delete(path: string) {
+		fs.unlinkSync(getPath(path));
+	}
+}

src/util/Storage.ts

@@ -0,0 +1,15 @@
+import { FileStorage } from "./FileStorage";
+
+export interface Storage {
+	set(path: string, data: Buffer): Promise<void>;
+	get(path: string): Promise<Buffer | null>;
+	delete(path: string): Promise<void>;
+}
+
+var storage: Storage;
+
+if (process.env.STORAGE_PROVIDER === "file") {
+	storage = new FileStorage();
+}
+
+export { storage };

src/util/multer.ts

@@ -0,0 +1,10 @@
+import multerConfig from "multer";
+
+export const multer = multerConfig({
+	storage: multerConfig.memoryStorage(),
+	limits: {
+		fields: 10,
+		files: 10,
+		fileSize: 1024 * 1024 * 100, // 100 mb
+	},
+});

tsconfig.json

@@ -0,0 +1,70 @@
+{
+	"include": ["src/**/*.ts"],
+	"compilerOptions": {
+		/* Visit https://aka.ms/tsconfig.json to read more about this file */
+
+		/* Basic Options */
+		// "incremental": true,                   /* Enable incremental compilation */
+		"target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
+		"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
+		"lib": ["ES2015", "dom"] /* Specify library files to be included in the compilation. */,
+		"allowJs": true /* Allow javascript files to be compiled. */,
+		"checkJs": true /* Report errors in .js files. */,
+		// "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
+		"declaration": true /* Generates corresponding '.d.ts' file. */,
+		"declarationMap": false /* Generates a sourcemap for each corresponding '.d.ts' file. */,
+		"inlineSourceMap": true /* Emit a single file with source maps instead of having a separate file. */,
+		// "sourceMap": true /* Generates corresponding '.map' file. */,
+		// "outFile": "./",                       /* Concatenate and emit output to single file. */
+		"outDir": "./dist/" /* Redirect output structure to the directory. */,
+		"rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
+		// "composite": true,                     /* Enable project compilation */
+		// "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
+		// "removeComments": true,                /* Do not emit comments to output. */
+		// "noEmit": true,                        /* Do not emit outputs. */
+		// "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
+		// "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
+		// "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
+
+		/* Strict Type-Checking Options */
+		"strict": true /* Enable all strict type-checking options. */,
+		"noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */,
+		// "strictNullChecks": true,              /* Enable strict null checks. */
+		// "strictFunctionTypes": true,           /* Enable strict checking of function types. */
+		// "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
+		"strictPropertyInitialization": false /* Enable strict checking of property initialization in classes. */,
+		// "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
+		"alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */,
+
+		/* Additional Checks */
+		// "noUnusedLocals": true,                /* Report errors on unused locals. */
+		// "noUnusedParameters": true,            /* Report errors on unused parameters. */
+		// "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
+		// "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
+
+		/* Module Resolution Options */
+		// "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
+		// "baseUrl": "./",                       /* Base directory to resolve non-absolute module names. */
+		// "paths": {},                           /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
+		// "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
+		// "typeRoots": [],                       /* List of folders to include type definitions from. */
+		"types": ["node"] /* Type declaration files to be included in compilation. */,
+		// "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
+		"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
+		// "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
+		// "allowUmdGlobalAccess": true,          /* Allow accessing UMD globals from modules. */
+
+		/* Source Map Options */
+		// "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
+		// "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
+		// "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
+
+		/* Experimental Options */
+		// "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
+		// "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
+
+		/* Advanced Options */
+		"skipLibCheck": true /* Skip type checking of declaration files. */,
+		"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
+	}
+}