Dekescordes/spacebar
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'cdn'

Browse Source
This commit is contained in:
Flam3rboy 2021-08-12 20:16:17 +02:00
commit 87c7cf33d0
17 changed files with 4036 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.env.example Normal file
View File

@ -0,0 +1,3 @@
STORAGE_LOCATION=files/
STORAGE_PROVIDER=file
PORT=3003

6
.gitignore vendored Normal file
View File

@ -0,0 +1,6 @@
.vscode/
node_modules/
.DS_Store
.env
dist/
files/

7
Dockerfile Normal file
View File

@ -0,0 +1,7 @@
FROM node:lts-alpine
WORKDIR /usr/src/fosscord-cdn
COPY package.json .
RUN npm install
COPY . .
EXPOSE 3003
CMD ["node", "dist/"]

89
README.md Normal file
View File

@ -0,0 +1,89 @@
# Fosscord-CDN
CDN for Fosscord
## Run localy:
```
npm i
node dist/
```
## Endpoints:
### POST `/attachments/<filename>`
```
Content-Type: form-data
attachment: File (binary-data)
```
##### Returns:
```
{
"success": boolean, // true
"message": string, // "attachment uploaded"
"id": snowflake, // "794183329158135808"
"filename": string // "lakdoiauej.png"
}
```
### GET `/attachments/<id>/<filename>`
```
requests image from database with given <id> and <filename>
```
##### Returns:
```
Content-Type: image/<imageType(png,img,gif)>
Image
```
### DELETE `/attachments/<id>/<filename>`
```
deletes database entry
```
##### Returns:
```
Content-Type: application/json
{
"success": true,
"message": "attachment deleted"
}
```
<hr>
_(endpoints for crawler):_
### POST `/external`
```
requests crawling of `og:`metadata and the download of the `og:image` property
--------
Content-Type: application/json
body:
{"url": URL} // "https://discord.com"
```
##### Returns:
```
Content-Type: application/json
{
"id": string, // "aHR0cHM6Ly9kaXNjb3JkLmNvbQ=="
"ogTitle": string, // "Discord | Your Place to Talk and Hang Out"
"ogDescription": string, // "Discord is the easiest way to talk over voice, video, and text. Talk, chat, hang out, and stay close with your friends and communities."
"cachedImage": string, // "/external/aHR0cHM6Ly9kaXNjb3JkLmNvbQ==/discord.png"
"ogUrl": string, // "https://discord.com/"
"ogType": string // "website"
}
```
### GET `/external/<id>/<filename>`
- requests cached crawled image
```
url-params:
:id // aHR0cHM6Ly9kaXNjb3JkLmNvbQ==
:filename // discord.png
```
```
/external/aHR0cHM6Ly9kaXNjb3JkLmNvbQ==/discord.png
```
##### Returns:
```
Content-Type: image/<imageType(png,img,gif)>
Image
```

3428
package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

53
package.json Normal file
View File

@ -0,0 +1,53 @@
{
"name": "@fosscord/cdn",
"version": "1.0.0",
"description": "cdn for discord clone",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"build": "tsc -b .",
"start": "npm run build && node dist/start.js"
},
"repository": {
"type": "git",
"url": "git+https://github.com/discord-open-source/discord-cdn.git"
},
"keywords": [],
"author": "",
"license": "ISC",
"bugs": {
"url": "https://github.com/discord-open-source/discord-cdn/issues"
},
"homepage": "https://github.com/discord-open-source/discord-cdn#readme",
"dependencies": {
"@fosscord/server-util": "^1.3.42",
"body-parser": "^1.19.0",
"btoa": "^1.2.1",
"cheerio": "^1.0.0-rc.5",
"dotenv": "^10.0.0",
"exif-be-gone": "^1.2.0",
"express": "^4.17.1",
"express-async-errors": "^3.1.1",
"file-type": "^16.5.0",
"fs-extra": "^10.0.0",
"image-size": "^1.0.0",
"lambert-db": "^1.2.3",
"lambert-server": "^1.2.8",
"missing-native-js-functions": "^1.0.8",
"multer": "^1.4.2",
"node-fetch": "^2.6.1",
"uuid": "^8.3.2"
},
"devDependencies": {
"@types/body-parser": "^1.19.0",
"@types/btoa": "^1.2.3",
"@types/dotenv": "^8.2.0",
"@types/express": "^4.17.12",
"@types/fs-extra": "^9.0.12",
"@types/multer": "^1.4.7",
"@types/node": "^14.17.0",
"@types/node-fetch": "^2.5.7",
"@types/uuid": "^8.3.0"
}
}

68
src/Server.ts Normal file
View File

@ -0,0 +1,68 @@
import { Server, ServerOptions } from "lambert-server";
import { Config, db } from "@fosscord/server-util";
import path from "path";
import avatarsRoute from "./routes/avatars";
export interface CDNServerOptions extends ServerOptions {}
export class CDNServer extends Server {
public options: CDNServerOptions;
constructor(options?: Partial<CDNServerOptions>) {
super(options);
}
async start() {
console.log("[Database] connecting ...");
// @ts-ignore
await (db as Promise<Connection>);
await Config.init();
console.log("[Database] connected");
this.app.use((req, res, next) => {
res.set("Access-Control-Allow-Origin", "*");
// TODO: use better CSP policy
res.set(
"Content-security-policy",
"default-src * data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';"
);
res.set("Access-Control-Allow-Headers", req.header("Access-Control-Request-Headers") || "*");
res.set("Access-Control-Allow-Methods", req.header("Access-Control-Request-Methods") || "*");
next();
});
await this.registerRoutes(path.join(__dirname, "routes/"));
this.app.use("/icons/", avatarsRoute);
this.log("info", "[Server] Route /icons registered");
this.app.use("/emojis/", avatarsRoute);
this.log("info", "[Server] Route /emojis registered");
this.app.use("/stickers/", avatarsRoute);
this.log("info", "[Server] Route /stickers registered");
this.app.use("/banners/", avatarsRoute);
this.log("info", "[Server] Route /banners registered");
this.app.use("/splashes/", avatarsRoute);
this.log("info", "[Server] Route /splashes registered");
this.app.use("/app-icons/", avatarsRoute);
this.log("info", "[Server] Route /app-icons registered");
this.app.use("/app-assets/", avatarsRoute);
this.log("info", "[Server] Route /app-assets registered");
this.app.use("/discover-splashes/", avatarsRoute);
this.log("info", "[Server] Route /discover-splashes registered");
this.app.use("/team-icons/", avatarsRoute);
this.log("info", "[Server] Route /team-icons registered");
return super.start();
}
async stop() {
return super.stop();
}
}

1
src/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./Server";

73
src/routes/attachments.ts Normal file
View File

@ -0,0 +1,73 @@
import { Router, Response, Request } from "express";
import { Config, Snowflake } from "@fosscord/server-util";
import { storage } from "../util/Storage";
import FileType from "file-type";
import { HTTPError } from "lambert-server";
import { multer } from "../util/multer";
import imageSize from "image-size";
const router = Router();
router.post("/:channel_id", multer.single("file"), async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
if (!req.file) return;
const { buffer, mimetype, size, originalname, fieldname } = req.file;
const { channel_id } = req.params;
const filename = originalname.replaceAll(" ", "_").replace(/[^a-zA-Z0-9._]+/g, "");
const id = Snowflake.generate();
const path = `attachments/${channel_id}/${id}/${filename}`;
const endpoint = Config.get()?.cdn.endpoint || "http://localhost:3003";
await storage.set(path, buffer);
var width;
var height;
if (mimetype.includes("image")) {
const dimensions = imageSize(buffer);
if (dimensions) {
width = dimensions.width;
height = dimensions.height;
}
}
const file = {
id,
content_type: mimetype,
filename: filename,
size,
url: `${endpoint}/${path}`,
width,
height,
};
return res.json(file);
});
router.get("/:channel_id/:id/:filename", async (req: Request, res: Response) => {
const { channel_id, id, filename } = req.params;
const file = await storage.get(`attachments/${channel_id}/${id}/${filename}`);
if (!file) throw new HTTPError("File not found");
const type = await FileType.fromBuffer(file);
res.set("Content-Type", type?.mime);
res.set("Cache-Control", "public, max-age=31536000");
return res.send(file);
});
router.delete("/:channel_id/:id/:filename", async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
const { channel_id, id, filename } = req.params;
const path = `attachments/${channel_id}/${id}/${filename}`;
await storage.delete(path);
return res.send({ success: true });
});
export default router;

72
src/routes/avatars.ts Normal file
View File

@ -0,0 +1,72 @@
import { Router, Response, Request } from "express";
import { Config, Snowflake } from "@fosscord/server-util";
import { storage } from "../util/Storage";
import FileType from "file-type";
import { HTTPError } from "lambert-server";
import { multer } from "../util/multer";
import crypto from "crypto";
// TODO: check premium and animated pfp are allowed in the config
// TODO: generate different sizes of icon
// TODO: generate different image types of icon
// TODO: delete old icons
const ANIMATED_MIME_TYPES = ["image/apng", "image/gif", "image/gifv"];
const STATIC_MIME_TYPES = ["image/png", "image/jpeg", "image/webp", "image/svg+xml", "image/svg"];
const ALLOWED_MIME_TYPES = [...ANIMATED_MIME_TYPES, ...STATIC_MIME_TYPES];
const router = Router();
router.post("/:user_id", multer.single("file"), async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
if (!req.file) throw new HTTPError("Missing file");
const { buffer, mimetype, size, originalname, fieldname } = req.file;
const { user_id } = req.params;
var hash = crypto.createHash("md5").update(Snowflake.generate()).digest("hex");
const type = await FileType.fromBuffer(buffer);
if (!type || !ALLOWED_MIME_TYPES.includes(type.mime)) throw new HTTPError("Invalid file type");
if (ANIMATED_MIME_TYPES.includes(type.mime)) hash = `a_${hash}`; // animated icons have a_ infront of the hash
const path = `avatars/${user_id}/${hash}`;
const endpoint = Config.get().cdn.endpoint || "http://localhost:3003";
await storage.set(path, buffer);
return res.json({
id: hash,
content_type: type.mime,
size,
url: `${endpoint}${req.baseUrl}/${user_id}/${hash}`,
});
});
router.get("/:user_id/:hash", async (req: Request, res: Response) => {
var { user_id, hash } = req.params;
hash = hash.split(".")[0]; // remove .file extension
const path = `avatars/${user_id}/${hash}`;
const file = await storage.get(path);
if (!file) throw new HTTPError("not found", 404);
const type = await FileType.fromBuffer(file);
res.set("Content-Type", type?.mime);
res.set("Cache-Control", "public, max-age=31536000");
return res.send(file);
});
router.delete("/:user_id/:id", async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
const { user_id, id } = req.params;
const path = `avatars/${user_id}/${id}`;
await storage.delete(path);
return res.send({ success: true });
});
export default router;

65
src/routes/external.ts Normal file
View File

@ -0,0 +1,65 @@
// @ts-nocheck
import bodyParser from "body-parser";
import { Router, Response, Request } from "express";
import fetch from "node-fetch";
import crypto from "crypto";
import { HTTPError } from "lambert-server";
import { Snowflake } from "@fosscord/server-util";
import { storage } from "../util/Storage";
const router = Router();
type crawled = {
id: string;
ogTitle: string;
ogType: string;
ogDescription: string;
ogUrl: string;
cachedImage: string;
};
const DEFAULT_FETCH_OPTIONS: any = {
redirect: "follow",
follow: 1,
headers: {
"user-agent": "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)",
},
size: 1024 * 1024 * 8,
compress: true,
method: "GET",
};
router.post("/", bodyParser.json(), async (req: Request, res: Response) => {
if (req.headers.signature !== Config.get().security.requestSignature)
throw new HTTPError("Invalid request signature");
if (!req.body) throw new HTTPError("Invalid Body");
const { url } = req.body;
if (!url || typeof url !== "string") throw new HTTPError("Invalid url");
const id = Snowflake.generate();
try {
const response = await fetch(ogImage, DEFAULT_FETCH_OPTIONS);
const buffer = await response.buffer();
await storage.set(`/external/${id}`, buffer);
res.send({ id });
} catch (error) {
throw new HTTPError("Couldn't fetch website");
}
});
router.get("/:id/", async (req: Request, res: Response) => {
const { id } = req.params;
const file = await storage.get(`/external/${id}`);
if (!file) throw new HTTPError("File not found");
const result = await FileType.fromBuffer(file);
res.set("Content-Type", result?.mime);
return res.send(file);
});
export default router;

9
src/routes/ping.ts Normal file
View File

@ -0,0 +1,9 @@
import { Router, Response, Request } from "express";
const router = Router();
router.get("/", (req: Request, res: Response) => {
res.send("pong");
});
export default router;

27
src/start.ts Normal file
View File

@ -0,0 +1,27 @@
import path from "path";
import dotenv from "dotenv";
import fse from "fs-extra";
dotenv.config();
if (!process.env.STORAGE_PROVIDER) process.env.STORAGE_PROVIDER = "file";
// TODO:nodejs path.join trailing slash windows compatible
if (process.env.STORAGE_PROVIDER === "file") {
if (process.env.STORAGE_LOCATION) {
if (!process.env.STORAGE_LOCATION.startsWith("/")) {
process.env.STORAGE_LOCATION = path.join(__dirname, "..", process.env.STORAGE_LOCATION, "/");
}
} else {
process.env.STORAGE_LOCATION = path.join(__dirname, "..", "files", "/");
}
fse.ensureDirSync(process.env.STORAGE_LOCATION);
}
import { CDNServer } from "./Server";
const server = new CDNServer({ port: Number(process.env.PORT) || 3003 });
server
.start()
.then(() => {
console.log("[Server] started on :" + server.options.port);
})
.catch((e) => console.error("[Server] Error starting: ", e));

40
src/util/FileStorage.ts Normal file
View File

@ -0,0 +1,40 @@
import { Storage } from "./Storage";
import fs from "fs";
import fse from "fs-extra";
import { join, relative, dirname } from "path";
import "missing-native-js-functions";
import { Readable } from "stream";
import ExifTransformer = require("exif-be-gone");
function getPath(path: string) {
// STORAGE_LOCATION has a default value in start.ts
const root = process.env.STORAGE_LOCATION || "../";
var filename = join(root, path);
if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) throw new Error("invalid path");
return filename;
}
export class FileStorage implements Storage {
async get(path: string): Promise<Buffer | null> {
try {
return fs.readFileSync(getPath(path));
} catch (error) {
return null;
}
}
async set(path: string, value: any) {
path = getPath(path);
fse.ensureDirSync(dirname(path));
value = Readable.from(value);
const cleaned_file = fs.createWriteStream(path);
return value.pipe(new ExifTransformer()).pipe(cleaned_file);
}
async delete(path: string) {
fs.unlinkSync(getPath(path));
}
}

15
src/util/Storage.ts Normal file
View File

@ -0,0 +1,15 @@
import { FileStorage } from "./FileStorage";
export interface Storage {
set(path: string, data: Buffer): Promise<void>;
get(path: string): Promise<Buffer | null>;
delete(path: string): Promise<void>;
}
var storage: Storage;
if (process.env.STORAGE_PROVIDER === "file") {
storage = new FileStorage();
}
export { storage };

10
src/util/multer.ts Normal file
View File

@ -0,0 +1,10 @@
import multerConfig from "multer";
export const multer = multerConfig({
storage: multerConfig.memoryStorage(),
limits: {
fields: 10,
files: 10,
fileSize: 1024 * 1024 * 100, // 100 mb
},
});

70
tsconfig.json Normal file
View File

@ -0,0 +1,70 @@
{
"include": ["src/**/*.ts"],
"compilerOptions": {
/* Visit https://aka.ms/tsconfig.json to read more about this file */
/* Basic Options */
// "incremental": true, /* Enable incremental compilation */
"target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
"lib": ["ES2015", "dom"] /* Specify library files to be included in the compilation. */,
"allowJs": true /* Allow javascript files to be compiled. */,
"checkJs": true /* Report errors in .js files. */,
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
"declaration": true /* Generates corresponding '.d.ts' file. */,
"declarationMap": false /* Generates a sourcemap for each corresponding '.d.ts' file. */,
"inlineSourceMap": true /* Emit a single file with source maps instead of having a separate file. */,
// "sourceMap": true /* Generates corresponding '.map' file. */,
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "./dist/" /* Redirect output structure to the directory. */,
"rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
// "composite": true, /* Enable project compilation */
// "tsBuildInfoFile": "./", /* Specify file to store incremental compilation information */
// "removeComments": true, /* Do not emit comments to output. */
// "noEmit": true, /* Do not emit outputs. */
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
// "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
/* Strict Type-Checking Options */
"strict": true /* Enable all strict type-checking options. */,
"noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */,
// "strictNullChecks": true, /* Enable strict null checks. */
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
// "strictBindCallApply": true, /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
"strictPropertyInitialization": false /* Enable strict checking of property initialization in classes. */,
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
"alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */,
/* Additional Checks */
// "noUnusedLocals": true, /* Report errors on unused locals. */
// "noUnusedParameters": true, /* Report errors on unused parameters. */
// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
/* Module Resolution Options */
// "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
// "typeRoots": [], /* List of folders to include type definitions from. */
"types": ["node"] /* Type declaration files to be included in compilation. */,
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
// "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */
/* Source Map Options */
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
/* Experimental Options */
// "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
// "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
/* Advanced Options */
"skipLibCheck": true /* Skip type checking of declaration files. */,
"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
}
}