Dekescordes/spacebar
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

prettier formatted /cdn

Browse Source
This commit is contained in:
developomp 2021-10-20 09:14:48 +09:00
parent 99f6da9762
commit af73cb6993
6 changed files with 164 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
cdn/README.md
View File

@ -1,20 +1,26 @@
# Fosscord-CDN # Fosscord-CDN
CDN for Fosscord CDN for Fosscord
## Run localy: ## Run localy:
``` ```
npm i npm i
node dist/ node dist/
``` ```
## Endpoints: ## Endpoints:
### POST `/attachments/<filename>` ### POST `/attachments/<filename>`
``` ```
Content-Type: form-data Content-Type: form-data
attachment: File (binary-data) attachment: File (binary-data)
``` ```
##### Returns: ##### Returns:
``` ```
{ {
"success": boolean, // true "success": boolean, // true
@ -23,20 +29,28 @@ attachment: File (binary-data)
"filename": string // "lakdoiauej.png" "filename": string // "lakdoiauej.png"
} }
``` ```
### GET `/attachments/<id>/<filename>` ### GET `/attachments/<id>/<filename>`
``` ```
requests image from database with given <id> and <filename> requests image from database with given <id> and <filename>
``` ```
##### Returns: ##### Returns:
``` ```
Content-Type: image/<imageType(png,img,gif)> Content-Type: image/<imageType(png,img,gif)>
Image Image
``` ```
### DELETE `/attachments/<id>/<filename>` ### DELETE `/attachments/<id>/<filename>`
``` ```
deletes database entry deletes database entry
``` ```
##### Returns: ##### Returns:
``` ```
Content-Type: application/json Content-Type: application/json
@ -49,7 +63,8 @@ Content-Type: application/json
<hr> <hr>
_(endpoints for crawler):_ _(endpoints for crawler):_
### POST `/external`
### POST `/external`
``` ```
requests crawling of `og:`metadata and the download of the `og:image` property requests crawling of `og:`metadata and the download of the `og:image` property
@ -59,7 +74,9 @@ Content-Type: application/json
body: body:
{"url": URL} // "https://discord.com" {"url": URL} // "https://discord.com"
``` ```
##### Returns: ##### Returns:
``` ```
Content-Type: application/json Content-Type: application/json
@ -72,17 +89,23 @@ Content-Type: application/json
"ogType": string // "website" "ogType": string // "website"
} }
``` ```
### GET `/external/<id>/<filename>` ### GET `/external/<id>/<filename>`
- requests cached crawled image
- requests cached crawled image
``` ```
url-params: url-params:
:id // aHR0cHM6Ly9kaXNjb3JkLmNvbQ== :id // aHR0cHM6Ly9kaXNjb3JkLmNvbQ==
:filename // discord.png :filename // discord.png
``` ```
``` ```
/external/aHR0cHM6Ly9kaXNjb3JkLmNvbQ==/discord.png /external/aHR0cHM6Ly9kaXNjb3JkLmNvbQ==/discord.png
``` ```
##### Returns: ##### Returns:
``` ```
Content-Type: image/<imageType(png,img,gif)> Content-Type: image/<imageType(png,img,gif)>
Image Image

130
cdn/package.json
View File

@ -1,67 +1,67 @@
{ {
"name": "@fosscord/cdn", "name": "@fosscord/cdn",
"version": "1.0.0", "version": "1.0.0",
"description": "cdn for fosscord", "description": "cdn for fosscord",
"main": "dist/index.js", "main": "dist/index.js",
"types": "src/index.ts", "types": "src/index.ts",
"scripts": { "scripts": {
"test": "npm run build && jest --coverage ./tests", "test": "npm run build && jest --coverage ./tests",
"build": "npx tsc -p .", "build": "npx tsc -p .",
"start": "node dist/start.js" "start": "node dist/start.js"
}, },
"repository": { "repository": {
"type": "git", "type": "git",
"url": "git+https://github.com/fosscord/fosscord-server.git" "url": "git+https://github.com/fosscord/fosscord-server.git"
}, },
"keywords": [], "keywords": [],
"author": "", "author": "",
"license": "ISC", "license": "ISC",
"bugs": { "bugs": {
"url": "https://github.com/fosscord/fosscord-server/issues" "url": "https://github.com/fosscord/fosscord-server/issues"
}, },
"homepage": "https://github.com/fosscord/fosscord-server#readme", "homepage": "https://github.com/fosscord/fosscord-server#readme",
"devDependencies": { "devDependencies": {
"@types/amqplib": "^0.8.1", "@types/amqplib": "^0.8.1",
"@types/body-parser": "^1.19.0", "@types/body-parser": "^1.19.0",
"@types/btoa": "^1.2.3", "@types/btoa": "^1.2.3",
"@types/dotenv": "^8.2.0", "@types/dotenv": "^8.2.0",
"@types/express": "^4.17.12", "@types/express": "^4.17.12",
"@types/fs-extra": "^9.0.12", "@types/fs-extra": "^9.0.12",
"@types/jsonwebtoken": "^8.5.0", "@types/jsonwebtoken": "^8.5.0",
"@types/multer": "^1.4.7", "@types/multer": "^1.4.7",
"@types/node": "^14.17.0", "@types/node": "^14.17.0",
"@types/node-fetch": "^2.5.7", "@types/node-fetch": "^2.5.7",
"@zerollup/ts-transform-paths": "^1.7.18", "@zerollup/ts-transform-paths": "^1.7.18",
"ts-patch": "^1.4.4" "ts-patch": "^1.4.4"
}, },
"dependencies": { "dependencies": {
"@aws-sdk/client-s3": "^3.36.1", "@aws-sdk/client-s3": "^3.36.1",
"@aws-sdk/node-http-handler": "^3.36.0", "@aws-sdk/node-http-handler": "^3.36.0",
"@fosscord/util": "file:../util", "@fosscord/util": "file:../util",
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"btoa": "^1.2.1", "btoa": "^1.2.1",
"dotenv": "^10.0.0", "dotenv": "^10.0.0",
"exif-be-gone": "^1.2.0", "exif-be-gone": "^1.2.0",
"express": "^4.17.1", "express": "^4.17.1",
"express-async-errors": "^3.1.1", "express-async-errors": "^3.1.1",
"file-type": "^16.5.0", "file-type": "^16.5.0",
"form-data": "^4.0.0", "form-data": "^4.0.0",
"fs-extra": "^10.0.0", "fs-extra": "^10.0.0",
"image-size": "^1.0.0", "image-size": "^1.0.0",
"jest": "^27.0.6", "jest": "^27.0.6",
"lambert-db": "^1.2.3", "lambert-db": "^1.2.3",
"lambert-server": "^1.2.12", "lambert-server": "^1.2.12",
"missing-native-js-functions": "^1.2.17", "missing-native-js-functions": "^1.2.17",
"multer": "^1.4.2", "multer": "^1.4.2",
"nanocolors": "^0.2.12", "nanocolors": "^0.2.12",
"node-fetch": "^2.6.1", "node-fetch": "^2.6.1",
"supertest": "^6.1.6", "supertest": "^6.1.6",
"typescript": "^4.1.2" "typescript": "^4.1.2"
}, },
"jest": { "jest": {
"setupFilesAfterEnv": [ "setupFilesAfterEnv": [
"<rootDir>/jest/setup.js" "<rootDir>/jest/setup.js"
], ],
"verbose": true "verbose": true
} }
} }

20
cdn/src/util/S3Storage.ts
View File

@ -5,30 +5,30 @@ import { Storage } from "./Storage";
const readableToBuffer = (readable: Readable): Promise<Buffer> => const readableToBuffer = (readable: Readable): Promise<Buffer> =>
new Promise((resolve, reject) => { new Promise((resolve, reject) => {
const chunks: Buffer[] = []; const chunks: Buffer[] = [];
readable.on('data', chunk => chunks.push(chunk)); readable.on("data", (chunk) => chunks.push(chunk));
readable.on('error', reject); readable.on("error", reject);
readable.on('end', () => resolve(Buffer.concat(chunks))); readable.on("end", () => resolve(Buffer.concat(chunks)));
}); });
export class S3Storage implements Storage { export class S3Storage implements Storage {
public constructor( public constructor(
private client: S3, private client: S3,
private bucket: string, private bucket: string,
private basePath?: string, private basePath?: string
) {} ) {}
/** /**
* Always return a string, to ensure consistency. * Always return a string, to ensure consistency.
*/ */
get bucketBasePath() { get bucketBasePath() {
return this.basePath ?? ''; return this.basePath ?? "";
} }
async set(path: string, data: Buffer): Promise<void> { async set(path: string, data: Buffer): Promise<void> {
await this.client.putObject({ await this.client.putObject({
Bucket: this.bucket, Bucket: this.bucket,
Key: `${this.bucketBasePath}${path}`, Key: `${this.bucketBasePath}${path}`,
Body: data Body: data,
}); });
} }
@ -36,15 +36,15 @@ export class S3Storage implements Storage {
try { try {
const s3Object = await this.client.getObject({ const s3Object = await this.client.getObject({
Bucket: this.bucket, Bucket: this.bucket,
Key: `${this.bucketBasePath ?? ''}${path}` Key: `${this.bucketBasePath ?? ""}${path}`,
}); });
if (!s3Object.Body) return null; if (!s3Object.Body) return null;
const body = s3Object.Body; const body = s3Object.Body;
return await readableToBuffer(<Readable> body); return await readableToBuffer(<Readable>body);
} catch(err) { } catch (err) {
console.error(`[CDN] Unable to get S3 object at path ${path}.`); console.error(`[CDN] Unable to get S3 object at path ${path}.`);
console.error(err); console.error(err);
return null; return null;
@ -54,7 +54,7 @@ export class S3Storage implements Storage {
async delete(path: string): Promise<void> { async delete(path: string): Promise<void> {
await this.client.deleteObject({ await this.client.deleteObject({
Bucket: this.bucket, Bucket: this.bucket,
Key: `${this.bucketBasePath}${path}` Key: `${this.bucketBasePath}${path}`,
}); });
} }
} }

19
cdn/src/util/Storage.ts
View File

@ -2,7 +2,7 @@ import { FileStorage } from "./FileStorage";
import path from "path"; import path from "path";
import fse from "fs-extra"; import fse from "fs-extra";
import { bgCyan, black } from "nanocolors"; import { bgCyan, black } from "nanocolors";
import { S3 } from '@aws-sdk/client-s3'; import { S3 } from "@aws-sdk/client-s3";
import { S3Storage } from "./S3Storage"; import { S3Storage } from "./S3Storage";
process.cwd(); process.cwd();
@ -27,17 +27,20 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
storage = new FileStorage(); storage = new FileStorage();
} else if (process.env.STORAGE_PROVIDER === "s3") { } else if (process.env.STORAGE_PROVIDER === "s3") {
const const region = process.env.STORAGE_REGION,
region = process.env.STORAGE_REGION,
bucket = process.env.STORAGE_BUCKET; bucket = process.env.STORAGE_BUCKET;
if (!region) { if (!region) {
console.error(`[CDN] You must provide a region when using the S3 storage provider.`); console.error(
`[CDN] You must provide a region when using the S3 storage provider.`
);
process.exit(1); process.exit(1);
} }
if (!bucket) { if (!bucket) {
console.error(`[CDN] You must provide a bucket when using the S3 storage provider.`); console.error(
`[CDN] You must provide a bucket when using the S3 storage provider.`
);
process.exit(1); process.exit(1);
} }
@ -45,8 +48,10 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
let location = process.env.STORAGE_LOCATION; let location = process.env.STORAGE_LOCATION;
if (!location) { if (!location) {
console.warn(`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`); console.warn(
location = undefined; `[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`
);
location = undefined;
} }
const client = new S3({ region }); const client = new S3({ region });

63
cdn/tests/cdn_endpoints.test.js
View File

@ -11,7 +11,12 @@ if (!process.env.STORAGE_PROVIDER) process.env.STORAGE_PROVIDER = "file";
if (process.env.STORAGE_PROVIDER === "file") { if (process.env.STORAGE_PROVIDER === "file") {
if (process.env.STORAGE_LOCATION) { if (process.env.STORAGE_LOCATION) {
if (!process.env.STORAGE_LOCATION.startsWith("/")) { if (!process.env.STORAGE_LOCATION.startsWith("/")) {
process.env.STORAGE_LOCATION = path.join(__dirname, "..", process.env.STORAGE_LOCATION, "/"); process.env.STORAGE_LOCATION = path.join(
__dirname,
"..",
process.env.STORAGE_LOCATION,
"/"
);
} }
} else { } else {
process.env.STORAGE_LOCATION = path.join(__dirname, "..", "files", "/"); process.env.STORAGE_LOCATION = path.join(__dirname, "..", "files", "/");
@ -67,7 +72,9 @@ describe("/attachments", () => {
.set({ signature: Config.get().security.requestSignature }) .set({ signature: Config.get().security.requestSignature })
.attach("file", __dirname + "/antman.jpg"); .attach("file", __dirname + "/antman.jpg");
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.headers["content-type"]).toEqual(expect.stringContaining("json")); expect(response.headers["content-type"]).toEqual(
expect.stringContaining("json")
);
expect(response.body.url).toBeDefined(); expect(response.body.url).toBeDefined();
}); });
}); });
@ -79,9 +86,11 @@ describe("/attachments", () => {
.post("/attachments/123456789") .post("/attachments/123456789")
.set({ signature: Config.get().security.requestSignature }) .set({ signature: Config.get().security.requestSignature })
.attach("file", __dirname + "/antman.jpg"); .attach("file", __dirname + "/antman.jpg");
request.get(response.body.url.replace("http://localhost:3003", "")).then((x) => { request
expect(x.statusCode).toBe(200); .get(response.body.url.replace("http://localhost:3003", ""))
}); .then((x) => {
expect(x.statusCode).toBe(200);
});
}); });
}); });
}); });
@ -92,9 +101,13 @@ describe("/attachments", () => {
.post("/attachments/123456789") .post("/attachments/123456789")
.set({ signature: Config.get().security.requestSignature }) .set({ signature: Config.get().security.requestSignature })
.attach("file", __dirname + "/antman.jpg"); .attach("file", __dirname + "/antman.jpg");
request.delete(response.body.url.replace("http://localhost:3003", "")).then((x) => { request
expect(x.body.success).toBeDefined(); .delete(
}); response.body.url.replace("http://localhost:3003", "")
)
.then((x) => {
expect(x.body.success).toBeDefined();
});
}); });
}); });
}); });
@ -123,7 +136,9 @@ describe("/avatars", () => {
.set({ signature: Config.get().security.requestSignature }) .set({ signature: Config.get().security.requestSignature })
.attach("file", __dirname + "/antman.jpg"); .attach("file", __dirname + "/antman.jpg");
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.headers["content-type"]).toEqual(expect.stringContaining("json")); expect(response.headers["content-type"]).toEqual(
expect.stringContaining("json")
);
expect(response.body.url).toBeDefined(); expect(response.body.url).toBeDefined();
}); });
}); });
@ -135,9 +150,11 @@ describe("/avatars", () => {
.post("/avatars/123456789") .post("/avatars/123456789")
.set({ signature: Config.get().security.requestSignature }) .set({ signature: Config.get().security.requestSignature })
.attach("file", __dirname + "/antman.jpg"); .attach("file", __dirname + "/antman.jpg");
request.get(response.body.url.replace("http://localhost:3003", "")).then((x) => { request
expect(x.statusCode).toBe(200); .get(response.body.url.replace("http://localhost:3003", ""))
}); .then((x) => {
expect(x.statusCode).toBe(200);
});
}); });
}); });
}); });
@ -148,9 +165,13 @@ describe("/avatars", () => {
.post("/avatars/123456789") .post("/avatars/123456789")
.set({ signature: Config.get().security.requestSignature }) .set({ signature: Config.get().security.requestSignature })
.attach("file", __dirname + "/antman.jpg"); .attach("file", __dirname + "/antman.jpg");
request.delete(response.body.url.replace("http://localhost:3003", "")).then((x) => { request
expect(x.body.success).toBeDefined(); .delete(
}); response.body.url.replace("http://localhost:3003", "")
)
.then((x) => {
expect(x.body.success).toBeDefined();
});
}); });
}); });
}); });
@ -177,9 +198,13 @@ describe("/external", () => {
const response = await request const response = await request
.post("/external") .post("/external")
.set({ signature: Config.get().security.requestSignature }) .set({ signature: Config.get().security.requestSignature })
.send({ url: "https://i.ytimg.com/vi_webp/TiXzhQr5AUc/mqdefault.webp" }); .send({
url: "https://i.ytimg.com/vi_webp/TiXzhQr5AUc/mqdefault.webp",
});
expect(response.statusCode).toBe(200); expect(response.statusCode).toBe(200);
expect(response.headers["content-type"]).toEqual(expect.stringContaining("json")); expect(response.headers["content-type"]).toEqual(
expect.stringContaining("json")
);
expect(response.body.id).toBeDefined(); expect(response.body.id).toBeDefined();
}); });
}); });
@ -201,7 +226,9 @@ describe("/external", () => {
let response = await request let response = await request
.post("/external") .post("/external")
.set({ signature: Config.get().security.requestSignature }) .set({ signature: Config.get().security.requestSignature })
.send({ url: "https://i.ytimg.com/vi_webp/TiXzhQr5AUc/mqdefault.webp" }); .send({
url: "https://i.ytimg.com/vi_webp/TiXzhQr5AUc/mqdefault.webp",
});
request.get(`external/${response.body.id}`).then((x) => { request.get(`external/${response.body.id}`).then((x) => {
expect(x.statusCode).toBe(200); expect(x.statusCode).toBe(200);
}); });

9
cdn/tsconfig.json
View File

@ -7,7 +7,10 @@
"incremental": true /* Enable incremental compilation */, "incremental": true /* Enable incremental compilation */,
"target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, "target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, "module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
"lib": ["ES2015", "dom"] /* Specify library files to be included in the compilation. */, "lib": [
"ES2015",
"dom"
] /* Specify library files to be included in the compilation. */,
"allowJs": true /* Allow javascript files to be compiled. */, "allowJs": true /* Allow javascript files to be compiled. */,
"checkJs": true /* Report errors in .js files. */, "checkJs": true /* Report errors in .js files. */,
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
@ -48,7 +51,9 @@
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
// "typeRoots": [], /* List of folders to include type definitions from. */ // "typeRoots": [], /* List of folders to include type definitions from. */
"types": ["node"] /* Type declaration files to be included in compilation. */, "types": [
"node"
] /* Type declaration files to be included in compilation. */,
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */