Merge pull request #1126 from DEVTomatoCake/feat/local-image-proxy

This commit is contained in:
Madeline 2024-08-19 16:38:03 +10:00 committed by GitHub
commit dbd93bdd7c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 2667 additions and 934 deletions

View File

@ -2,4 +2,5 @@ assets
dist dist
node_modules node_modules
.github .github
.vscode .vscode
hashes.json

View File

@ -13,11 +13,21 @@
inherit system; inherit system;
}; };
hashesFile = builtins.fromJSON (builtins.readFile ./hashes.json); hashesFile = builtins.fromJSON (builtins.readFile ./hashes.json);
lib = pkgs.lib;
in rec { in rec {
packages.default = pkgs.buildNpmPackage { packages.default = pkgs.buildNpmPackage {
pname = "spacebar-server-ts"; pname = "spacebar-server-ts";
src = ./.;
name = "spacebar-server-ts"; name = "spacebar-server-ts";
meta = with lib; {
description = "Spacebar server, a FOSS reimplementation of the Discord backend.";
homepage = "https://github.com/spacebarchat/server";
license = licenses.agpl3Plus;
platforms = platforms.all;
mainProgram = "start-bundle";
};
src = ./.;
nativeBuildInputs = with pkgs; [ python3 ]; nativeBuildInputs = with pkgs; [ python3 ];
npmDepsHash = hashesFile.npmDepsHash; npmDepsHash = hashesFile.npmDepsHash;
makeCacheWritable = true; makeCacheWritable = true;

View File

@ -1,3 +1,3 @@
{ {
"npmDepsHash": "sha256-kdS1SwcBu6Dor92iO1ickLgz0T5UL16nyA49xXGajf4=" "npmDepsHash": "sha256-qcHlktC4qrhOJ6AwKbccPkr0cVrAtPhGK+xD/eV+scU="
} }

View File

@ -1,10 +1,10 @@
#!/usr/bin/env nix-shell #!/usr/bin/env nix-shell
#!nix-shell -i "bash -x" -p bash prefetch-npm-deps jq git nix-output-monitor #!nix-shell -i "bash -x" -p bash prefetch-npm-deps jq git nix-output-monitor
nix flake update nix flake update --extra-experimental-features 'nix-command flakes'
DEPS_HASH=`prefetch-npm-deps package-lock.json` DEPS_HASH=`prefetch-npm-deps package-lock.json`
TMPFILE=$(mktemp) TMPFILE=$(mktemp)
jq '.npmDepsHash = "'$DEPS_HASH'"' hashes.json > $TMPFILE jq '.npmDepsHash = "'$DEPS_HASH'"' hashes.json > $TMPFILE
mv -- "$TMPFILE" hashes.json mv -- "$TMPFILE" hashes.json
nom build .# || exit $? nom build .# --extra-experimental-features 'nix-command flakes' || exit $?
git add hashes.json flake.lock flake.nix git add hashes.json flake.lock flake.nix

3354
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -117,6 +117,7 @@
}, },
"optionalDependencies": { "optionalDependencies": {
"erlpack": "^0.1.4", "erlpack": "^0.1.4",
"jimp": "^0.22.12",
"mysql": "^2.18.1", "mysql": "^2.18.1",
"nodemailer-mailgun-transport": "^2.1.5", "nodemailer-mailgun-transport": "^2.1.5",
"nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport", "nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport",

View File

@ -34,7 +34,7 @@ import "missing-native-js-functions";
import morgan from "morgan"; import morgan from "morgan";
import path from "path"; import path from "path";
import { red } from "picocolors"; import { red } from "picocolors";
import { Authentication, CORS } from "./middlewares/"; import { Authentication, CORS, ImageProxy } from "./middlewares/";
import { BodyParser } from "./middlewares/BodyParser"; import { BodyParser } from "./middlewares/BodyParser";
import { ErrorHandler } from "./middlewares/ErrorHandler"; import { ErrorHandler } from "./middlewares/ErrorHandler";
import { initRateLimits } from "./middlewares/RateLimit"; import { initRateLimits } from "./middlewares/RateLimit";
@ -137,6 +137,8 @@ export class SpacebarServer extends Server {
app.use("/api/v9", api); app.use("/api/v9", api);
app.use("/api", api); // allow unversioned requests app.use("/api", api); // allow unversioned requests
app.use("/imageproxy/:hash/:size/:url", ImageProxy);
app.get("/", (req, res) => app.get("/", (req, res) =>
res.sendFile(path.join(PUBLIC_ASSETS_FOLDER, "index.html")), res.sendFile(path.join(PUBLIC_ASSETS_FOLDER, "index.html")),
); );

View File

@ -0,0 +1,180 @@
/*
Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
Copyright (C) 2023 Spacebar and Spacebar Contributors
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { Config, JimpType } from "@spacebar/util";
import { Request, Response } from "express";
import { yellow } from "picocolors";
import crypto from "crypto";
import fetch from "node-fetch";
let sharp: undefined | false | { default: typeof import("sharp") } = undefined;
let Jimp: JimpType | undefined = undefined;
try {
Jimp = require("jimp") as JimpType;
} catch {
// empty
}
let sentImageProxyWarning = false;
const sharpSupported = new Set([
"image/jpeg",
"image/png",
"image/bmp",
"image/tiff",
"image/gif",
"image/webp",
"image/avif",
"image/svg+xml",
]);
const jimpSupported = new Set([
"image/jpeg",
"image/png",
"image/bmp",
"image/tiff",
"image/gif",
]);
const resizeSupported = new Set([...sharpSupported, ...jimpSupported]);
export async function ImageProxy(req: Request, res: Response) {
const path = req.originalUrl.split("/").slice(2);
// src/api/util/utility/EmbedHandlers.ts getProxyUrl
const hash = crypto
.createHmac("sha1", Config.get().security.requestSignature)
.update(path.slice(1).join("/"))
.digest("base64")
.replace(/\+/g, "-")
.replace(/\//g, "_");
try {
if (!crypto.timingSafeEqual(Buffer.from(hash), Buffer.from(path[0])))
throw new Error("Invalid signature");
} catch {
console.log("Invalid signature, expected " + hash + " got " + path[0]);
res.status(403).send("Invalid signature");
return;
}
const abort = new AbortController();
setTimeout(() => abort.abort(), 5000);
const request = await fetch(path.slice(2).join("/"), {
headers: {
"User-Agent": "SpacebarImageProxy/1.0.0 (https://spacebar.chat)",
},
signal: abort.signal,
}).catch((e) => {
if (e.name === "AbortError") res.status(504).send("Request timed out");
else res.status(500).send("Unable to proxy origin: " + e.message);
});
if (!request) return;
if (request.status !== 200) {
res.status(request.status).send(
"Origin failed to respond: " +
request.status +
" " +
request.statusText,
);
return;
}
if (
!request.headers.get("Content-Type") ||
!request.headers.get("Content-Length")
) {
res.status(500).send(
"Origin did not provide a Content-Type or Content-Length header",
);
return;
}
// @ts-expect-error TS doesn't believe that the header cannot be null (it's checked for falsiness above)
if (parseInt(request.headers.get("Content-Length")) > 1024 * 1024 * 10) {
res.status(500).send(
"Origin provided a Content-Length header that is too large",
);
return;
}
// @ts-expect-error TS doesn't believe that the header cannot be null (it's checked for falsiness above)
let contentType: string = request.headers.get("Content-Type");
const arrayBuffer = await request.arrayBuffer();
let resultBuffer = Buffer.from(arrayBuffer);
if (
!sentImageProxyWarning &&
resizeSupported.has(contentType) &&
/^\d+x\d+$/.test(path[1])
) {
if (sharp !== false) {
try {
sharp = await import("sharp");
} catch {
sharp = false;
}
}
if (sharp === false && !Jimp) {
try {
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore Typings don't fit
Jimp = await import("jimp");
} catch {
sentImageProxyWarning = true;
console.log(
`[ImageProxy] ${yellow(
'Neither "sharp" or "jimp" NPM packages are installed, image resizing will be disabled',
)}`,
);
}
}
const [width, height] = path[1].split("x").map((x) => parseInt(x));
const buffer = Buffer.from(arrayBuffer);
if (sharp && sharpSupported.has(contentType)) {
resultBuffer = await sharp
.default(buffer)
// Sharp doesn't support "scaleToFit"
.resize(width)
.toBuffer();
} else if (Jimp && jimpSupported.has(contentType)) {
resultBuffer = await Jimp.read(buffer).then((image) => {
contentType = image.getMIME();
return (
image
.scaleToFit(width, height)
// @ts-expect-error Jimp is defined at this point
.getBufferAsync(Jimp.AUTO)
);
});
}
}
res.header("Content-Type", contentType);
res.setHeader(
"Cache-Control",
"public, max-age=" + Config.get().cdn.proxyCacheHeaderSeconds,
);
res.send(resultBuffer);
}

View File

@ -21,3 +21,4 @@ export * from "./BodyParser";
export * from "./CORS"; export * from "./CORS";
export * from "./ErrorHandler"; export * from "./ErrorHandler";
export * from "./RateLimit"; export * from "./RateLimit";
export * from "./ImageProxy";

View File

@ -1,17 +1,17 @@
/* /*
Spacebar: A FOSS re-implementation and extension of the Discord.com backend. Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
Copyright (C) 2023 Spacebar and Spacebar Contributors Copyright (C) 2023 Spacebar and Spacebar Contributors
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. (at your option) any later version.
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details. GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
@ -25,4 +25,6 @@ export class CdnConfiguration extends EndpointConfiguration {
endpointPublic: string | null = null; endpointPublic: string | null = null;
endpointPrivate: string | null = null; endpointPrivate: string | null = null;
proxyCacheHeaderSeconds: number = 60 * 60 * 24;
} }

View File

@ -1,17 +1,17 @@
/* /*
Spacebar: A FOSS re-implementation and extension of the Discord.com backend. Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
Copyright (C) 2023 Spacebar and Spacebar Contributors Copyright (C) 2023 Spacebar and Spacebar Contributors
This program is free software: you can redistribute it and/or modify This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version. (at your option) any later version.
This program is distributed in the hope that it will be useful, This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details. GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>. along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */

23
src/util/imports/Jimp.ts Normal file
View File

@ -0,0 +1,23 @@
/*
Spacebar: A FOSS re-implementation and extension of the Discord.com backend.
Copyright (C) 2023 Spacebar and Spacebar Contributors
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/* eslint-disable @typescript-eslint/no-explicit-any */
export type JimpType = {
read: (data: Buffer) => Promise<any>;
};

View File

@ -18,3 +18,4 @@
export * from "./OrmUtils"; export * from "./OrmUtils";
export * from "./Erlpack"; export * from "./Erlpack";
export * from "./Jimp";