Config: First rewrite of config and working implementation of getting values

This commit is contained in:
Diego Magdaleno 2021-05-19 20:39:31 -05:00
parent bb2d3715ea
commit e3f6a29df7
14 changed files with 12873 additions and 188 deletions

12590
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -33,14 +33,22 @@
"dependencies": {
"@fosscord/server-util": "^1.1.5",
"@types/jest": "^26.0.22",
"@types/json-schema": "^7.0.7",
"ajv": "^8.4.0",
"ajv-formats": "^2.1.0",
"assert": "^1.5.0",
"atomically": "^1.7.0",
"bcrypt": "^5.0.0",
"body-parser": "^1.19.0",
"dot-prop": "^6.0.1",
"dotenv": "^8.2.0",
"env-paths": "^2.2.1",
"express": "^4.17.1",
"express-validator": "^6.9.2",
"i18next": "^19.8.5",
"i18next-http-middleware": "^3.1.1",
"i18next-node-fs-backend": "^2.1.3",
"json-schema-typed": "^7.0.3",
"jsonwebtoken": "^8.5.1",
"lambert-server": "^1.2.2",
"missing-native-js-functions": "^1.2.6",

View File

@ -3,7 +3,7 @@ import fs from "fs/promises";
import { Connection } from "mongoose";
import { Server, ServerOptions } from "lambert-server";
import { Authentication, CORS, GlobalRateLimit } from "./middlewares/";
import Config from "./util/Config";
import * as Config from "./util/Config";
import { db } from "@fosscord/server-util";
import i18next from "i18next";
import i18nextMiddleware, { I18next } from "i18next-http-middleware";
@ -51,7 +51,7 @@ export class FosscordServer extends Server {
await (db as Promise<Connection>);
await this.setupSchema();
console.log("[DB] connected");
await Promise.all([Config.init()]);
//await Promise.all([Config.init()]);
this.app.use(GlobalRateLimit);
this.app.use(Authentication);

View File

@ -1,5 +1,6 @@
import { NextFunction, Request, Response } from "express";
import Config from "../util/Config";
import * as Config from '../util/Config'
import crypto from "crypto";
// TODO: use mongodb ttl index
// TODO: increment count on serverside
@ -43,7 +44,8 @@ export async function GlobalRateLimit(req: Request, res: Response, next: NextFun
}
export function getIpAdress(req: Request): string {
const { forwadedFor } = Config.get().security;
const rateLimitProperties = Config.apiConfig.get('security', {jwtSecret: crypto.randomBytes(256).toString("base64"), forwadedFor: null, captcha: {enabled:false, service: null, sitekey: null, secret: null}}) as Config.DefaultOptions;
const { forwadedFor } = rateLimitProperties.security;
const ip = forwadedFor ? <string>req.headers[forwadedFor] : req.ip;
return ip.replaceAll(".", "_").replaceAll(":", "_");
}

View File

@ -3,7 +3,7 @@ import { check, FieldErrors, Length } from "../../util/instanceOf";
import bcrypt from "bcrypt";
import jwt from "jsonwebtoken";
import { UserModel } from "@fosscord/server-util";
import Config from "../../util/Config";
import * as Config from "../../util/Config";
import { adjustEmail } from "./register";
const router: Router = Router();
@ -25,7 +25,9 @@ router.post(
const query: any[] = [{ phone: login }];
if (email) query.push({ email });
const config = Config.get();
// TODO: Rewrite this to have the proper config syntax on the new method
const config = Config.apiConfig.store as unknown as Config.DefaultOptions;
if (config.login.requireCaptcha && config.security.captcha.enabled) {
if (!captcha_key) {
@ -67,9 +69,10 @@ export async function generateToken(id: string) {
const algorithm = "HS256";
return new Promise((res, rej) => {
const securityPropertiesSecret = Config.apiConfig.get('security.jwtSecret') as Config.DefaultOptions;
jwt.sign(
{ id: id, iat },
Config.get().security.jwtSecret,
securityPropertiesSecret.security.jwtSecret,
{
algorithm,
},

View File

@ -1,5 +1,5 @@
import { Request, Response, Router } from "express";
import Config from "../../util/Config";
import * as Config from "../../util/Config";
import { trimSpecial, User, Snowflake, UserModel } from "@fosscord/server-util";
import bcrypt from "bcrypt";
import { check, Email, EMAIL_REGEX, FieldErrors, Length } from "../../util/instanceOf";
@ -52,7 +52,8 @@ router.post(
let discriminator = "";
// get register Config
const { register, security } = Config.get();
const securityProperties = Config.apiConfig.store as unknown as Config.DefaultOptions;
const { register, security } = securityProperties;
// check if registration is allowed
if (!register.allowNewRegistration) {
@ -90,13 +91,13 @@ router.post(
},
});
}
} else if (register.email.required) {
} else if (register.email.necessary) {
throw FieldErrors({
email: { code: "BASE_TYPE_REQUIRED", message: req.t("common:field.BASE_TYPE_REQUIRED") },
});
}
if (register.dateOfBirth.required && !date_of_birth) {
if (register.dateOfBirth.necessary && !date_of_birth) {
throw FieldErrors({
date_of_birth: { code: "BASE_TYPE_REQUIRED", message: req.t("common:field.BASE_TYPE_REQUIRED") },
});

View File

@ -1,7 +1,7 @@
import { Router } from "express";
import { ChannelModel, getPermission, MessageDeleteBulkEvent, MessageModel } from "@fosscord/server-util";
import { HTTPError } from "lambert-server";
import Config from "../../../../util/Config";
import * as Config from "../../../../util/Config";
import { emitEvent } from "../../../../util/Event";
import { check } from "../../../../util/instanceOf";
@ -20,7 +20,8 @@ router.post("/", check({ messages: [String] }), async (req, res) => {
const permission = await getPermission(req.user_id, channel?.guild_id, channel_id, { channel });
permission.hasThrow("MANAGE_MESSAGES");
const { maxBulkDelete } = Config.get().limits.message;
const limitsProperties = Config.apiConfig.get('limits.message') as Config.DefaultOptions;
const { maxBulkDelete } = limitsProperties.limits.message;
const { messages } = req.body as { messages: string[] };
if (messages.length < 2) throw new HTTPError("You must at least specify 2 messages to bulk delete");

View File

@ -1,6 +1,6 @@
import { ChannelModel, getPermission, MessageModel, toObject } from "@fosscord/server-util";
import { Router, Request, Response } from "express";
import Config from "../../../util/Config";
import * as Config from "../../../util/Config";
import { HTTPError } from "lambert-server";
const router: Router = Router();
@ -18,7 +18,8 @@ router.put("/:message_id", async (req: Request, res: Response) => {
if (channel.guild_id) permission.hasThrow("MANAGE_MESSAGES");
const pinned_count = await MessageModel.count({ channel_id, pinned: true }).exec();
const { maxPins } = Config.get().limits.channel;
const limitsProperties = Config.apiConfig.get('limits.channel') as Config.DefaultOptions;
const { maxPins } = limitsProperties.limits.channel;
if (pinned_count >= maxPins) throw new HTTPError("Max pin count reached: " + maxPins);
await MessageModel.updateOne({ id: message_id }, { pinned: true }).exec();

View File

@ -1,11 +1,12 @@
import { Router } from "express";
import Config from "../util/Config"
import * as Config from "../util/Config"
const router = Router();
router.get("/", (req, res) => {
const { endpoint } = Config.getAll().gateway;
res.send({ url: endpoint || "ws://localhost:3002" });
const generalConfig = Config.apiConfig.get('gateway', 'ws://localhost:3002') as Config.DefaultOptions;
const { gateway } = generalConfig;
res.send({ url: gateway || "ws://localhost:3002" });
});
export default router;

View File

@ -3,7 +3,7 @@ import { RoleModel, GuildModel, Snowflake, Guild, RoleDocument } from "@fosscord
import { HTTPError } from "lambert-server";
import { check } from "./../../util/instanceOf";
import { GuildCreateSchema } from "../../schema/Guild";
import Config from "../../util/Config";
import * as Config from "../../util/Config";
import { getPublicUser } from "../../util/User";
import { addMember } from "../../util/Member";
@ -14,7 +14,8 @@ const router: Router = Router();
router.post("/", check(GuildCreateSchema), async (req: Request, res: Response) => {
const body = req.body as GuildCreateSchema;
const { maxGuilds } = Config.get().limits.user;
const limitsProperties = Config.apiConfig.get('limits.user') as Config.DefaultOptions;
const { maxGuilds } = limitsProperties.limits.user;
const user = await getPublicUser(req.user_id, { guilds: true });
if (user.guilds.length >= maxGuilds) {

View File

@ -5,7 +5,7 @@ import { HTTPError } from "lambert-server";
import { GuildTemplateCreateSchema } from "../../../schema/Guild";
import { getPublicUser } from "../../../util/User";
import { check } from "../../../util/instanceOf";
import Config from "../../../util/Config";
import * as Config from "../../../util/Config";
import { addMember } from "../../../util/Member";
router.get("/:code", async (req: Request, res: Response) => {
@ -21,7 +21,8 @@ router.post("/:code", check(GuildTemplateCreateSchema), async (req: Request, res
const { code } = req.params;
const body = req.body as GuildTemplateCreateSchema;
const { maxGuilds } = Config.get().limits.user;
const limitsProperties = Config.apiConfig.get('limits.user') as Config.DefaultOptions;
const { maxGuilds } = limitsProperties.limits.user;
const user = await getPublicUser(req.user_id, { guilds: true });
if (user.guilds.length >= maxGuilds) {

View File

@ -1,4 +1,12 @@
import Ajv, {JTDSchemaType} from "ajv/dist/jtd"
import Ajv, {JSONSchemaType} from "ajv"
import {ValidateFunction} from 'ajv'
import ajvFormats from 'ajv-formats';
import dotProp from "dot-prop";
import envPaths from "env-paths";
import path from "node:path";
import fs from 'fs'
import assert from "assert";
import atomically from "atomically"
export interface RateLimitOptions {
count: number;
@ -6,6 +14,7 @@ export interface RateLimitOptions {
}
export interface DefaultOptions {
gateway: string;
general: {
instance_id: string;
};
@ -69,13 +78,13 @@ export interface DefaultOptions {
};
register: {
email: {
required: boolean;
necessary: boolean;
allowlist: boolean;
blocklist: boolean;
domains: string[];
};
dateOfBirth: {
required: boolean;
necessary: boolean;
minimum: number; // in years
};
requireCaptcha: boolean;
@ -92,139 +101,380 @@ export interface DefaultOptions {
};
}
const schema: JTDSchemaType<DefaultOptions, {rateLimitOptions: RateLimitOptions}> = {
const schema: JSONSchemaType<DefaultOptions> & {
definitions: {
rateLimitOptions: JSONSchemaType<RateLimitOptions>
}
} = {
type: "object",
definitions: {
rateLimitOptions: {
type: "object",
properties: {
count: {type: "int32"},
timespan: {type: "int32"}
}
}
count: {type: "number"},
timespan: {type: "number"},
},
required: ["count", "timespan"],
},
},
properties: {
gateway: {
type: "string"
},
general: {
type: "object",
properties: {
instance_id: {type: "string"}
}
instance_id: {
type: "string"
}
},
required: ["instance_id"],
additionalProperties: false
},
permissions: {
type: "object",
properties: {
user: {
type: "object",
properties: {
createGuilds: {type: "boolean"}
}
createGuilds: {
type: "boolean"
}
},
required: ["createGuilds"],
additionalProperties: false
}
}
},
required: ["user"],
additionalProperties: false
},
limits: {
type: "object",
properties: {
user: {
type: "object",
properties: {
maxGuilds: {type: "int32"},
maxFriends: {type: "int32"},
maxUsername: {type: "int32"}
}
maxFriends: {
type: "number"
},
maxGuilds: {
type: "number"
},
maxUsername: {
type: "number"
}
},
required: ["maxFriends", "maxGuilds", "maxUsername"],
additionalProperties: false
},
guild: {
type: "object",
properties: {
maxRoles: {type: "int32"},
maxMembers: {type: "int32"},
maxChannels: {type: "int32"},
maxChannelsInCategory: {type: "int32"},
hideOfflineMember: {type: "int32"}
}
maxRoles: {
type: "number"
},
maxMembers: {
type: "number"
},
maxChannels: {
type: "number"
},
maxChannelsInCategory: {
type: "number"
},
hideOfflineMember: {
type: "number"
}
},
required: ["maxRoles", "maxMembers", "maxChannels", "maxChannelsInCategory", "hideOfflineMember"],
additionalProperties: false
},
message: {
type: "object",
properties: {
characters: {type: "int32"},
ttsCharacters: {type: "int32"},
maxReactions: {type: "int32"},
maxAttachmentSize: {type: "int32"},
maxBulkDelete: {type: "int32"}
}
characters: {
type: "number"
},
ttsCharacters: {
type: "number"
},
maxReactions: {
type: "number"
},
maxAttachmentSize: {
type: "number"
},
maxBulkDelete: {
type: "number"
}
},
required: ["characters", "ttsCharacters", "maxReactions", "maxAttachmentSize", "maxBulkDelete"],
additionalProperties: false
},
channel: {
type: "object",
properties: {
maxPins: {type: "int32"},
maxTopic: {type: "int32"},
maxPins: {
type: "number"
},
maxTopic: {
type: "number"
}
},
required: ["maxPins", "maxTopic"],
additionalProperties: false
},
rate: {
type: "object",
properties: {
ip: {
type: "object",
properties: {
enabled: {type: "boolean"},
count: {type: "int32"},
timespan: {type: "int32"},
}
count: {type: "number"},
timespan: {type: "number"}
},
required: ["enabled", "count", "timespan"],
additionalProperties: false
},
routes: {
optionalProperties: {
type: "object",
properties: {
auth: {
optionalProperties: {
login: {ref: 'rateLimitOptions'},
register: {ref: 'rateLimitOptions'}
}
type: "object",
properties: {
login: {$ref: '#/definitions/rateLimitOptions'},
register: {$ref: '#/definitions/rateLimitOptions'}
},
nullable: true,
required: [],
additionalProperties: false
},
channel: {type: "string"}
}
channel: {
type: "string",
nullable: true
}
},
required: [],
additionalProperties: false
}
}
},
required: ["ip", "routes"]
}
}
},
required: ["channel", "guild", "message", "rate", "user"],
additionalProperties: false
},
security: {
type: "object",
properties: {
jwtSecret: {type: "string"},
forwadedFor: {type: "string", nullable: true},
jwtSecret: {
type: "string"
},
forwadedFor: {
type: "string",
nullable: true
},
captcha: {
type: "object",
properties: {
enabled: {type: "boolean"},
service: {enum: ['hcaptcha', 'recaptcha'], nullable: true},
sitekey: {type: "string", nullable: true},
secret: {type: "string", nullable: true}
}
service: {
type: "string",
enum: ["hcaptcha", "recaptcha", null],
nullable: true
},
sitekey: {
type: "string",
nullable: true
},
secret: {
type: "string",
nullable: true
}
},
required: ["enabled", "secret", "service", "sitekey"],
additionalProperties: false
}
}
},
required: ["captcha", "forwadedFor", "jwtSecret"],
additionalProperties: false
},
login: {
type: "object",
properties: {
requireCaptcha: {type: "boolean"}
}
},
required: ["requireCaptcha"],
additionalProperties: false
},
register: {
type: "object",
properties: {
email: {
type: "object",
properties: {
required: {type: "boolean"},
necessary: {type: "boolean"},
allowlist: {type: "boolean"},
blocklist: {type: "boolean"},
domains: { elements: {
type: "string"
domains: {
type: "array",
items: {
type: "string"
}
}
}
},
required: ["allowlist", "blocklist", "domains", "necessary"],
additionalProperties: false
},
dateOfBirth: {
type: "object",
properties: {
necessary: {type: "boolean"},
minimum: {type: "number"}
},
required: ["minimum", "necessary"],
additionalProperties: false
},
requireCaptcha: {type: "boolean"},
requireInvite: {type: "boolean"},
allowNewRegistration: {type: "boolean"},
allowMultipleAccounts: {type: "boolean"},
password: {
type: "object",
properties: {
minLength: {type: "number"},
minNumbers: {type: "number"},
minUpperCase: {type: "number"},
minSymbols: {type: "number"},
blockInsecureCommonPasswords: {type: "boolean"}
},
required: ["minLength", "minNumbers", "minUpperCase", "minSymbols", "blockInsecureCommonPasswords"],
additionalProperties: false
}
},
dateOfBirth: {
properties: {
required: {type: "boolean"},
minimum: {type: "int32"}
}
},
requireCaptcha: {type: "boolean"},
requireInvite: {type: "boolean"},
allowNewRegistration: {type: "boolean"},
allowMultipleAccounts: {type: "boolean"},
password: {
properties: {
minLength: {type: "int32"},
minNumbers: {type: "int32"},
minUpperCase: {type: "int32"},
minSymbols: {type: "int32"},
blockInsecureCommonPasswords: {type: "boolean"}
}
required: ["allowMultipleAccounts", "allowNewRegistration", "dateOfBirth", "email", "password", "requireCaptcha", "requireInvite"],
additionalProperties: false
},
},
required: ["gateway", "general", "limits", "login", "permissions", "register", "security"],
additionalProperties: false
}
const createPlainObject = <T = unknown>(): T => {
return Object.create(null);
};
type Serialize<T> = (value: T) => string;
type Deserialize<T> = (text: string) => T;
class Config<T extends Record<string, any> = Record<string, unknown>> implements Iterable<[keyof T, T[keyof T]]> {
readonly path: string;
readonly #validator?: ValidateFunction;
readonly #defaultOptions: Partial<T> = {};
constructor() {
const ajv = new Ajv();
ajvFormats(ajv);
this.#validator = ajv.compile(schema);
const base = envPaths('fosscord').config;
this.path = path.resolve(base, 'api.json');
const fileStore = this.store;
const store = Object.assign(createPlainObject<T>(), fileStore);
this._validate(store);
try {
assert.deepStrictEqual(fileStore, store);
} catch {
this.store = store;
}
}
private _validate(data: T | unknown): void {
if (!this.#validator) {
return;
}
const valid = this.#validator(data);
if (valid || !this.#validator.errors) {
return;
}
const errors = this.#validator.errors.map(({instancePath, message = ''}) => `\`${instancePath.slice(1)}\` ${message}`);
throw new Error('The config schema was violated!: ' + errors.join('; '));
}
get store(): T {
try {
const data = fs.readFileSync(this.path).toString();
const deserializedData = this._deserialize(data);
this._validate(deserializedData);
return Object.assign(Object.create(null), deserializedData)
} catch (error) {
if (error.code == 'ENOENT') {
this._ensureDirectory();
return createPlainObject();
}
throw error;
}
}
private _ensureDirectory(): void {
fs.mkdirSync(path.dirname(this.path), {recursive: true})
}
set store(value: T) {
this._validate(value);
this._write(value);
}
private readonly _deserialize: Deserialize<T> = value => JSON.parse(value);
private readonly _serialize: Serialize<T> = value => JSON.stringify(value, undefined, '\t')
get<Key extends keyof T>(key: Key): T[Key];
get<Key extends keyof T>(key: Key, defaultValue: Required<T>[Key]): Required<T>[Key];
get<Key extends string, Value = unknown>(key: Exclude<Key, keyof T>, defaultValue?: Value): Value;
get(key: string, defaultValue?: unknown): unknown {
return this._get(key, defaultValue);
}
private _get<Key extends keyof T>(key: Key): T[Key] | undefined;
private _get<Key extends keyof T, Default = unknown>(key: Key, defaultValue: Default): T[Key] | Default;
private _get<Key extends keyof T, Default = unknown>(key: Key | string, defaultValue?: Default): Default | undefined {
return dotProp.get<T[Key] | undefined>(this.store, key as string, defaultValue as T[Key]);
}
* [Symbol.iterator](): IterableIterator<[keyof T, T[keyof T]]> {
for (const [key, value] of Object.entries(this.store)) {
yield [key, value];
}
}
private _write(value: T): void {
let data: string | Buffer = this._serialize(value);
try {
atomically.writeFileSync(this.path, data);
} catch (error) {
if (error.code == 'EXDEV') {
fs.writeFileSync(this.path, data)
return
}
throw error;
}
}
}
}
export const apiConfig = new Config();

View File

@ -14,7 +14,7 @@ import {
} from "@fosscord/server-util";
import { HTTPError } from "lambert-server";
import Config from "./Config";
import * as Config from "./Config";
import { emitEvent } from "./Event";
import { getPublicUser } from "./User";
@ -39,7 +39,8 @@ export async function isMember(user_id: string, guild_id: string) {
export async function addMember(user_id: string, guild_id: string, cache?: { guild?: GuildDocument }) {
const user = await getPublicUser(user_id, { guilds: true });
const { maxGuilds } = Config.get().limits.user;
const limitsUserProperties = Config.apiConfig.get('limits.user', {maxGuilds: 100, masxUsername: 32, maxFriends: 1000}) as Config.DefaultOptions;
const { maxGuilds } = limitsUserProperties.limits.user;
if (user.guilds.length >= maxGuilds) {
throw new HTTPError(`You are at the ${maxGuilds} server limit.`, 403);
}

View File

@ -1,5 +1,5 @@
import "missing-native-js-functions";
import Config from "./Config";
import * as Config from "./Config";
const reNUMBER = /[0-9]/g;
const reUPPERCASELETTER = /[A-Z]/g;
@ -17,13 +17,14 @@ const blocklist: string[] = []; // TODO: update ones passwordblocklist is stored
* Returns: 0 > pw > 1
*/
export function check(password: string): number {
const passwordProperties = Config.apiConfig.get('register.password', { minLength: 8, minNumbers: 2, minUpperCase: 2, minSymbols: 0, blockInsecureCommonPasswords: false }) as Config.DefaultOptions;
const {
minLength,
minNumbers,
minUpperCase,
minSymbols,
blockInsecureCommonPasswords,
} = Config.get().register.password;
} = passwordProperties.register.password;
var strength = 0;
// checks for total password len