Merge branch 'master' of https://github.com/fosscord/fosscord-server

.github/workflows/release.yml

@@ -34,27 +34,9 @@ jobs:
         with:
           node-version: 14
       - run: |
-          npm config set ignore-scripts true
+          cd bundle
-          cd util
+          npm run setup
-          npm i
+          npx caxa -i . -m 'This_may_take_a_while_to_run_the_first_time_please_wait...' --output '${{matrix.file}}' -- '{{caxa}}/node_modules/.bin/node' '{{caxa}}/dist/bundle/src/start.js'
-          npm run build
-          npm pack
-          cd ../api
-          npm i ../util/
-          npm run build
-          npm pack
-          cd ../cdn
-          npm i ../util/
-          npm run build
-          npm pack
-          cd ../gateway
-          npm i ../util/
-          npm run build
-          npm pack
-          cd ../bundle
-          npm i ../cdn/fosscord-cdn-1.0.0.tgz ../gateway/fosscord-gateway-1.0.0.tgz ../api/fosscord-api-1.0.0.tgz ../util/fosscord-util-1.0.0.tgz caxa
-          npm run build:bundle
-          npx caxa -i . -m 'This_may_take_a_while_to_run_the_first_time_please_wait...' --output '${{matrix.file}}' -- '{{caxa}}/node_modules/.bin/node' '{{caxa}}/dist/start.js'
           ${{ matrix.package }}
       - uses: actions/upload-artifact@v2
         with:

.gitignore

@@ -6,3 +6,5 @@ api/assets/*.js
 api/assets/*.css
 database.db
 tsconfig.tsbuildinfo
+files/
+.env

Dockerfile

@@ -1,5 +1,4 @@
-FROM alpine
+FROM nikolaik/python-nodejs:latest
-RUN apk add --update nodejs npm
 WORKDIR /usr/src/fosscord-server/
 COPY . .
 WORKDIR /usr/src/fosscord-server/bundle

README.md

@@ -30,6 +30,6 @@ This repository contains:
 
 -   [Contributing](https://docs.fosscord.com/contributing/server/)
 
-## [Download](https://github.com/fosscord/fosscord-server/releases)
+## [Setup](https://docs.fosscord.com/setup/server/)
 
--   _Work in progress_
+- [Download](https://github.com/fosscord/fosscord-server/releases)

api/.vscode/api-snippets.code-snippets

@@ -0,0 +1,29 @@
+{
+	"API Router": {
+		"scope": "javascript,typescript",
+		"prefix": "router",
+		"body": [
+			"import { Router, Response, Request } from \"express\";",
+			"import { route } from \"@fosscord/api\";",
+			"",
+			"const router = Router();",
+			"",
+			"router.get(\"/\", route({}), (req: Request, res: Response) => {",
+			"\tres.json({});",
+			"});",
+			"",
+			"export default router;"
+		],
+		"description": "A basic API router setup for a blank route."
+	},
+	"Route": {
+		"scope": "typescript",
+		"prefix": "route",
+		"body": [
+			"router.get(\"$1\", route({}), (req: Request, res: Response) => {",
+			"\t$2",
+			"});"
+		],
+		"description": "An API endpoint"
+	},
+}

api/Dockerfile

@@ -8,5 +8,5 @@ RUN npm rebuild bcrypt --build-from-source && npm install canvas --build-from-so
 RUN npm install
 COPY . .
 EXPOSE 3001
-RUN npm run build-docker
+RUN npm run build
 CMD ["node", "dist/start.js"]

api/assets/schemas.json

@@ -514,6 +514,12 @@
             "attachments": {
                 "type": "array",
                 "items": {}
+            },
+            "sticker_ids": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                }
             }
         },
         "definitions": {
@@ -2887,47 +2893,324 @@
         },
         "$schema": "http://json-schema.org/draft-07/schema#"
     },
-    "EmojiListResponse": {
+    "EmojiCreateSchema": {
-        "type": "array",
+        "type": "object",
-        "items": {
+        "properties": {
-            "type": "object",
+            "name": {
-            "properties": {
+                "type": "string"
-                "animated": {
+            },
-                    "type": "boolean"
+            "image": {
-                },
+                "type": "string"
-                "available": {
+            },
-                    "type": "boolean"
+            "require_colons": {
-                },
+                "type": [
-                "id": {
+                    "null",
+                    "boolean"
+                ]
+            },
+            "roles": {
+                "type": "array",
+                "items": {
                     "type": "string"
-                },
+                }
-                "managed": {
+            }
-                    "type": "boolean"
+        },
-                },
+        "required": [
-                "name": {
+            "image"
-                    "type": "string"
+        ],
-                },
+        "definitions": {
-                "require_colons": {
+            "ChannelPermissionOverwriteType": {
-                    "type": "boolean"
+                "enum": [
-                },
+                    0,
-                "guild_id": {
+                    1
-                    "type": "string"
+                ],
-                },
+                "type": "number"
-                "roles": {
+            },
-                    "type": "array",
+            "Embed": {
-                    "items": {
+                "type": "object",
+                "properties": {
+                    "title": {
                         "type": "string"
+                    },
+                    "type": {
+                        "enum": [
+                            "article",
+                            "gifv",
+                            "image",
+                            "link",
+                            "rich",
+                            "video"
+                        ],
+                        "type": "string"
+                    },
+                    "description": {
+                        "type": "string"
+                    },
+                    "url": {
+                        "type": "string"
+                    },
+                    "timestamp": {
+                        "type": "string",
+                        "format": "date-time"
+                    },
+                    "color": {
+                        "type": "integer"
+                    },
+                    "footer": {
+                        "type": "object",
+                        "properties": {
+                            "text": {
+                                "type": "string"
+                            },
+                            "icon_url": {
+                                "type": "string"
+                            },
+                            "proxy_icon_url": {
+                                "type": "string"
+                            }
+                        },
+                        "required": [
+                            "text"
+                        ]
+                    },
+                    "image": {
+                        "$ref": "#/definitions/EmbedImage"
+                    },
+                    "thumbnail": {
+                        "$ref": "#/definitions/EmbedImage"
+                    },
+                    "video": {
+                        "$ref": "#/definitions/EmbedImage"
+                    },
+                    "provider": {
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "type": "string"
+                            },
+                            "url": {
+                                "type": "string"
+                            }
+                        }
+                    },
+                    "author": {
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "type": "string"
+                            },
+                            "url": {
+                                "type": "string"
+                            },
+                            "icon_url": {
+                                "type": "string"
+                            },
+                            "proxy_icon_url": {
+                                "type": "string"
+                            }
+                        }
+                    },
+                    "fields": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": {
+                                    "type": "string"
+                                },
+                                "value": {
+                                    "type": "string"
+                                },
+                                "inline": {
+                                    "type": "boolean"
+                                }
+                            },
+                            "required": [
+                                "name",
+                                "value"
+                            ]
+                        }
                     }
                 }
             },
-            "required": [
+            "EmbedImage": {
-                "animated",
+                "type": "object",
-                "available",
+                "properties": {
-                "id",
+                    "url": {
-                "managed",
+                        "type": "string"
-                "name",
+                    },
-                "require_colons"
+                    "proxy_url": {
-            ]
+                        "type": "string"
+                    },
+                    "height": {
+                        "type": "integer"
+                    },
+                    "width": {
+                        "type": "integer"
+                    }
+                }
+            },
+            "ChannelModifySchema": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "maxLength": 100,
+                        "type": "string"
+                    },
+                    "type": {
+                        "enum": [
+                            0,
+                            1,
+                            10,
+                            11,
+                            12,
+                            13,
+                            2,
+                            3,
+                            4,
+                            5,
+                            6
+                        ],
+                        "type": "number"
+                    },
+                    "topic": {
+                        "type": "string"
+                    },
+                    "icon": {
+                        "type": [
+                            "null",
+                            "string"
+                        ]
+                    },
+                    "bitrate": {
+                        "type": "integer"
+                    },
+                    "user_limit": {
+                        "type": "integer"
+                    },
+                    "rate_limit_per_user": {
+                        "type": "integer"
+                    },
+                    "position": {
+                        "type": "integer"
+                    },
+                    "permission_overwrites": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "id": {
+                                    "type": "string"
+                                },
+                                "type": {
+                                    "$ref": "#/definitions/ChannelPermissionOverwriteType"
+                                },
+                                "allow": {
+                                    "type": "string"
+                                },
+                                "deny": {
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "allow",
+                                "deny",
+                                "id",
+                                "type"
+                            ]
+                        }
+                    },
+                    "parent_id": {
+                        "type": "string"
+                    },
+                    "id": {
+                        "type": "string"
+                    },
+                    "nsfw": {
+                        "type": "boolean"
+                    },
+                    "rtc_region": {
+                        "type": "string"
+                    },
+                    "default_auto_archive_duration": {
+                        "type": "integer"
+                    }
+                }
+            },
+            "UserPublic": {
+                "type": "object",
+                "properties": {
+                    "username": {
+                        "type": "string"
+                    },
+                    "discriminator": {
+                        "type": "string"
+                    },
+                    "id": {
+                        "type": "string"
+                    },
+                    "public_flags": {
+                        "type": "integer"
+                    },
+                    "avatar": {
+                        "type": "string"
+                    },
+                    "accent_color": {
+                        "type": "integer"
+                    },
+                    "banner": {
+                        "type": "string"
+                    },
+                    "bio": {
+                        "type": "string"
+                    },
+                    "bot": {
+                        "type": "boolean"
+                    }
+                },
+                "required": [
+                    "bio",
+                    "bot",
+                    "discriminator",
+                    "id",
+                    "public_flags",
+                    "username"
+                ]
+            },
+            "PublicConnectedAccount": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string"
+                    },
+                    "type": {
+                        "type": "string"
+                    },
+                    "verifie": {
+                        "type": "boolean"
+                    }
+                },
+                "required": [
+                    "name",
+                    "type",
+                    "verifie"
+                ]
+            }
+        },
+        "$schema": "http://json-schema.org/draft-07/schema#"
+    },
+    "EmojiModifySchema": {
+        "type": "object",
+        "properties": {
+            "name": {
+                "type": "string"
+            },
+            "roles": {
+                "type": "array",
+                "items": {
+                    "type": "string"
+                }
+            }
         },
         "definitions": {
             "ChannelPermissionOverwriteType": {
@@ -4470,7 +4753,7 @@
                 "type": "string"
             },
             "permissions": {
-                "type": "bigint"
+                "type": "string"
             },
             "color": {
                 "type": "integer"
@@ -5064,6 +5347,308 @@
         },
         "$schema": "http://json-schema.org/draft-07/schema#"
     },
+    "ModifyGuildStickerSchema": {
+        "type": "object",
+        "properties": {
+            "name": {
+                "minLength": 2,
+                "maxLength": 30,
+                "type": "string"
+            },
+            "description": {
+                "maxLength": 100,
+                "type": "string"
+            },
+            "tags": {
+                "maxLength": 200,
+                "type": "string"
+            }
+        },
+        "required": [
+            "name",
+            "tags"
+        ],
+        "definitions": {
+            "ChannelPermissionOverwriteType": {
+                "enum": [
+                    0,
+                    1
+                ],
+                "type": "number"
+            },
+            "Embed": {
+                "type": "object",
+                "properties": {
+                    "title": {
+                        "type": "string"
+                    },
+                    "type": {
+                        "enum": [
+                            "article",
+                            "gifv",
+                            "image",
+                            "link",
+                            "rich",
+                            "video"
+                        ],
+                        "type": "string"
+                    },
+                    "description": {
+                        "type": "string"
+                    },
+                    "url": {
+                        "type": "string"
+                    },
+                    "timestamp": {
+                        "type": "string",
+                        "format": "date-time"
+                    },
+                    "color": {
+                        "type": "integer"
+                    },
+                    "footer": {
+                        "type": "object",
+                        "properties": {
+                            "text": {
+                                "type": "string"
+                            },
+                            "icon_url": {
+                                "type": "string"
+                            },
+                            "proxy_icon_url": {
+                                "type": "string"
+                            }
+                        },
+                        "required": [
+                            "text"
+                        ]
+                    },
+                    "image": {
+                        "$ref": "#/definitions/EmbedImage"
+                    },
+                    "thumbnail": {
+                        "$ref": "#/definitions/EmbedImage"
+                    },
+                    "video": {
+                        "$ref": "#/definitions/EmbedImage"
+                    },
+                    "provider": {
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "type": "string"
+                            },
+                            "url": {
+                                "type": "string"
+                            }
+                        }
+                    },
+                    "author": {
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "type": "string"
+                            },
+                            "url": {
+                                "type": "string"
+                            },
+                            "icon_url": {
+                                "type": "string"
+                            },
+                            "proxy_icon_url": {
+                                "type": "string"
+                            }
+                        }
+                    },
+                    "fields": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "name": {
+                                    "type": "string"
+                                },
+                                "value": {
+                                    "type": "string"
+                                },
+                                "inline": {
+                                    "type": "boolean"
+                                }
+                            },
+                            "required": [
+                                "name",
+                                "value"
+                            ]
+                        }
+                    }
+                }
+            },
+            "EmbedImage": {
+                "type": "object",
+                "properties": {
+                    "url": {
+                        "type": "string"
+                    },
+                    "proxy_url": {
+                        "type": "string"
+                    },
+                    "height": {
+                        "type": "integer"
+                    },
+                    "width": {
+                        "type": "integer"
+                    }
+                }
+            },
+            "ChannelModifySchema": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "maxLength": 100,
+                        "type": "string"
+                    },
+                    "type": {
+                        "enum": [
+                            0,
+                            1,
+                            10,
+                            11,
+                            12,
+                            13,
+                            2,
+                            3,
+                            4,
+                            5,
+                            6
+                        ],
+                        "type": "number"
+                    },
+                    "topic": {
+                        "type": "string"
+                    },
+                    "icon": {
+                        "type": [
+                            "null",
+                            "string"
+                        ]
+                    },
+                    "bitrate": {
+                        "type": "integer"
+                    },
+                    "user_limit": {
+                        "type": "integer"
+                    },
+                    "rate_limit_per_user": {
+                        "type": "integer"
+                    },
+                    "position": {
+                        "type": "integer"
+                    },
+                    "permission_overwrites": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "id": {
+                                    "type": "string"
+                                },
+                                "type": {
+                                    "$ref": "#/definitions/ChannelPermissionOverwriteType"
+                                },
+                                "allow": {
+                                    "type": "string"
+                                },
+                                "deny": {
+                                    "type": "string"
+                                }
+                            },
+                            "required": [
+                                "allow",
+                                "deny",
+                                "id",
+                                "type"
+                            ]
+                        }
+                    },
+                    "parent_id": {
+                        "type": "string"
+                    },
+                    "id": {
+                        "type": "string"
+                    },
+                    "nsfw": {
+                        "type": "boolean"
+                    },
+                    "rtc_region": {
+                        "type": "string"
+                    },
+                    "default_auto_archive_duration": {
+                        "type": "integer"
+                    }
+                }
+            },
+            "UserPublic": {
+                "type": "object",
+                "properties": {
+                    "username": {
+                        "type": "string"
+                    },
+                    "discriminator": {
+                        "type": "string"
+                    },
+                    "id": {
+                        "type": "string"
+                    },
+                    "public_flags": {
+                        "type": "integer"
+                    },
+                    "avatar": {
+                        "type": "string"
+                    },
+                    "accent_color": {
+                        "type": "integer"
+                    },
+                    "banner": {
+                        "type": "string"
+                    },
+                    "bio": {
+                        "type": "string"
+                    },
+                    "bot": {
+                        "type": "boolean"
+                    }
+                },
+                "required": [
+                    "bio",
+                    "bot",
+                    "discriminator",
+                    "id",
+                    "public_flags",
+                    "username"
+                ]
+            },
+            "PublicConnectedAccount": {
+                "type": "object",
+                "properties": {
+                    "name": {
+                        "type": "string"
+                    },
+                    "type": {
+                        "type": "string"
+                    },
+                    "verifie": {
+                        "type": "boolean"
+                    }
+                },
+                "required": [
+                    "name",
+                    "type",
+                    "verifie"
+                ]
+            }
+        },
+        "$schema": "http://json-schema.org/draft-07/schema#"
+    },
     "TemplateCreateSchema": {
         "type": "object",
         "properties": {

api/client_test/index.html

@@ -5,6 +5,7 @@
 		<meta name="viewport" content="width=device-width, initial-scale=1.0" />
 		<title>Discord Test Client</title>
 	</head>
+
 	<body>
 		<div id="app-mount"></div>
 		<script>
@@ -36,6 +37,7 @@
 				HTML_TIMESTAMP: Date.now(),
 				ALGOLIA_KEY: "aca0d7082e4e63af5ba5917d5e96bed0"
 			};
+			GLOBAL_ENV.MEDIA_PROXY_ENDPOINT = location.protocol + "//" + GLOBAL_ENV.CDN_HOST;
 			const localStorage = window.localStorage;
 			// TODO: remote auth
 			// window.GLOBAL_ENV.REMOTE_AUTH_ENDPOINT = window.GLOBAL_ENV.GATEWAY_ENDPOINT.replace(/wss?:/, "");
@@ -46,12 +48,52 @@
 			);
 
 			// Auto register guest account:
+			const prefix = [
+				"mysterious",
+				"adventurous",
+				"courageous",
+				"precious",
+				"cynical",
+				"despicable",
+				"suspicious",
+				"gorgeous",
+				"lovely",
+				"stunning",
+				"based",
+				"keyed",
+				"ratioed",
+				"twink",
+				"phoned"
+			];
+			const suffix = [
+				"Anonymous",
+				"Lurker",
+				"User",
+				"Enjoyer",
+				"Hunk",
+				"Top",
+				"Bottom",
+				"Sub",
+				"Coolstar",
+				"Wrestling",
+				"TylerTheCreator",
+				"Ad"
+			];
+
+			Array.prototype.random = function () {
+				return this[Math.floor(Math.random() * this.length)];
+			};
+
+			function _generateName() {
+				return `${prefix.random()}${suffix.random()}`;
+			}
+
 			const token = JSON.parse(localStorage.getItem("token"));
 			if (!token && location.pathname !== "/login" && location.pathname !== "/register") {
 				fetch(`${window.GLOBAL_ENV.API_ENDPOINT}/auth/register`, {
 					method: "POST",
 					headers: { "content-type": "application/json" },
-					body: JSON.stringify({ username: "Anonymous", consent: true })
+					body: JSON.stringify({ username: `${_generateName()}`, consent: true }) //${Date.now().toString().slice(-4)}
 				})
 					.then((x) => x.json())
 					.then((x) => {
@@ -64,7 +106,8 @@
 			}
 
 			const settings = JSON.parse(localStorage.getItem("UserSettingsStore"));
-			if (settings && settings.locale === "en") {
+			if (settings && settings.locale.length <= 2) {
+				// fix client locale wrong and client not loading at all
 				settings.locale = "en-US";
 				localStorage.setItem("UserSettingsStore", JSON.stringify(settings));
 			}

api/package.json

@@ -10,8 +10,7 @@
 		"test": "npm run build && npm run test:only",
 		"test:watch": "jest --watch",
 		"start": "npm run build && node dist/start",
-		"build": "npx tsc -b .",
+		"build": "npx tsc -p .",
-		"build-docker": "tsc -p tsconfig-docker.json",
 		"dev": "tsnd --respawn src/start.ts",
 		"patch": "ts-patch install -s && npx patch-package",
 		"postinstall": "npm run patch",
@@ -38,10 +37,8 @@
 	"homepage": "https://fosscord.com",
 	"devDependencies": {
 		"@babel/core": "^7.15.5",
-		"@babel/preset-env": "^7.15.6",
+		"@babel/preset-env": "^7.15.8",
 		"@babel/preset-typescript": "^7.15.0",
-		"@swc/cli": "^0.1.51",
-		"@swc/core": "^1.2.93",
 		"@types/amqplib": "^0.8.1",
 		"@types/bcrypt": "^5.0.0",
 		"@types/express": "^4.17.9",
@@ -49,65 +46,48 @@
 		"@types/jest": "^27.0.1",
 		"@types/jest-expect-message": "^1.0.3",
 		"@types/jsonwebtoken": "^8.5.0",
-		"@types/mongodb": "^3.6.9",
+		"@types/morgan": "^1.9.3",
-		"@types/mongoose": "^5.10.5",
-		"@types/mongoose-autopopulate": "^0.10.1",
-		"@types/mongoose-lean-virtuals": "^0.5.1",
 		"@types/multer": "^1.4.5",
 		"@types/node": "^14.17.9",
 		"@types/node-fetch": "^2.5.7",
 		"@types/supertest": "^2.0.11",
 		"@zerollup/ts-transform-paths": "^1.7.18",
-		"0x": "^4.10.2",
+		"jest": "^27.2.5",
-		"babel-jest": "^27.2.0",
-		"caxa": "^2.1.0",
-		"image-size": "^1.0.0",
-		"jest": "^26.6.3",
 		"jest-expect-message": "^1.0.2",
 		"jest-runtime": "^27.2.1",
-		"saslprep": "^1.0.3",
 		"ts-node": "^9.1.1",
 		"ts-node-dev": "^1.1.6",
 		"ts-patch": "^1.4.4",
-		"tsup": "^5.4.0",
 		"typescript": "^4.4.2",
 		"typescript-json-schema": "0.50.1"
 	},
 	"dependencies": {
+		"@babel/preset-env": "^7.15.8",
+		"@babel/preset-typescript": "^7.15.0",
 		"@fosscord/util": "file:../util",
-		"@types/morgan": "^1.9.3",
 		"ajv": "8.6.2",
 		"ajv-formats": "^2.1.1",
 		"amqplib": "^0.8.0",
 		"assert": "^1.5.0",
-		"atomically": "^1.7.0",
 		"bcrypt": "^5.0.1",
 		"body-parser": "^1.19.0",
-		"cheerio": "^1.0.0-rc.9",
+		"cheerio": "^1.0.0-rc.10",
-		"dot-prop": "^6.0.1",
 		"dotenv": "^8.2.0",
-		"env-paths": "^2.2.1",
-		"esbuild": "^0.13.4",
 		"express": "^4.17.1",
-		"express-validator": "^6.9.2",
 		"form-data": "^3.0.0",
 		"i18next": "^19.9.2",
 		"i18next-http-middleware": "^3.1.3",
 		"i18next-node-fs-backend": "^2.1.3",
+		"image-size": "^1.0.0",
 		"jsonwebtoken": "^8.5.1",
-		"lambert-server": "^1.2.11",
+		"lambert-server": "^1.2.12",
-		"missing-native-js-functions": "^1.2.17",
+		"missing-native-js-functions": "^1.2.18",
-		"mongoose": "^5.12.3",
-		"mongoose-autopopulate": "^0.12.3",
-		"mongoose-long": "^0.3.2",
 		"morgan": "^1.10.0",
 		"multer": "^1.4.2",
 		"node-fetch": "^2.6.1",
 		"patch-package": "^6.4.7",
 		"supertest": "^6.1.6",
-		"tsconfig-paths": "^3.11.0",
+		"typeorm": "^0.2.37"
-		"typeorm": "^0.2.37",
-		"wsc": "^0.3.0"
 	},
 	"jest": {
 		"setupFiles": [

api/src/Server.ts

@@ -1,19 +1,17 @@
-import { OptionsJson } from "body-parser";
 import "missing-native-js-functions";
-import { Connection } from "mongoose";
 import { Server, ServerOptions } from "lambert-server";
 import { Authentication, CORS } from "./middlewares/";
 import { Config, initDatabase, initEvent } from "@fosscord/util";
 import { ErrorHandler } from "./middlewares/ErrorHandler";
 import { BodyParser } from "./middlewares/BodyParser";
 import { Router, Request, Response, NextFunction } from "express";
-import mongoose from "mongoose";
 import path from "path";
 import { initRateLimits } from "./middlewares/RateLimit";
 import TestClient from "./middlewares/TestClient";
 import { initTranslation } from "./middlewares/Translation";
 import morgan from "morgan";
 import { initInstance } from "./util/Instance";
+import { registerRoutes } from "@fosscord/util";
 
 export interface FosscordServerOptions extends ServerOptions {}
 
@@ -75,12 +73,12 @@ export class FosscordServer extends Server {
 		await initRateLimits(api);
 		await initTranslation(api);
 
-		this.routes = await this.registerRoutes(path.join(__dirname, "routes", "/"));
+		this.routes = await registerRoutes(this, path.join(__dirname, "routes", "/"));
 
 		api.use("*", (error: any, req: Request, res: Response, next: NextFunction) => {
 			if (error) return next(error);
 			res.status(404).json({
-				message: "404: Not Found",
+				message: "404 endpoint not found",
 				code: 0
 			});
 			next();

api/src/middlewares/Authentication.ts

@@ -9,6 +9,8 @@ export const NO_AUTHORIZATION_ROUTES = [
 	"/ping",
 	"/gateway",
 	"/experiments",
+	"/-/readyz",
+	"/-/healthz",
 	/\/guilds\/\d+\/widget\.(json|png)/
 ];
 

api/src/routes/-/healthz.ts

@@ -0,0 +1,17 @@
+import { Router, Response, Request } from "express";
+import { route } from "@fosscord/api";
+import { getConnection } from "typeorm";
+
+const router = Router();
+
+router.get("/", route({}), (req: Request, res: Response) => {
+	try {
+		// test that the database is alive & responding
+		getConnection();
+		return res.sendStatus(200);
+	} catch(e) {
+		res.sendStatus(503);
+	}
+});
+
+export default router;

api/src/routes/-/readyz.ts

@@ -0,0 +1,17 @@
+import { Router, Response, Request } from "express";
+import { route } from "@fosscord/api";
+import { getConnection } from "typeorm";
+
+const router = Router();
+
+router.get("/", route({}), (req: Request, res: Response) => {
+	try {
+		// test that the database is alive & responding
+		getConnection();
+		return res.sendStatus(200);
+	} catch(e) {
+		res.sendStatus(503);
+	}
+});
+
+export default router;

api/src/routes/channels/#channel_id/invites.ts

@@ -2,7 +2,7 @@ import { Router, Request, Response } from "express";
 import { HTTPError } from "lambert-server";
 import { route } from "@fosscord/api";
 import { random } from "@fosscord/api";
-import { getPermission, Channel, Invite, InviteCreateEvent, emitEvent, User, Guild, PublicInviteRelation } from "@fosscord/util";
+import { Channel, Invite, InviteCreateEvent, emitEvent, User, Guild, PublicInviteRelation } from "@fosscord/util";
 import { isTextChannel } from "./messages";
 
 const router: Router = Router();

api/src/routes/channels/#channel_id/messages/index.ts

@@ -10,7 +10,8 @@ import {
 	getPermission,
 	Message,
 	MessageCreateEvent,
-	uploadFile
+	uploadFile,
+	Member
 } from "@fosscord/util";
 import { HTTPError } from "lambert-server";
 import { handleMessage, postHandleMessage, route } from "@fosscord/api";
@@ -22,7 +23,7 @@ const router: Router = Router();
 
 export default router;
 
-function isTextChannel(type: ChannelType): boolean {
+export function isTextChannel(type: ChannelType): boolean {
 	switch (type) {
 		case ChannelType.GUILD_STORE:
 		case ChannelType.GUILD_VOICE:
@@ -39,7 +40,6 @@ function isTextChannel(type: ChannelType): boolean {
 			return true;
 	}
 }
-module.exports.isTextChannel = isTextChannel;
 
 export interface MessageCreateSchema {
 	content?: string;
@@ -64,6 +64,7 @@ export interface MessageCreateSchema {
 	payload_json?: string;
 	file?: any;
 	attachments?: any[]; //TODO we should create an interface for attachments
+	sticker_ids?: string[];
 }
 
 // https://discord.com/developers/docs/resources/channel#create-message
@@ -187,33 +188,34 @@ router.post(
 
 		message = await message.save();
 
-		await channel.assign({ last_message_id: message.id }).save();
-
 		if (channel.isDm()) {
 			const channel_dto = await DmChannelDTO.from(channel);
 
-			for (let recipient of channel.recipients!) {
-				if (recipient.closed) {
-					await emitEvent({
-						event: "CHANNEL_CREATE",
-						data: channel_dto.excludedRecipients([recipient.user_id]),
-						user_id: recipient.user_id
-					});
-				}
-			}
-
 			//Only one recipients should be closed here, since in group DMs the recipient is deleted not closed
+
 			await Promise.all(
-				channel
+				channel.recipients!.map((recipient) => {
-					.recipients!.filter((r) => r.closed)
+					if (recipient.closed) {
-					.map(async (r) => {
+						recipient.closed = false;
-						r.closed = false;
+						return Promise.all([
-						return await r.save();
+							recipient.save(),
-					})
+							emitEvent({
+								event: "CHANNEL_CREATE",
+								data: channel_dto.excludedRecipients([recipient.user_id]),
+								user_id: recipient.user_id
+							})
+						]);
+					}
+				})
 			);
 		}
 
-		await emitEvent({ event: "MESSAGE_CREATE", channel_id: channel_id, data: message } as MessageCreateEvent);
+		await Promise.all([
+			channel.assign({ last_message_id: message.id }).save(),
+			message.guild_id ? Member.update({ id: req.user_id, guild_id: message.guild_id }, { last_message_id: message.id }) : null,
+			emitEvent({ event: "MESSAGE_CREATE", channel_id: channel_id, data: message } as MessageCreateEvent)
+		]);
+
 		postHandleMessage(message).catch((e) => {}); // no await as it shouldnt block the message send function and silently catch error
 
 		return res.json(message);

api/src/routes/channels/#channel_id/permissions.ts

@@ -44,8 +44,8 @@ router.put(
 			};
 			channel.permission_overwrites!.push(overwrite);
 		}
-		overwrite.allow = String(req.permission!.bitfield & (BigInt(body.allow) || 0n));
+		overwrite.allow = String(req.permission!.bitfield & (BigInt(body.allow) || BigInt("0")));
-		overwrite.deny = String(req.permission!.bitfield & (BigInt(body.deny) || 0n));
+		overwrite.deny = String(req.permission!.bitfield & (BigInt(body.deny) || BigInt("0")));
 
 		await Promise.all([
 			channel.save(),

api/src/routes/channels/#channel_id/typing.ts

@@ -9,14 +9,13 @@ router.post("/", route({ permission: "SEND_MESSAGES" }), async (req: Request, re
 	const user_id = req.user_id;
 	const timestamp = Date.now();
 	const channel = await Channel.findOneOrFail({ id: channel_id });
-	const member = await Member.findOneOrFail({ where: { id: user_id }, relations: ["roles", "user"] });
+	const member = await Member.findOne({ where: { id: user_id, guild_id: channel.guild_id }, relations: ["roles", "user"] });
 
 	await emitEvent({
 		event: "TYPING_START",
 		channel_id: channel_id,
 		data: {
-			// this is the paylod
+			...(member ? { member: { ...member, roles: member?.roles?.map((x) => x.id) } } : null),
-			member: { ...member, roles: member.roles?.map((x) => x.id) },
 			channel_id,
 			timestamp,
 			user_id,

api/src/routes/gateway/bot.ts

@@ -1,15 +1,29 @@
 import { Config } from "@fosscord/util";
 import { Router, Response, Request } from "express";
-import { route } from "@fosscord/api";
+import { route, RouteOptions } from "@fosscord/api";
 
 const router = Router();
 
-router.get("/", route({}), (req: Request, res: Response) => {
+export interface GatewayBotResponse {
-	const { endpointPublic } = Config.get().gateway;
+	url: string;
-	res.json({ url: endpointPublic || process.env.GATEWAY || "ws://localhost:3002" });
+	shards: number;
-});
+	session_start_limit: {
+		total: number;
+		remaining: number;
+		reset_after: number;
+		max_concurrency: number;
+	}
+}
 
-router.get("/bot", route({}), (req: Request, res: Response) => {
+const options: RouteOptions = {
+	test: {
+		response: {
+			body: "GatewayBotResponse"
+		}
+	}
+};
+
+router.get("/", route(options), (req: Request, res: Response) => {
 	const { endpointPublic } = Config.get().gateway;
 	res.json({
 		url: endpointPublic || process.env.GATEWAY || "ws://localhost:3002",

api/src/routes/gateway/index.ts

@@ -0,0 +1,24 @@
+import { Config } from "@fosscord/util";
+import { Router, Response, Request } from "express";
+import { route, RouteOptions } from "@fosscord/api";
+
+const router = Router();
+
+export interface GatewayResponse {
+	url: string;
+}
+
+const options: RouteOptions = {
+	test: {
+		response: {
+			body: "GatewayResponse"
+		}
+	}
+};
+
+router.get("/", route(options), (req: Request, res: Response) => {
+	const { endpointPublic } = Config.get().gateway;
+	res.json({ url: endpointPublic || process.env.GATEWAY || "ws://localhost:3002" });
+});
+
+export default router;

api/src/routes/gifs/search.ts

@@ -1,37 +1,24 @@
 import { Router, Response, Request } from "express";
 import fetch from "node-fetch";
 import { route } from "@fosscord/api";
+import { getGifApiKey, parseGifResult } from "./trending";
 
 const router = Router();
 
 router.get("/", route({}), async (req: Request, res: Response) => {
-	// TODO: Custom providers and code quality
+	// TODO: Custom providers
-	const { q, media_format, locale, provider } = req.query;
+	const { q, media_format, locale } = req.query;
 
-	const parseResult = (result: any) => {
+	const apiKey = getGifApiKey();
-		return {
-			id: result.id,
-			title: result.title,
-			url: result.itemurl,
-			src: result.media[0].mp4.url,
-			gif_src: result.media[0].gif.url,
-			width: result.media[0].mp4.dims[0],
-			height: result.media[0].mp4.dims[1],
-			preview: result.media[0].mp4.preview
-		};
-	};
 
-	const response = await fetch(`https://g.tenor.com/v1/search?q=${q}&media_format=${media_format}&locale=${locale}&key=LIVDSRZULELA`, {
+	const response = await fetch(`https://g.tenor.com/v1/search?q=${q}&media_format=${media_format}&locale=${locale}&key=${apiKey}`, {
 		method: "get",
 		headers: { "Content-Type": "application/json" }
 	});
 
 	const { results } = await response.json();
-	let cache = new Array() as any[];
+
-	results.forEach((result: any) => {
+	res.json(results.map(parseGifResult)).status(200);
-		cache.push(parseResult(result));
-	});
-	res.json(cache).status(200);
 });
 
 export default router;

api/src/routes/gifs/trending-gifs.ts

@@ -1,37 +1,24 @@
 import { Router, Response, Request } from "express";
 import fetch from "node-fetch";
 import { route } from "@fosscord/api";
+import { getGifApiKey, parseGifResult } from "./trending";
 
 const router = Router();
 
 router.get("/", route({}), async (req: Request, res: Response) => {
-	// TODO: Custom providers and code quality
+	// TODO: Custom providers
-	const { media_format, locale, provider } = req.query;
+	const { media_format, locale } = req.query;
 
-	const parseResult = (result: any) => {
+	const apiKey = getGifApiKey();
-		return {
-			id: result.id,
-			title: result.title,
-			url: result.itemurl,
-			src: result.media[0].mp4.url,
-			gif_src: result.media[0].gif.url,
-			width: result.media[0].mp4.dims[0],
-			height: result.media[0].mp4.dims[1],
-			preview: result.media[0].mp4.preview
-		};
-	};
 
-	const response = await fetch(`https://g.tenor.com/v1/trending?media_format=${media_format}&locale=${locale}&key=LIVDSRZULELA`, {
+	const response = await fetch(`https://g.tenor.com/v1/trending?media_format=${media_format}&locale=${locale}&key=${apiKey}`, {
 		method: "get",
 		headers: { "Content-Type": "application/json" }
 	});
 
 	const { results } = await response.json();
-	let cache = new Array() as any[];
+
-	results.forEach((result: any) => {
+	res.json(results.map(parseGifResult)).status(200);
-		cache.push(parseResult(result));
-	});
-	res.json(cache).status(200);
 });
 
 export default router;

api/src/routes/gifs/trending.ts

@@ -1,48 +1,57 @@
 import { Router, Response, Request } from "express";
 import fetch from "node-fetch";
 import { route } from "@fosscord/api";
+import { Config } from "@fosscord/util";
+import { HTTPError } from "lambert-server";
 
 const router = Router();
 
-router.get("/", route({}), async (req: Request, res: Response) => {
+export function parseGifResult(result: any) {
-	// TODO: Custom providers and code quality
+	return {
-	const { media_format, locale, provider } = req.query;
+		id: result.id,
-
+		title: result.title,
-	const parseResult = (result: any) => {
+		url: result.itemurl,
-		return {
+		src: result.media[0].mp4.url,
-			id: result.id,
+		gif_src: result.media[0].gif.url,
-			title: result.title,
+		width: result.media[0].mp4.dims[0],
-			url: result.itemurl,
+		height: result.media[0].mp4.dims[1],
-			src: result.media[0].mp4.url,
+		preview: result.media[0].mp4.preview
-			gif_src: result.media[0].gif.url,
-			width: result.media[0].mp4.dims[0],
-			height: result.media[0].mp4.dims[1],
-			preview: result.media[0].mp4.preview
-		};
 	};
+}
 
-	const responseSource = await fetch(`https://g.tenor.com/v1/categories?media_format=${media_format}&locale=${locale}&key=LIVDSRZULELA`, {
+export function getGifApiKey() {
-		method: "get",
+	const { enabled, provider, apiKey } = Config.get().gif;
-		headers: { "Content-Type": "application/json" }
+	if (!enabled) throw new HTTPError(`Gifs are disabled`);
-	});
+	if (provider !== "tenor" || !apiKey) throw new HTTPError(`${provider} gif provider not supported`);
 
-	const trendGifSource = await fetch(`https://g.tenor.com/v1/trending?media_format=${media_format}&locale=${locale}&key=LIVDSRZULELA`, {
+	return apiKey;
-		method: "get",
+}
-		headers: { "Content-Type": "application/json" }
+
-	});
+router.get("/", route({}), async (req: Request, res: Response) => {
+	// TODO: Custom providers
+	// TODO: return gifs as mp4
+	const { media_format, locale } = req.query;
+
+	const apiKey = getGifApiKey();
+
+	const [responseSource, trendGifSource] = await Promise.all([
+		fetch(`https://g.tenor.com/v1/categories?locale=${locale}&key=${apiKey}`, {
+			method: "get",
+			headers: { "Content-Type": "application/json" }
+		}),
+		fetch(`https://g.tenor.com/v1/trending?locale=${locale}&key=${apiKey}`, {
+			method: "get",
+			headers: { "Content-Type": "application/json" }
+		})
+	]);
 
 	const { tags } = await responseSource.json();
 	const { results } = await trendGifSource.json();
-	let cache = new Array() as any[];
 
-	tags.forEach((result: any) => {
+	res.json({
-		cache.push({
+		categories: tags.map((x: any) => ({ name: x.searchterm, src: x.image })),
-			name: result.searchterm,
+		gifs: [parseGifResult(results[0])]
-			src: result.image
+	}).status(200);
-		});
-	});
-
-	res.json({ categories: [cache], gifs: [parseResult(results[0])] }).status(200);
 });
 
 export default router;

api/src/routes/guilds/#guild_id/channels.ts

@@ -31,10 +31,10 @@ router.patch("/", route({ body: "ChannelReorderSchema", permission: "MANAGE_CHAN
 
 	await Promise.all([
 		body.map(async (x) => {
-			if (!x.position && !x.parent_id) throw new HTTPError(`You need to at least specify position or parent_id`, 400);
+			if (x.position == null && !x.parent_id) throw new HTTPError(`You need to at least specify position or parent_id`, 400);
 
 			const opts: any = {};
-			if (x.position) opts.position = x.position;
+			if (x.position != null) opts.position = x.position;
 
 			if (x.parent_id) {
 				opts.parent_id = x.parent_id;

api/src/routes/guilds/#guild_id/emojis.ts

@@ -0,0 +1,118 @@
+import { Router, Request, Response } from "express";
+import { Config, DiscordApiErrors, emitEvent, Emoji, GuildEmojisUpdateEvent, handleFile, Member, Snowflake, User } from "@fosscord/util";
+import { route } from "@fosscord/api";
+
+const router = Router();
+
+export interface EmojiCreateSchema {
+	name?: string;
+	image: string;
+	require_colons?: boolean | null;
+	roles?: string[];
+}
+
+export interface EmojiModifySchema {
+	name?: string;
+	roles?: string[];
+}
+
+router.get("/", route({}), async (req: Request, res: Response) => {
+	const { guild_id } = req.params;
+
+	await Member.IsInGuildOrFail(req.user_id, guild_id);
+
+	const emojis = await Emoji.find({ where: { guild_id: guild_id }, relations: ["user"] });
+
+	return res.json(emojis);
+});
+
+router.get("/:emoji_id", route({}), async (req: Request, res: Response) => {
+	const { guild_id, emoji_id } = req.params;
+
+	await Member.IsInGuildOrFail(req.user_id, guild_id);
+
+	const emoji = await Emoji.findOneOrFail({ where: { guild_id: guild_id, id: emoji_id }, relations: ["user"] });
+
+	return res.json(emoji);
+});
+
+router.post("/", route({ body: "EmojiCreateSchema", permission: "MANAGE_EMOJIS_AND_STICKERS" }), async (req: Request, res: Response) => {
+	const { guild_id } = req.params;
+	const body = req.body as EmojiCreateSchema;
+
+	const id = Snowflake.generate();
+	const emoji_count = await Emoji.count({ guild_id: guild_id });
+	const { maxEmojis } = Config.get().limits.guild;
+
+	if (emoji_count >= maxEmojis) throw DiscordApiErrors.MAXIMUM_NUMBER_OF_EMOJIS_REACHED.withParams(maxEmojis);
+	if (body.require_colons == null) body.require_colons = true;
+
+	const user = await User.findOneOrFail({ id: req.user_id });
+	body.image = (await handleFile(`/emojis/${id}`, body.image)) as string;
+
+	const emoji = await new Emoji({
+		id: id,
+		guild_id: guild_id,
+		...body,
+		user: user,
+		managed: false,
+		animated: false, // TODO: Add support animated emojis
+		available: true,
+		roles: []
+	}).save();
+
+	await emitEvent({
+		event: "GUILD_EMOJIS_UPDATE",
+		guild_id: guild_id,
+		data: {
+			guild_id: guild_id,
+			emojis: await Emoji.find({ guild_id: guild_id })
+		}
+	} as GuildEmojisUpdateEvent);
+
+	return res.status(201).json(emoji);
+});
+
+router.patch(
+	"/:emoji_id",
+	route({ body: "EmojiModifySchema", permission: "MANAGE_EMOJIS_AND_STICKERS" }),
+	async (req: Request, res: Response) => {
+		const { emoji_id, guild_id } = req.params;
+		const body = req.body as EmojiModifySchema;
+
+		const emoji = await new Emoji({ ...body, id: emoji_id, guild_id: guild_id }).save();
+
+		await emitEvent({
+			event: "GUILD_EMOJIS_UPDATE",
+			guild_id: guild_id,
+			data: {
+				guild_id: guild_id,
+				emojis: await Emoji.find({ guild_id: guild_id })
+			}
+		} as GuildEmojisUpdateEvent);
+
+		return res.json(emoji);
+	}
+);
+
+router.delete("/:emoji_id", route({ permission: "MANAGE_EMOJIS_AND_STICKERS" }), async (req: Request, res: Response) => {
+	const { emoji_id, guild_id } = req.params;
+
+	await Emoji.delete({
+		id: emoji_id,
+		guild_id: guild_id
+	});
+
+	await emitEvent({
+		event: "GUILD_EMOJIS_UPDATE",
+		guild_id: guild_id,
+		data: {
+			guild_id: guild_id,
+			emojis: await Emoji.find({ guild_id: guild_id })
+		}
+	} as GuildEmojisUpdateEvent);
+
+	res.sendStatus(204);
+});
+
+export default router;

api/src/routes/guilds/#guild_id/premium.ts

@@ -0,0 +1,10 @@
+import { Router, Request, Response } from "express";
+import { route } from "@fosscord/api";
+const router = Router();
+
+router.get("/subscriptions", route({}), async (req: Request, res: Response) => {
+	// TODO:
+	res.json([]);
+});
+
+export default router;

api/src/routes/guilds/#guild_id/prune.ts

@@ -0,0 +1,82 @@
+import { Router, Request, Response } from "express";
+import { Guild, Member, Snowflake } from "@fosscord/util";
+import { LessThan, IsNull } from "typeorm";
+import { route } from "@fosscord/api";
+const router = Router();
+
+//Returns all inactive members, respecting role hierarchy
+export const inactiveMembers = async (guild_id: string, user_id: string, days: number, roles: string[] = []) => {
+	var date = new Date();
+	date.setDate(date.getDate() - days);
+	//Snowflake should have `generateFromTime` method? Or similar?
+	var minId = BigInt(date.valueOf() - Snowflake.EPOCH) << BigInt(22);
+
+	var members = await Member.find({
+		where: [
+			{
+				guild_id,
+				last_message_id: LessThan(minId.toString())
+			},
+			{
+				last_message_id: IsNull()
+			}
+		],
+		relations: ["roles"]
+	});
+	console.log(members);
+	if (!members.length) return [];
+
+	//I'm sure I can do this in the above db query ( and it would probably be better to do so ), but oh well.
+	if (roles.length && members.length) members = members.filter((user) => user.roles?.some((role) => roles.includes(role.id)));
+
+	const me = await Member.findOneOrFail({ id: user_id, guild_id }, { relations: ["roles"] });
+	const myHighestRole = Math.max(...(me.roles?.map((x) => x.position) || []));
+
+	const guild = await Guild.findOneOrFail({ where: { id: guild_id } });
+
+	members = members.filter(
+		(member) =>
+			member.id !== guild.owner_id && //can't kick owner
+			member.roles?.some(
+				(role) =>
+					role.position < myHighestRole || //roles higher than me can't be kicked
+					me.id === guild.owner_id //owner can kick anyone
+			)
+	);
+
+	return members;
+};
+
+router.get("/", route({ permission: "KICK_MEMBERS" }), async (req: Request, res: Response) => {
+	const days = parseInt(req.query.days as string);
+
+	var roles = req.query.include_roles;
+	if (typeof roles === "string") roles = [roles]; //express will return array otherwise
+
+	const members = await inactiveMembers(req.params.guild_id, req.user_id, days, roles as string[]);
+
+	res.send({ pruned: members.length });
+});
+
+export interface PruneSchema {
+	/**
+	 * @min 0
+	 */
+	days: number;
+}
+
+router.post("/", route({ permission: "KICK_MEMBERS" }), async (req: Request, res: Response) => {
+	const days = parseInt(req.body.days);
+
+	var roles = req.query.include_roles;
+	if (typeof roles === "string") roles = [roles];
+
+	const { guild_id } = req.params;
+	const members = await inactiveMembers(guild_id, req.user_id, days, roles as string[]);
+
+	await Promise.all(members.map((x) => Member.removeFromGuild(x.id, guild_id)));
+
+	res.send({ purged: members.length });
+});
+
+export default router;

api/src/routes/guilds/#guild_id/roles.ts

@@ -17,7 +17,7 @@ const router: Router = Router();
 
 export interface RoleModifySchema {
 	name?: string;
-	permissions?: bigint;
+	permissions?: string;
 	color?: number;
 	hoist?: boolean; // whether the role should be displayed separately in the sidebar
 	mentionable?: boolean; // whether the role should be mentionable
@@ -57,7 +57,7 @@ router.post("/", route({ body: "RoleModifySchema", permission: "MANAGE_ROLES" })
 		...body,
 		guild_id: guild_id,
 		managed: false,
-		permissions: String(req.permission!.bitfield & (body.permissions || 0n)),
+		permissions: String(req.permission!.bitfield & BigInt(body.permissions || "0")),
 		tags: undefined
 	});
 
@@ -105,7 +105,12 @@ router.patch("/:role_id", route({ body: "RoleModifySchema", permission: "MANAGE_
 	const { role_id, guild_id } = req.params;
 	const body = req.body as RoleModifySchema;
 
-	const role = new Role({ ...body, id: role_id, guild_id, permissions: String(req.permission!.bitfield & (body.permissions || 0n)) });
+	const role = new Role({
+		...body,
+		id: role_id,
+		guild_id,
+		permissions: String(req.permission!.bitfield & BigInt(body.permissions || "0"))
+	});
 
 	await Promise.all([
 		role.save(),

api/src/routes/guilds/#guild_id/stickers.ts

@@ -0,0 +1,135 @@
+import {
+	emitEvent,
+	GuildStickersUpdateEvent,
+	handleFile,
+	Member,
+	Snowflake,
+	Sticker,
+	StickerFormatType,
+	StickerType,
+	uploadFile
+} from "@fosscord/util";
+import { Router, Request, Response } from "express";
+import { route } from "@fosscord/api";
+import multer from "multer";
+import { HTTPError } from "lambert-server";
+const router = Router();
+
+router.get("/", route({}), async (req: Request, res: Response) => {
+	const { guild_id } = req.params;
+	await Member.IsInGuildOrFail(req.user_id, guild_id);
+
+	res.json(await Sticker.find({ guild_id }));
+});
+
+const bodyParser = multer({
+	limits: {
+		fileSize: 1024 * 1024 * 100,
+		fields: 10,
+		files: 1
+	},
+	storage: multer.memoryStorage()
+}).single("file");
+
+router.post(
+	"/",
+	bodyParser,
+	route({ permission: "MANAGE_EMOJIS_AND_STICKERS", body: "ModifyGuildStickerSchema" }),
+	async (req: Request, res: Response) => {
+		if (!req.file) throw new HTTPError("missing file");
+
+		const { guild_id } = req.params;
+		const body = req.body as ModifyGuildStickerSchema;
+		const id = Snowflake.generate();
+
+		const [sticker] = await Promise.all([
+			new Sticker({
+				...body,
+				guild_id,
+				id,
+				type: StickerType.GUILD,
+				format_type: getStickerFormat(req.file.mimetype),
+				available: true
+			}).save(),
+			uploadFile(`/stickers/${id}`, req.file)
+		]);
+
+		await sendStickerUpdateEvent(guild_id);
+
+		res.json(sticker);
+	}
+);
+
+export function getStickerFormat(mime_type: string) {
+	switch (mime_type) {
+		case "image/apng":
+			return StickerFormatType.APNG;
+		case "application/json":
+			return StickerFormatType.LOTTIE;
+		case "image/png":
+			return StickerFormatType.PNG;
+		case "image/gif":
+			return StickerFormatType.GIF;
+		default:
+			throw new HTTPError("invalid sticker format: must be png, apng or lottie");
+	}
+}
+
+router.get("/:sticker_id", route({}), async (req: Request, res: Response) => {
+	const { guild_id, sticker_id } = req.params;
+	await Member.IsInGuildOrFail(req.user_id, guild_id);
+
+	res.json(await Sticker.findOneOrFail({ guild_id, id: sticker_id }));
+});
+
+export interface ModifyGuildStickerSchema {
+	/**
+	 * @minLength 2
+	 * @maxLength 30
+	 */
+	name: string;
+	/**
+	 * @maxLength 100
+	 */
+	description?: string;
+	/**
+	 * @maxLength 200
+	 */
+	tags: string;
+}
+
+router.patch(
+	"/:sticker_id",
+	route({ body: "ModifyGuildStickerSchema", permission: "MANAGE_EMOJIS_AND_STICKERS" }),
+	async (req: Request, res: Response) => {
+		const { guild_id, sticker_id } = req.params;
+		const body = req.body as ModifyGuildStickerSchema;
+
+		const sticker = await new Sticker({ ...body, guild_id, id: sticker_id }).save();
+		await sendStickerUpdateEvent(guild_id);
+
+		return res.json(sticker);
+	}
+);
+
+async function sendStickerUpdateEvent(guild_id: string) {
+	return emitEvent({
+		event: "GUILD_STICKERS_UPDATE",
+		guild_id: guild_id,
+		data: {
+			guild_id: guild_id,
+			stickers: await Sticker.find({ guild_id: guild_id })
+		}
+	} as GuildStickersUpdateEvent);
+}
+
+router.delete("/:sticker_id", route({ permission: "MANAGE_EMOJIS_AND_STICKERS" }), async (req: Request, res: Response) => {
+	const { guild_id, sticker_id } = req.params;
+
+	await Sticker.delete({ guild_id, id: sticker_id });
+	await sendStickerUpdateEvent(guild_id);
+
+	return res.sendStatus(204);
+});
+
+export default router;

api/src/routes/guilds/#guild_id/vanity-url.ts

@@ -10,10 +10,10 @@ const InviteRegex = /\W/g;
 router.get("/", route({ permission: "MANAGE_GUILD" }), async (req: Request, res: Response) => {
 	const { guild_id } = req.params;
 
-	const guild = await Guild.findOneOrFail({ where: { id: guild_id }, relations: ["vanity_url"] });
+	const invite = await Invite.findOne({ where: { guild_id: guild_id, vanity_url: true } });
-	if (!guild.vanity_url) return res.json({ code: null });
+	if (!invite) return res.json({ code: null });
 
-	return res.json({ code: guild.vanity_url_code, uses: guild.vanity_url.uses });
+	return res.json({ code: invite.code, uses: invite.uses });
 });
 
 export interface VanityUrlSchema {
@@ -33,20 +33,9 @@ router.patch("/", route({ body: "VanityUrlSchema", permission: "MANAGE_GUILD" })
 	const invite = await Invite.findOne({ code });
 	if (invite) throw new HTTPError("Invite already exists");
 
-	const guild = await Guild.findOneOrFail({ id: guild_id });
 	const { id } = await Channel.findOneOrFail({ guild_id, type: ChannelType.GUILD_TEXT });
 
-	Promise.all([
+	await Invite.update({ vanity_url: true, guild_id }, { code: code, channel_id: id });
-		Guild.update({ id: guild_id }, { vanity_url_code: code }),
-		Invite.delete({ code: guild.vanity_url_code }),
-		new Invite({
-			code: code,
-			uses: 0,
-			created_at: new Date(),
-			guild_id,
-			channel_id: id
-		}).save()
-	]);
 
 	return res.json({ code: code });
 });

api/src/routes/guilds/templates/index.ts

@@ -47,7 +47,7 @@ router.post("/:code", route({ body: "GuildTemplateCreateSchema" }), async (req:
 			managed: true,
 			mentionable: true,
 			name: "@everyone",
-			permissions: 2251804225n,
+			permissions: BigInt("2251804225"),
 			position: 0,
 			tags: null
 		}).save()

api/src/routes/invites/index.ts

@@ -33,7 +33,6 @@ router.delete("/:code", route({}), async (req: Request, res: Response) => {
 
 	await Promise.all([
 		Invite.delete({ code }),
-		Guild.update({ vanity_url_code: code }, { vanity_url_code: undefined }),
 		emitEvent({
 			event: "INVITE_DELETE",
 			guild_id: guild_id,

api/src/routes/sticker-packs/#id/index.ts

@@ -1,19 +0,0 @@
-import { Request, Response, Router } from "express";
-import { route } from "@fosscord/api";
-
-const router: Router = Router();
-
-router.get("/", route({}), async (req: Request, res: Response) => {
-	//TODO
-	res.json({
-		id: "",
-		stickers: [],
-		name: "",
-		sku_id: "",
-		cover_sticker_id: "",
-		description: "",
-		banner_asset_id: ""
-	}).status(200);
-});
-
-export default router;

api/src/routes/sticker-packs/index.ts

@@ -1,11 +1,13 @@
 import { Request, Response, Router } from "express";
 import { route } from "@fosscord/api";
+import { StickerPack } from "@fosscord/util";
 
 const router: Router = Router();
 
 router.get("/", route({}), async (req: Request, res: Response) => {
-	//TODO
+	const sticker_packs = await StickerPack.find({ relations: ["stickers"] });
-	res.json({ sticker_packs: [] }).status(200);
+
+	res.json({ sticker_packs });
 });
 
 export default router;

api/src/routes/stickers/#sticker_id/index.ts

@@ -0,0 +1,12 @@
+import { Sticker } from "@fosscord/util";
+import { Router, Request, Response } from "express";
+import { route } from "@fosscord/api";
+const router = Router();
+
+router.get("/", route({}), async (req: Request, res: Response) => {
+	const { sticker_id } = req.params;
+
+	res.json(await Sticker.find({ id: sticker_id }));
+});
+
+export default router;

api/src/routes/template.ts.disabled

@@ -1,10 +1,11 @@
 //TODO: this is a template for a generic route
 
 import { Router, Request, Response } from "express";
+import { route } from "@fosscord/api";
 const router = Router();
 
-router.get("/", async (req: Request, res: Response) => {
+router.get("/",route({}), async (req: Request, res: Response) => {
-	res.send({});
+	res.json({});
 });
 
 export default router;

api/src/routes/users/@me/settings.ts

@@ -10,8 +10,9 @@ router.patch("/", route({ body: "UserSettingsSchema" }), async (req: Request, re
 	const body = req.body as UserSettings;
 	if (body.locale === "en") body.locale = "en-US"; // fix discord client crash on unkown locale
 
-	// only users can update user settings
+	const user = await User.findOneOrFail({ id: req.user_id, bot: false });
-	await User.update({ id: req.user_id, bot: false }, { settings: body });
+	user.settings = { ...user.settings, ...body };
+	await user.save();
 
 	res.sendStatus(204);
 });

api/src/test/jwt.ts

@@ -1,37 +0,0 @@
-const jwa = require("jwa");
-
-var STR64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".split("");
-
-function base64url(string: string, encoding: string) {
-	// @ts-ignore
-	return Buffer.from(string, encoding).toString("base64").replace(/=/g, "").replace(/\+/g, "-").replace(/\//g, "_");
-}
-
-function to64String(input: number, current = ""): string {
-	if (input < 0 && current.length == 0) {
-		input = input * -1;
-	}
-	var modify = input % 64;
-	var remain = Math.floor(input / 64);
-	var result = STR64[modify] + current;
-	return remain <= 0 ? result : to64String(remain, result);
-}
-
-function to64Parse(input: string) {
-	var result = 0;
-	var toProc = input.split("");
-	var e;
-	for (e in toProc) {
-		result = result * 64 + STR64.indexOf(toProc[e]);
-	}
-	return result;
-}
-
-// @ts-ignore
-const start = `${base64url("311129357362135041")}.${to64String(Date.now())}`;
-const signature = jwa("HS256").sign(start, `test`);
-const token = `${start}.${signature}`;
-console.log(token);
-
-// MzExMTI5MzU3MzYyMTM1MDQx.XdQb_rA.907VgF60kocnOTl32MSUWGSSzbAytQ0jbt36KjLaxuY
-// MzExMTI5MzU3MzYyMTM1MDQx.XdQbaPy.4vGx4L7IuFJGsRe6IL3BeybLIvbx4Vauvx12pwNsy2U

api/src/test/jwt2.ts

@@ -1,13 +0,0 @@
-import jwt from "jsonwebtoken";
-
-const algorithm = "HS256";
-const iat = Math.floor(Date.now() / 1000);
-
-// @ts-ignore
-const token = jwt.sign({ id: "311129357362135041" }, "secret", {
-	algorithm,
-});
-console.log(token);
-
-const decoded = jwt.verify(token, "secret", { algorithms: [algorithm] });
-console.log(decoded);

api/src/test/password_test.ts

@@ -1,12 +0,0 @@
-import { checkPassword } from "@fosscord/api";
-
-console.log(checkPassword("123456789012345"));
-// -> 0.25
-console.log(checkPassword("ABCDEFGHIJKLMOPQ"));
-// -> 0.25
-console.log(checkPassword("ABC123___...123"));
-// ->
-console.log(checkPassword(""));
-// ->
-// console.log(checkPassword(""));
-// // ->

api/src/util/Instance.ts

@@ -1,4 +1,4 @@
-import { Config, Guild } from "@fosscord/util";
+import { Config, Guild, Session } from "@fosscord/util";
 
 export async function initInstance() {
 	// TODO: clean up database and delete tombstone data
@@ -8,11 +8,14 @@ export async function initInstance() {
 	// TODO: check if any current user is not part of autoJoinGuilds
 	const { autoJoin } = Config.get().guild;
 
-	if (autoJoin.enabled && autoJoin.guilds?.length) {
+	if (autoJoin.enabled && !autoJoin.guilds?.length) {
 		let guild = await Guild.findOne({});
-		if (!guild) guild = await Guild.createGuild({});
+		if (guild) {
-
+			// @ts-ignore
-		// @ts-ignore
+			await Config.set({ guild: { autoJoin: { guilds: [guild.id] } } });
-		await Config.set({ guild: { autoJoin: { guilds: [guild.id] } } });
+		}
 	}
+
+	// TODO: do no clear sessions for instance cluster
+	await Session.delete({});
 }

api/src/util/Message.ts

@@ -24,7 +24,8 @@ import fetch from "node-fetch";
 import cheerio from "cheerio";
 import { MessageCreateSchema } from "../routes/channels/#channel_id/messages";
 
-// TODO: check webhook, application, system author
+// TODO: check webhook, application, system author, stickers
+// TODO: embed gifs/videos/images
 
 const LINK_REGEX = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/g;
 
@@ -45,6 +46,7 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 
 	const message = new Message({
 		...opts,
+		sticker_items: opts.sticker_ids?.map((x) => ({ id: x })),
 		guild_id: channel.guild_id,
 		channel_id: opts.channel_id,
 		attachments: opts.attachments || [],
@@ -81,7 +83,7 @@ export async function handleMessage(opts: MessageOptions): Promise<Message> {
 	}
 
 	// TODO: stickers/activity
-	if (!opts.content && !opts.embeds?.length && !opts.attachments?.length) {
+	if (!opts.content && !opts.embeds?.length && !opts.attachments?.length && !opts.sticker_ids?.length) {
 		throw new HTTPError("Empty messages are not allowed", 50006);
 	}
 

api/tests/routes.test.ts

@@ -43,7 +43,7 @@ const request = async (path: string, opts: any = {}): Promise<any> => {
 
 	var data = await response.text();
 	try {
-		data = JSON.stringify(data);
+		data = JSON.parse(data);
 		if (response.status >= 400) throw data;
 		return data;
 	} catch (error) {
@@ -56,9 +56,7 @@ beforeAll(async (done) => {
 		const response = await request("/auth/register", {
 			body: {
 				fingerprint: "805826570869932034.wR8vi8lGlFBJerErO9LG5NViJFw",
-				email: "test@example.com",
 				username: "tester",
-				password: "wtp9gep9gw",
 				invite: null,
 				consent: true,
 				date_of_birth: "2000-01-01",

api/tsconfig-docker.json

@@ -1,68 +0,0 @@
-{
-	"include": ["src/**/*.ts"],
-	"compilerOptions": {
-		/* Visit https://aka.ms/tsconfig.json to read more about this file */
-
-		/* Basic Options */
-		// "incremental": true,                   /* Enable incremental compilation */
-		"target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
-		"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
-		"lib": ["ES2021"] /* Specify library files to be included in the compilation. */,
-		"allowJs": true /* Allow javascript files to be compiled. */,
-		"checkJs": true /* Report errors in .js files. */,
-		// "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
-		"declaration": false /* Generates corresponding '.d.ts' file. */,
-		"declarationMap": false /* Generates a sourcemap for each corresponding '.d.ts' file. */,
-		"sourceMap": true /* Generates corresponding '.map' file. */,
-		// "outFile": "./",                       /* Concatenate and emit output to single file. */
-		"outDir": "./dist/" /* Redirect output structure to the directory. */,
-		"rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
-		// "composite": true,                     /* Enable project compilation */
-		// "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
-		// "removeComments": true,                /* Do not emit comments to output. */
-		// "noEmit": true,                        /* Do not emit outputs. */
-		// "importHelpers": true,                 /* Import emit helpers from 'tslib'. */
-		// "downlevelIteration": true,            /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
-		// "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
-
-		/* Strict Type-Checking Options */
-		"strict": true /* Enable all strict type-checking options. */,
-		"noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */,
-		"strictNullChecks": true /* Enable strict null checks. */,
-		// "strictFunctionTypes": true,           /* Enable strict checking of function types. */
-		// "strictBindCallApply": true,           /* Enable strict 'bind', 'call', and 'apply' methods on functions. */
-		"strictPropertyInitialization": false /* Enable strict checking of property initialization in classes. */,
-		// "noImplicitThis": true,                /* Raise error on 'this' expressions with an implied 'any' type. */
-		"alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */,
-
-		/* Additional Checks */
-		// "noUnusedLocals": true,                /* Report errors on unused locals. */
-		// "noUnusedParameters": true,            /* Report errors on unused parameters. */
-		// "noImplicitReturns": true,             /* Report error when not all code paths in function return a value. */
-		// "noFallthroughCasesInSwitch": true,    /* Report errors for fallthrough cases in switch statement. */
-
-		/* Module Resolution Options */
-		// "moduleResolution": "node",            /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
-		// "rootDirs": [],                        /* List of root folders whose combined content represents the structure of the project at runtime. */
-		// "typeRoots": [],                       /* List of folders to include type definitions from. */
-		"types": ["node"] /* Type declaration files to be included in compilation. */,
-		// "allowSyntheticDefaultImports": true,  /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
-		"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
-		// "preserveSymlinks": true,              /* Do not resolve the real path of symlinks. */
-		// "allowUmdGlobalAccess": true,          /* Allow accessing UMD globals from modules. */
-
-		/* Source Map Options */
-		// "sourceRoot": "",                      /* Specify the location where debugger should locate TypeScript files instead of source locations. */
-		// "mapRoot": "",                         /* Specify the location where debugger should locate map files instead of generated locations. */
-		// "inlineSourceMap": true,               /* Emit a single file with source maps instead of having a separate file. */
-		// "inlineSources": true,                 /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
-
-		/* Experimental Options */
-		// "experimentalDecorators": true,        /* Enables experimental support for ES7 decorators. */
-		// "emitDecoratorMetadata": true,         /* Enables experimental support for emitting type metadata for decorators. */
-
-		/* Advanced Options */
-		"skipLibCheck": true /* Skip type checking of declaration files. */,
-		"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */
-	}
-}

api/tsconfig.json

@@ -1,10 +1,11 @@
 {
+	"exclude": ["node_modules"],
 	"include": ["src/**/*.ts"],
 	"compilerOptions": {
 		/* Visit https://aka.ms/tsconfig.json to read more about this file */
 
 		/* Basic Options */
-		// "incremental": true,                   /* Enable incremental compilation */
+		"incremental": true /* Enable incremental compilation */,
 		"target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
 		"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
 		"lib": ["ES2021"] /* Specify library files to be included in the compilation. */,
@@ -66,9 +67,9 @@
 		"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */,
 		"baseUrl": ".",
 		"paths": {
-			"@fosscord/api": ["src/index"],
+			"@fosscord/api": ["src/index"]
-			"@fosscord/api/*": ["src/*"]
 		},
-		"plugins": [{ "transform": "@zerollup/ts-transform-paths" }]
+		"plugins": [{ "transform": "@zerollup/ts-transform-paths" }],
+		"experimentalDecorators": true
 	}
 }

bundle/.gitignore

@@ -1,2 +0,0 @@
-files/
-.env

bundle/.vscode/launch.json

@@ -8,13 +8,11 @@
 			"sourceMaps": true,
 			"type": "node",
 			"request": "launch",
-			"name": "Launch server bundle",
+			"name": "Launch Server",
-			"program": "${workspaceFolder}/dist/start.js",
+			"program": "${workspaceFolder}/dist/bundle/src/start.js",
-			"runtimeArgs": ["-r", "./tsconfig-paths-bootstrap.js"],
 			"preLaunchTask": "tsc: build - tsconfig.json",
-			"outFiles": ["${workspaceFolder}/dist/**/*.js", "${workspaceFolder}/node_modules/@fosscord/**/*.js"],
+			"outFiles": ["${workspaceFolder}/dist/**/*.js"],
-			"envFile": "${workspaceFolder}/.env",
+			"envFile": "${workspaceFolder}/.env"
-			"outDir": "${workspaceFolder}/dist"
 		}
 	]
 }

bundle/package.json

@@ -4,12 +4,12 @@
 	"description": "",
 	"main": "src/start.js",
 	"scripts": {
-		"setup": "cd ../util && npm --production=false i && cd ../api && npm --production=false i && cd ../cdn && npm --production=false i && cd ../gateway && npm --production=false i && cd ../bundle/ && npm --production=false i && npm run build",
+		"setup": "node scripts/install.js && npm install && ts-patch install -s && patch-package --patch-dir ../api/patches/ && npm run build",
 		"build": "node scripts/build.js",
-		"build:bundle": "npx tsc -b .",
+		"start": "node scripts/build.js && node dist/bundle/src/start.js",
-		"start": "node scripts/build.js && node -r tsconfig-paths/register dist/start.js",
+		"start:bundle": "node dist/bundle/src/start.js",
-		"start:bundle": "node -r tsconfig-paths/register dist/start.js",
+		"test": "echo \"Error: no test specified\" && exit 1",
-		"test": "echo \"Error: no test specified\" && exit 1"
+		"migrate": "cd ../util/ && npm i && node --require ts-node/register node_modules/typeorm/cli.js -f ../util/ormconfig.json migration:run"
 	},
 	"repository": {
 		"type": "git",
@@ -23,42 +23,80 @@
 	},
 	"homepage": "https://fosscord.com",
 	"devDependencies": {
-		"@swc/cli": "^0.1.51",
+		"@babel/core": "^7.15.5",
-		"@swc/core": "^1.2.93",
+		"@babel/preset-env": "^7.15.8",
+		"@babel/preset-typescript": "^7.15.0",
 		"@types/amqplib": "^0.8.1",
-		"@types/async-exit-hook": "^2.0.0",
 		"@types/bcrypt": "^5.0.0",
-		"@types/express": "^4.17.9",
+		"@types/body-parser": "^1.19.0",
+		"@types/btoa": "^1.2.3",
+		"@types/dotenv": "^8.2.0",
+		"@types/express": "^4.17.12",
+		"@types/fs-extra": "^9.0.12",
 		"@types/i18next-node-fs-backend": "^2.1.0",
+		"@types/jest": "^27.0.1",
+		"@types/jest-expect-message": "^1.0.3",
 		"@types/jsonwebtoken": "^8.5.0",
-		"@types/mongodb": "^3.6.9",
+		"@types/morgan": "^1.9.3",
-		"@types/mongoose-autopopulate": "^0.10.1",
+		"@types/multer": "^1.4.7",
-		"@types/mongoose-lean-virtuals": "^0.5.1",
+		"@types/node": "^14.17.9",
-		"@types/multer": "^1.4.5",
+		"@types/node-fetch": "^2.5.12",
-		"@types/node": "^14.17.20",
-		"@types/node-fetch": "^2.5.7",
 		"@types/node-os-utils": "^1.2.0",
-		"@types/uuid": "^8.3.0",
+		"@types/supertest": "^2.0.11",
 		"@types/ws": "^7.4.0",
 		"@zerollup/ts-transform-paths": "^1.7.18",
-		"esbuild": "^0.13.4",
+		"jest": "^27.0.6",
-		"esbuild-plugin-tsc": "^0.3.0",
+		"jest-expect-message": "^1.0.2",
+		"jest-runtime": "^27.2.1",
 		"ts-node": "^10.2.1",
+		"ts-node-dev": "^1.1.6",
 		"ts-patch": "^1.4.4",
-		"tsconfig-paths": "^3.11.0",
+		"typescript": "^4.2.3",
-		"typescript": "^4.4.3"
+		"typescript-json-schema": "0.50.1"
 	},
 	"dependencies": {
-		"@fosscord/api": "file:../api",
+		"@aws-sdk/client-s3": "^3.36.1",
-		"@fosscord/cdn": "file:../cdn",
+		"@aws-sdk/node-http-handler": "^3.36.0",
-		"@fosscord/gateway": "file:../gateway",
+		"@babel/preset-env": "^7.15.8",
-		"@fosscord/util": "file:../util",
+		"@babel/preset-typescript": "^7.15.0",
+		"ajv": "8.6.2",
+		"ajv-formats": "^2.1.1",
+		"amqplib": "^0.8.0",
+		"assert": "^1.5.0",
 		"async-exit-hook": "^2.0.1",
-		"dotenv": "^10.0.0",
+		"bcrypt": "^5.0.1",
+		"body-parser": "^1.19.0",
+		"btoa": "^1.2.1",
+		"cheerio": "^1.0.0-rc.10",
+		"dotenv": "^8.2.0",
+		"exif-be-gone": "^1.2.0",
 		"express": "^4.17.1",
-		"missing-native-js-functions": "^1.2.17",
+		"express-async-errors": "^3.1.1",
+		"file-type": "^16.5.0",
+		"form-data": "^3.0.0",
+		"fs-extra": "^10.0.0",
+		"i18next": "^19.9.2",
+		"i18next-http-middleware": "^3.1.3",
+		"i18next-node-fs-backend": "^2.1.3",
+		"image-size": "^1.0.0",
+		"jest": "^27.0.6",
+		"jsonwebtoken": "^8.5.1",
+		"lambert-db": "^1.2.3",
+		"lambert-server": "^1.2.11",
+		"missing-native-js-functions": "^1.2.18",
+		"morgan": "^1.10.0",
+		"multer": "^1.4.2",
 		"nanocolors": "^0.2.12",
+		"node-fetch": "^2.6.1",
 		"node-os-utils": "^1.3.5",
-		"reflect-metadata": "^0.1.13"
+		"patch-package": "^6.4.7",
+		"pg": "^8.7.1",
+		"reflect-metadata": "^0.1.13",
+		"sqlite3": "^5.0.2",
+		"supertest": "^6.1.6",
+		"typeorm": "^0.2.37",
+		"typescript": "^4.1.2",
+		"typescript-json-schema": "^0.50.1",
+		"ws": "^7.4.2"
 	}
 }

bundle/scripts/benchmark/connections.js

@@ -0,0 +1,59 @@
+require("dotenv").config();
+const cluster = require("cluster");
+const WebSocket = require("ws");
+const endpoint = process.env.GATEWAY || "ws://localhost:3001";
+const connections = Number(process.env.CONNECTIONS) || 50;
+const threads = Number(process.env.THREADS) || require("os").cpus().length || 1;
+const token = process.env.TOKEN;
+
+if (!token) {
+	console.error("TOKEN env var missing");
+	process.exit();
+}
+
+if (cluster.isMaster) {
+	for (let i = 0; i < threads; i++) {
+		cluster.fork();
+	}
+
+	cluster.on("exit", (worker, code, signal) => {
+		console.log(`worker ${worker.process.pid} died`);
+	});
+} else {
+	for (let i = 0; i < connections; i++) {
+		connect();
+	}
+}
+
+function connect() {
+	const client = new WebSocket(endpoint);
+	client.on("message", (data) => {
+		data = JSON.parse(data);
+
+		switch (data.op) {
+			case 10:
+				client.interval = setInterval(() => {
+					client.send(JSON.stringify({ op: 1 }));
+				}, data.d.heartbeat_interval);
+
+				client.send(
+					JSON.stringify({
+						op: 2,
+						d: {
+							token,
+							properties: {},
+						},
+					})
+				);
+
+				break;
+		}
+	});
+	client.once("close", (code, reason) => {
+		clearInterval(client.interval);
+		connect();
+	});
+	client.on("error", (err) => {
+		// console.log(err);
+	});
+}

bundle/scripts/benchmark/index.js

@@ -0,0 +1,4 @@
+require("dotenv").config();
+
+require("./connections");
+require("./messages");

bundle/scripts/benchmark/users.js

@@ -0,0 +1,25 @@
+require("dotenv").config();
+const fetch = require("node-fetch");
+const count = Number(process.env.COUNT) || 50;
+const endpoint = process.env.API || "http://localhost:3001";
+
+async function main() {
+	for (let i = 0; i < count; i++) {
+		fetch(`${endpoint}/api/auth/register`, {
+			method: "POST",
+			body: JSON.stringify({
+				fingerprint: `${i}.wR8vi8lGlFBJerErO9LG5NViJFw`,
+				username: `test${i}`,
+				invite: null,
+				consent: true,
+				date_of_birth: "2000-01-01",
+				gift_code_sku_id: null,
+				captcha_key: null,
+			}),
+			headers: { "content-type": "application/json" },
+		});
+		console.log(i);
+	}
+}
+
+main();

bundle/scripts/build.js

@@ -1,103 +1,49 @@
-const { spawn } = require("child_process");
+const { execSync } = require("child_process");
 const path = require("path");
-const fs = require("fs");
+const fse = require("fs-extra");
-const { performance } = require("perf_hooks");
+const { getSystemErrorMap } = require("util");
+const { argv } = require("process");
 
-let parts = "api,cdn,gateway,bundle".split(",");
+const dirs = ["api", "util", "cdn", "gateway", "bundle"];
-const tscBin = path.join(__dirname, "..", "..", "util", "node_modules", "typescript", "bin", "tsc");
-const swcBin = path.join(__dirname, "..", "..", "util", "node_modules", "@swc", "cli", "bin", "swc");
 
-// because npm run is slow we directly get the build script of the package.json script
+const verbose = argv.includes("verbose") || argv.includes("v");
 
-function buildPackage(dir) {
+if (argv.includes("clean")) {
-	const element = path.basename(dir);
+	dirs.forEach((a) => {
-
+		var d = "../" + a + "/dist";
-	return require("esbuild").build({
+		if (fse.existsSync(d)) {
-		entryPoints: walk(path.join(dir, "src")),
+			fse.rmSync(d, { recursive: true });
-		bundle: false,
+			if (verbose) console.log(`Deleted ${d}!`);
-		outdir: path.join(dir, "dist"),
-		target: "es2021",
-		// plugins don't really work because bundle is false
-		keepNames: false,
-		tsconfig: path.join(dir, "tsconfig.json"),
-	});
-}
-
-const importPart = /import (\* as )?(({[^}]+})|(\w+)) from ("[.\w-/@q]+")/g;
-const importMod = /import ("[\w-/@q.]+")/g;
-const exportDefault = /export default/g;
-const exportAllAs = /export \* from (".+")/g;
-const exportMod = /export ({[\w, ]+})/g;
-const exportConst = /export (const|var|let) (\w+)/g;
-const exportPart = /export ((async )?\w+) (\w+)/g;
-
-// resolves tsconfig paths + rewrites es6 imports/exports to require (because esbuild/swc doesn't work properly)
-function transpileFiles() {
-	for (const part of ["gateway", "api", "cdn", "bundle"]) {
-		const files = walk(path.join(__dirname, "..", "..", part, "dist"));
-		for (const file of files) {
-			let content = fs.readFileSync(file, { encoding: "utf8" });
-			content = content
-				.replace(
-					new RegExp(`@fosscord/${part}`),
-					path.relative(file, path.join(__dirname, "..", "..", part, "dist")).slice(3)
-				)
-				.replace(importPart, `const $2 = require($5)`)
-				.replace(importMod, `require($1)`)
-				.replace(exportDefault, `module.exports =`)
-				.replace(exportAllAs, `module.exports = {...(module.exports)||{}, ...require($1)}`)
-				.replace(exportMod, "module.exports = $1")
-				.replace(exportConst, `let $2 = {};\nmodule.exports.$2 = $2`)
-				.replace(exportPart, `module.exports.$3 = $1 $3`);
-			fs.writeFileSync(file, content);
 		}
-	}
-}
-
-function util() {
-	// const child = spawn("node", `${swcBin}  src --out-dir dist --sync`.split(" "), {
-	const child = spawn("node", `${tscBin} -b .`.split(" "), {
-		cwd: path.join(__dirname, "..", "..", "util"),
-		env: process.env,
-		shell: true,
 	});
-	function log(data) {
-		console.log(`[util] ` + data.toString().slice(0, -1));
-	}
-	child.stdout.on("data", log);
-	child.stderr.on("data", log);
-	child.on("error", (err) => console.error("util", err));
-	return child;
 }
 
-const start = performance.now();
+fse.copySync(path.join(__dirname, "..", "..", "api", "assets"), path.join(__dirname, "..", "dist", "api", "assets"));
-
+fse.copySync(
-async function main() {
+	path.join(__dirname, "..", "..", "api", "client_test"),
-	console.log("[Build] starting ...");
+	path.join(__dirname, "..", "dist", "api", "client_test")
-	util();
+);
-	await Promise.all(parts.map((part) => buildPackage(path.join(__dirname, "..", "..", part))));
+fse.copySync(path.join(__dirname, "..", "..", "api", "locales"), path.join(__dirname, "..", "dist", "api", "locales"));
-	transpileFiles();
+dirs.forEach((a) => {
-}
+	fse.copySync("../" + a + "/src", "dist/" + a + "/src");
-
+	if (verbose) console.log(`Copied ${"../" + a + "/dist"} -> ${"dist/" + a + "/src"}!`);
-main();
-
-process.on("exit", () => {
-	console.log("[Build] took " + Math.round(performance.now() - start) + "ms");
 });
 
-function walk(dir) {
+console.log("Copying src files done");
-	var results = [];
+console.log("Compiling src files ...");
-	var list = fs.readdirSync(dir);
+
-	list.forEach(function (file) {
+console.log(
-		file = path.join(dir, file);
+	execSync(
-		var stat = fs.statSync(file);
+		'node "' +
-		if (stat && stat.isDirectory()) {
+			path.join(__dirname, "..", "node_modules", "typescript", "lib", "tsc.js") +
-			/* Recurse into a subdirectory */
+			'" -p "' +
-			results = results.concat(walk(file));
+			path.join(__dirname, "..") +
-		} else if (file.endsWith(".ts") || file.endsWith(".js")) {
+			'"',
-			/* Is a file */
+		{
-			results.push(file);
+			cwd: path.join(__dirname, ".."),
+			shell: true,
+			env: process.env,
+			encoding: "utf8",
 		}
-	});
+	)
-	return results;
+);
-}

bundle/scripts/install.js

@@ -0,0 +1,14 @@
+const path = require("path");
+const fs = require("fs");
+const parts = ["api", "util", "cdn", "gateway"];
+
+const bundle = require("../package.json");
+
+for (const part of parts) {
+	const { devDependencies, dependencies } = require(path.join("..", "..", part, "package.json"));
+	bundle.devDependencies = { ...bundle.devDependencies, ...devDependencies };
+	bundle.dependencies = { ...bundle.dependencies, ...dependencies };
+	delete bundle.dependencies["@fosscord/util"];
+}
+
+fs.writeFileSync(path.join(__dirname, "..", "package.json"), JSON.stringify(bundle, null, "\t"), { encoding: "utf8" });

bundle/src/Server.ts

@@ -4,7 +4,7 @@ process.on("uncaughtException", console.error);
 import http from "http";
 import * as Api from "@fosscord/api";
 import * as Gateway from "@fosscord/gateway";
-import { CDNServer } from "@fosscord/cdn/";
+import { CDNServer } from "@fosscord/cdn";
 import express from "express";
 import { green, bold } from "nanocolors";
 import { Config, initDatabase } from "@fosscord/util";

bundle/src/start.ts

@@ -1,20 +1,4 @@
 // process.env.MONGOMS_DEBUG = "true";
-const tsConfigPaths = require("tsconfig-paths");
-const path = require("path");
-const baseUrl = path.join(__dirname, "..");
-const cleanup = tsConfigPaths.register({
-	baseUrl,
-	paths: {
-		"@fosscord/api": ["../api/dist/index.js"],
-		"@fosscord/api/*": ["../api/dist/*"],
-		"@fosscord/gateway": ["../gateway/dist/index.js"],
-		"@fosscord/gateway/*": ["../gateway/dist/*"],
-		"@fosscord/cdn": ["../cdn/dist/index.js"],
-		"@fosscord/cdn/*": ["../cdn/dist/*"],
-	},
-});
-console.log(require("@fosscord/gateway"));
-
 import "reflect-metadata";
 import cluster from "cluster";
 import os from "os";

bundle/src/stats.ts

@@ -1,11 +1,19 @@
 import os from "os";
 import osu from "node-os-utils";
+import { red } from "nanocolors";
 
 export function initStats() {
 	console.log(`[Path] running in ${__dirname}`);
 	console.log(`[CPU] ${osu.cpu.model()} Cores x${osu.cpu.count()}`);
 	console.log(`[System] ${os.platform()} ${os.arch()}`);
 	console.log(`[Process] running with pid: ${process.pid}`);
+	if (process.getuid && process.getuid() === 0) {
+		console.warn(
+			red(
+				`[Process] Warning fosscord is running as root, this highly discouraged and might expose your system vulnerable to attackers. Please run fosscord as a user without root privileges.`
+			)
+		);
+	}
 
 	setInterval(async () => {
 		const [cpuUsed, memory, network] = await Promise.all([
@@ -23,5 +31,6 @@ export function initStats() {
 				process.memoryUsage().rss / 1024 / 1024
 			)}mb/${memory.totalMemMb.toFixed(0)}mb ${networkUsage}`
 		);
-	}, 1000 * 5);
+		// TODO: node-os-utils might have a memory leak, more investigation needed
+	}, 1000 * 60 * 5);
 }

bundle/tsconfig.json

@@ -1,22 +1,23 @@
 {
-	"include": ["src/**/*.ts"],
+	"include": ["dist/**/*.ts"],
+	"exclude": [],
 	"compilerOptions": {
 		/* Visit https://aka.ms/tsconfig.json to read more about this file */
 
 		/* Basic Options */
-		"incremental": true /* Enable incremental compilation */,
+		"incremental": false /* Enable incremental compilation */,
-		"target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
+		"target": "ES6" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */,
 		"module": "commonjs" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */,
 		"lib": ["ES2021"] /* Specify library files to be included in the compilation. */,
 		"allowJs": true /* Allow javascript files to be compiled. */,
 		"checkJs": true /* Report errors in .js files. */,
 		// "jsx": "preserve",                     /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
-		"declaration": true /* Generates corresponding '.d.ts' file. */,
+		"declaration": false /* Generates corresponding '.d.ts' file. */,
 		"declarationMap": false /* Generates a sourcemap for each corresponding '.d.ts' file. */,
-		"sourceMap": true /* Generates corresponding '.map' file. */,
+		"sourceMap": false /* Generates corresponding '.map' file. */,
 		// "outFile": "./",                       /* Concatenate and emit output to single file. */
 		"outDir": "./dist/" /* Redirect output structure to the directory. */,
-		"rootDir": "./src/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
+		"rootDir": "./dist/" /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */,
 		// "composite": true,                     /* Enable project compilation */
 		// "tsBuildInfoFile": "./",               /* Specify file to store incremental compilation information */
 		// "removeComments": true,                /* Do not emit comments to output. */
@@ -66,6 +67,14 @@
 		"forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */,
 		"emitDecoratorMetadata": true,
 		"experimentalDecorators": true,
-		"baseUrl": "."
+		"resolveJsonModule": true,
+		"baseUrl": "./dist/",
+		"paths": {
+			"@fosscord/api": ["api/src/index"],
+			"@fosscord/gateway": ["gateway/src/index"],
+			"@fosscord/cdn": ["cdn/src/index"],
+			"@fosscord/util": ["util/src/index"]
+		},
+		"plugins": [{ "transform": "@zerollup/ts-transform-paths" }]
 	}
 }

cdn/Dockerfile

@@ -4,4 +4,5 @@ COPY package.json .
 RUN npm install
 COPY . .
 EXPOSE 3003
-CMD ["node", "dist/"]
+RUN npm run build
+CMD ["node", "dist/start.js"]

cdn/package.json

@@ -7,7 +7,7 @@
 	"scripts": {
 		"postinstall": "ts-patch install -s",
 		"test": "npm run build && jest --coverage ./tests",
-		"build": "npx tsc -b .",
+		"build": "npx tsc -p .",
 		"start": "npm run build && node dist/start.js"
 	},
 	"repository": {
@@ -22,8 +22,6 @@
 	},
 	"homepage": "https://github.com/fosscord/fosscord-server#readme",
 	"devDependencies": {
-		"@swc/cli": "^0.1.51",
-		"@swc/core": "^1.2.93",
 		"@types/amqplib": "^0.8.1",
 		"@types/body-parser": "^1.19.0",
 		"@types/btoa": "^1.2.3",
@@ -31,21 +29,18 @@
 		"@types/express": "^4.17.12",
 		"@types/fs-extra": "^9.0.12",
 		"@types/jsonwebtoken": "^8.5.0",
-		"@types/mongodb": "^3.6.9",
-		"@types/mongoose-autopopulate": "^0.10.1",
-		"@types/mongoose-lean-virtuals": "^0.5.1",
 		"@types/multer": "^1.4.7",
 		"@types/node": "^14.17.0",
 		"@types/node-fetch": "^2.5.7",
-		"@types/uuid": "^8.3.0",
 		"@zerollup/ts-transform-paths": "^1.7.18",
 		"ts-patch": "^1.4.4"
 	},
 	"dependencies": {
+		"@aws-sdk/client-s3": "^3.36.1",
+		"@aws-sdk/node-http-handler": "^3.36.0",
 		"@fosscord/util": "file:../util",
 		"body-parser": "^1.19.0",
 		"btoa": "^1.2.1",
-		"cheerio": "^1.0.0-rc.5",
 		"dotenv": "^10.0.0",
 		"exif-be-gone": "^1.2.0",
 		"express": "^4.17.1",
@@ -56,13 +51,12 @@
 		"jest": "^27.0.6",
 		"lambert-db": "^1.2.3",
 		"lambert-server": "^1.2.11",
-		"missing-native-js-functions": "^1.2.17",
+		"missing-native-js-functions": "^1.2.18",
 		"multer": "^1.4.2",
 		"nanocolors": "^0.2.12",
 		"node-fetch": "^2.6.1",
 		"supertest": "^6.1.6",
-		"typescript": "^4.1.2",
+		"typescript": "^4.1.2"
-		"uuid": "^8.3.2"
 	},
 	"jest": {
 		"setupFilesAfterEnv": [

cdn/src/Server.ts

@@ -1,5 +1,5 @@
 import { Server, ServerOptions } from "lambert-server";
-import { Config, initDatabase } from "@fosscord/util";
+import { Config, initDatabase, registerRoutes } from "@fosscord/util";
 import path from "path";
 import avatarsRoute from "./routes/avatars";
 import bodyParser from "body-parser";
@@ -23,13 +23,19 @@ export class CDNServer extends Server {
 				"Content-security-policy",
 				"default-src *  data: blob: filesystem: about: ws: wss: 'unsafe-inline' 'unsafe-eval'; script-src * data: blob: 'unsafe-inline' 'unsafe-eval'; connect-src * data: blob: 'unsafe-inline'; img-src * data: blob: 'unsafe-inline'; frame-src * data: blob: ; style-src * data: blob: 'unsafe-inline'; font-src * data: blob: 'unsafe-inline';"
 			);
-			res.set("Access-Control-Allow-Headers", req.header("Access-Control-Request-Headers") || "*");
+			res.set(
-			res.set("Access-Control-Allow-Methods", req.header("Access-Control-Request-Methods") || "*");
+				"Access-Control-Allow-Headers",
+				req.header("Access-Control-Request-Headers") || "*"
+			);
+			res.set(
+				"Access-Control-Allow-Methods",
+				req.header("Access-Control-Request-Methods") || "*"
+			);
 			next();
 		});
 		this.app.use(bodyParser.json({ inflate: true, limit: "10mb" }));
 
-		await this.registerRoutes(path.join(__dirname, "routes/"));
+		await registerRoutes(this, path.join(__dirname, "routes/"));
 
 		this.app.use("/icons/", avatarsRoute);
 		this.log("verbose", "[Server] Route /icons registered");

cdn/src/routes/avatars.ts

@@ -58,6 +58,21 @@ router.post(
 	}
 );
 
+router.get("/:user_id", async (req: Request, res: Response) => {
+	var { user_id } = req.params;
+	user_id = user_id.split(".")[0]; // remove .file extension
+	const path = `avatars/${user_id}`;
+
+	const file = await storage.get(path);
+	if (!file) throw new HTTPError("not found", 404);
+	const type = await FileType.fromBuffer(file);
+
+	res.set("Content-Type", type?.mime);
+	res.set("Cache-Control", "public, max-age=31536000");
+
+	return res.send(file);
+});
+
 router.get("/:user_id/:hash", async (req: Request, res: Response) => {
 	var { user_id, hash } = req.params;
 	hash = hash.split(".")[0]; // remove .file extension

cdn/src/util/FileStorage.ts

@@ -13,16 +13,24 @@ function getPath(path: string) {
 	const root = process.env.STORAGE_LOCATION || "../";
 	var filename = join(root, path);
 
-	if (path.indexOf("\0") !== -1 || !filename.startsWith(root)) throw new Error("invalid path");
+	if (path.indexOf("\0") !== -1 || !filename.startsWith(root))
+		throw new Error("invalid path");
 	return filename;
 }
 
 export class FileStorage implements Storage {
 	async get(path: string): Promise<Buffer | null> {
+		path = getPath(path);
 		try {
-			return fs.readFileSync(getPath(path));
+			return fs.readFileSync(path);
 		} catch (error) {
-			return null;
+			try {
+				const files = fs.readdirSync(path);
+				if (!files.length) return null;
+				return fs.readFileSync(join(path, files[0]));
+			} catch (error) {
+				return null;
+			}
 		}
 	}
 

cdn/src/util/S3Storage.ts

@@ -0,0 +1,60 @@
+import { S3 } from "@aws-sdk/client-s3";
+import { Readable } from "stream";
+import { Storage } from "./Storage";
+
+const readableToBuffer = (readable: Readable): Promise<Buffer> =>
+	new Promise((resolve, reject) => {
+		const chunks: Buffer[] = [];
+		readable.on('data', chunk => chunks.push(chunk));
+		readable.on('error', reject);
+		readable.on('end', () => resolve(Buffer.concat(chunks)));
+	});
+
+export class S3Storage implements Storage {
+	public constructor(
+		private client: S3,
+		private bucket: string,
+		private basePath?: string,
+	) {}
+
+	/**
+	 * Always return a string, to ensure consistency.
+	 */
+	get bucketBasePath() {
+		return this.basePath ?? '';
+	}
+
+	async set(path: string, data: Buffer): Promise<void> {
+		await this.client.putObject({
+			Bucket: this.bucket,
+			Key: `${this.bucketBasePath}${path}`,
+			Body: data
+		});
+	}
+
+	async get(path: string): Promise<Buffer | null> {
+		try {
+			const s3Object = await this.client.getObject({
+				Bucket: this.bucket,
+				Key: `${this.bucketBasePath ?? ''}${path}`
+			});
+
+			if (!s3Object.Body) return null;
+
+			const body = s3Object.Body;
+
+			return await readableToBuffer(<Readable> body);
+		} catch(err) {
+			console.error(`[CDN] Unable to get S3 object at path ${path}.`);
+			console.error(err);
+			return null;
+		}
+	}
+
+	async delete(path: string): Promise<void> {
+		await this.client.deleteObject({
+			Bucket: this.bucket,
+			Key: `${this.bucketBasePath}${path}`
+		});
+	}
+}

cdn/src/util/Storage.ts

@@ -2,6 +2,8 @@ import { FileStorage } from "./FileStorage";
 import path from "path";
 import fse from "fs-extra";
 import { bgCyan, black } from "nanocolors";
+import { S3 } from '@aws-sdk/client-s3';
+import { S3Storage } from "./S3Storage";
 process.cwd();
 
 export interface Storage {
@@ -10,10 +12,10 @@ export interface Storage {
 	delete(path: string): Promise<void>;
 }
 
-var storage: Storage;
+let storage: Storage;
 
 if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
-	var location = process.env.STORAGE_LOCATION;
+	let location = process.env.STORAGE_LOCATION;
 	if (location) {
 		location = path.resolve(location);
 	} else {
@@ -24,6 +26,32 @@ if (process.env.STORAGE_PROVIDER === "file" || !process.env.STORAGE_PROVIDER) {
 	process.env.STORAGE_LOCATION = location;
 
 	storage = new FileStorage();
+} else if (process.env.STORAGE_PROVIDER === "s3") {
+	const
+		region = process.env.STORAGE_REGION,
+		bucket = process.env.STORAGE_BUCKET;
+
+	if (!region) {
+		console.error(`[CDN] You must provide a region when using the S3 storage provider.`);
+		process.exit(1);
+	}
+
+	if (!bucket) {
+		console.error(`[CDN] You must provide a bucket when using the S3 storage provider.`);
+		process.exit(1);
+	}
+
+	// in the S3 provider, this should be the root path in the bucket
+	let location = process.env.STORAGE_LOCATION;
+
+	if (!location) {
+		console.warn(`[CDN] STORAGE_LOCATION unconfigured for S3 provider, defaulting to the bucket root...`);
+  		location = undefined;
+	}
+
+	const client = new S3({ region });
+
+	storage = new S3Storage(client, bucket, location);
 }
 
 export { storage };

dashboard/package-lock.json

@@ -0,0 +1,6 @@
+{
+  "name": "dashboard",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {}
+}

dashboard/package.json

@@ -0,0 +1 @@
+{}

gateway/client.js

@@ -3,7 +3,7 @@ const WebSocket = require("ws");
 const Constants = require("./dist/util/Constants");
 
 // const ws = new WebSocket("ws://127.0.0.1:8080");
-const ws = new WebSocket("wss://gateway.discord.gg");
+const ws = new WebSocket("wss://dev.fosscord.com");
 
 ws.on("open", () => {
 	// ws.send(JSON.stringify({ req_type: "new_auth" }));

gateway/package.json

@@ -8,23 +8,17 @@
 		"postinstall": "npx ts-patch install -s",
 		"test": "echo \"Error: no test specified\" && exit 1",
 		"start": "npm run build && node dist/start.js",
-		"build": "npx tsc -b .",
+		"build": "npx tsc -p .",
 		"dev": "tsnd --respawn src/start.ts"
 	},
 	"keywords": [],
 	"author": "Fosscord",
 	"license": "ISC",
 	"devDependencies": {
-		"@swc/cli": "^0.1.51",
-		"@swc/core": "^1.2.93",
 		"@types/amqplib": "^0.8.1",
 		"@types/jsonwebtoken": "^8.5.0",
-		"@types/mongodb": "^3.6.9",
-		"@types/mongoose-autopopulate": "^0.10.1",
-		"@types/mongoose-lean-virtuals": "^0.5.1",
 		"@types/node": "^14.17.9",
 		"@types/node-fetch": "^2.5.12",
-		"@types/uuid": "^8.3.0",
 		"@types/ws": "^7.4.0",
 		"@zerollup/ts-transform-paths": "^1.7.18",
 		"ts-node-dev": "^1.1.6",
@@ -33,16 +27,13 @@
 	},
 	"dependencies": {
 		"@fosscord/util": "file:../util",
-		"ajv": "^8.5.0",
 		"amqplib": "^0.8.0",
 		"dotenv": "^8.2.0",
 		"jsonwebtoken": "^8.5.1",
 		"lambert-server": "^1.2.11",
-		"missing-native-js-functions": "^1.2.17",
+		"missing-native-js-functions": "^1.2.18",
-		"mongoose-autopopulate": "^0.12.3",
 		"node-fetch": "^2.6.1",
 		"typeorm": "^0.2.37",
-		"uuid": "^8.3.2",
 		"ws": "^7.4.2"
 	},
 	"optionalDependencies": {

gateway/src/Server.ts

@@ -32,7 +32,6 @@ export class Server {
 		}
 
 		this.server.on("upgrade", (request, socket, head) => {
-			console.log("socket requests upgrade", request.url);
 			// @ts-ignore
 			this.ws.handleUpgrade(request, socket, head, (socket) => {
 				this.ws.emit("connection", socket, request);

gateway/src/events/Close.ts

@@ -1,10 +1,46 @@
 import { WebSocket } from "@fosscord/gateway";
-import { Message } from "./Message";
+import {
-import { Session } from "@fosscord/util";
+	emitEvent,
+	PresenceUpdateEvent,
+	PrivateSessionProjection,
+	Session,
+	SessionsReplace,
+	User,
+} from "@fosscord/util";
 
 export async function Close(this: WebSocket, code: number, reason: string) {
 	console.log("[WebSocket] closed", code, reason);
-	if (this.session_id) await Session.delete({ session_id: this.session_id });
+	if (this.heartbeatTimeout) clearTimeout(this.heartbeatTimeout);
-	// @ts-ignore
+	if (this.readyTimeout) clearTimeout(this.readyTimeout);
-	this.off("message", Message);
+	this.deflate?.close();
+	this.removeAllListeners();
+
+	if (this.session_id) {
+		await Session.delete({ session_id: this.session_id });
+		const sessions = await Session.find({
+			where: { user_id: this.user_id },
+			select: PrivateSessionProjection,
+		});
+		await emitEvent({
+			event: "SESSIONS_REPLACE",
+			user_id: this.user_id,
+			data: sessions,
+		} as SessionsReplace);
+		const session = sessions.first() || {
+			activities: [],
+			client_info: {},
+			status: "offline",
+		};
+
+		await emitEvent({
+			event: "PRESENCE_UPDATE",
+			user_id: this.user_id,
+			data: {
+				user: await User.getPublicUser(this.user_id),
+				activities: session.activities,
+				client_status: session?.client_info,
+				status: session.status,
+			},
+		} as PresenceUpdateEvent);
+	}
 }

gateway/src/events/Connection.ts

@@ -8,7 +8,6 @@ import { Close } from "./Close";
 import { Message } from "./Message";
 import { createDeflate } from "zlib";
 import { URL } from "url";
-import { Session } from "@fosscord/util";
 var erlpack: any;
 try {
 	erlpack = require("@yukikaze-bot/erlpack");
@@ -24,9 +23,11 @@ export async function Connection(
 	request: IncomingMessage
 ) {
 	try {
+		// @ts-ignore
 		socket.on("close", Close);
 		// @ts-ignore
 		socket.on("message", Message);
+		console.log(`[Gateway] Connections: ${this.clients.size}`);
 
 		const { searchParams } = new URL(`http://localhost${request.url}`);
 		// @ts-ignore
@@ -55,6 +56,7 @@ export async function Connection(
 		}
 
 		socket.events = {};
+		socket.member_events = {};
 		socket.permissions = {};
 		socket.sequence = 0;
 
@@ -68,12 +70,10 @@ export async function Connection(
 		});
 
 		socket.readyTimeout = setTimeout(() => {
-			Session.delete({ session_id: socket.session_id }); //should we await?
 			return socket.close(CLOSECODES.Session_timed_out);
 		}, 1000 * 30);
 	} catch (error) {
 		console.error(error);
-		Session.delete({ session_id: socket.session_id }); //should we await?
 		return socket.close(CLOSECODES.Unknown_error);
 	}
 }

gateway/src/events/Message.ts

@@ -37,8 +37,6 @@ export async function Message(this: WebSocket, buffer: WS.Data) {
 		return;
 	}
 
-	console.log("[Gateway] Opcode " + OPCODES[data.op]);
-
 	try {
 		return await OPCodeHandler.call(this, data);
 	} catch (error) {

gateway/src/listener/listener.ts

@@ -6,6 +6,9 @@ import {
 	EventOpts,
 	ListenEventOpts,
 	Member,
+	EVENTEnum,
+	Relationship,
+	RelationshipType,
 } from "@fosscord/util";
 import { OPCODES } from "../util/Constants";
 import { Send } from "../util/Send";
@@ -21,22 +24,45 @@ import { Recipient } from "@fosscord/util";
 // Sharding: calculate if the current shard id matches the formula: shard_id = (guild_id >> 22) % num_shards
 // https://discord.com/developers/docs/topics/gateway#sharding
 
+export function handlePresenceUpdate(
+	this: WebSocket,
+	{ event, acknowledge, data }: EventOpts
+) {
+	acknowledge?.();
+	if (event === EVENTEnum.PresenceUpdate) {
+		return Send(this, {
+			op: OPCODES.Dispatch,
+			t: event,
+			d: data,
+			s: this.sequence++,
+		});
+	}
+}
+
 // TODO: use already queried guilds/channels of Identify and don't fetch them again
 export async function setupListener(this: WebSocket) {
-	const members = await Member.find({
+	const [members, recipients, relationships] = await Promise.all([
-		where: { id: this.user_id },
+		Member.find({
-		relations: ["guild", "guild.channels"],
+			where: { id: this.user_id },
-	});
+			relations: ["guild", "guild.channels"],
+		}),
+		Recipient.find({
+			where: { user_id: this.user_id, closed: false },
+			relations: ["channel"],
+		}),
+		Relationship.find({
+			from_id: this.user_id,
+			type: RelationshipType.friends,
+		}),
+	]);
+
 	const guilds = members.map((x) => x.guild);
-	const recipients = await Recipient.find({
-		where: { user_id: this.user_id, closed: false },
-		relations: ["channel"],
-	});
 	const dm_channels = recipients.map((x) => x.channel);
 
 	const opts: { acknowledge: boolean; channel?: AMQChannel } = {
 		acknowledge: true,
 	};
+	this.listen_options = opts;
 	const consumer = consume.bind(this);
 
 	if (RabbitMQ.connection) {
@@ -47,45 +73,44 @@ export async function setupListener(this: WebSocket) {
 
 	this.events[this.user_id] = await listenEvent(this.user_id, consumer, opts);
 
-	for (const channel of dm_channels) {
+	relationships.forEach(async (relationship) => {
+		this.events[relationship.to_id] = await listenEvent(
+			relationship.to_id,
+			handlePresenceUpdate.bind(this),
+			opts
+		);
+	});
+
+	dm_channels.forEach(async (channel) => {
 		this.events[channel.id] = await listenEvent(channel.id, consumer, opts);
-	}
+	});
 
-	for (const guild of guilds) {
+	guilds.forEach(async (guild) => {
-		// contains guild and dm channels
+		const permission = await getPermission(this.user_id, guild.id);
+		this.permissions[guild.id] = permission;
+		this.events[guild.id] = await listenEvent(guild.id, consumer, opts);
 
-		getPermission(this.user_id, guild.id)
+		guild.channels.forEach(async (channel) => {
-			.then(async (x) => {
+			if (
-				this.permissions[guild.id] = x;
+				permission
-				this.listeners;
+					.overwriteChannel(channel.permission_overwrites!)
-				this.events[guild.id] = await listenEvent(
+					.has("VIEW_CHANNEL")
-					guild.id,
+			) {
+				this.events[channel.id] = await listenEvent(
+					channel.id,
 					consumer,
 					opts
 				);
-
+			}
-				for (const channel of guild.channels) {
+		});
-					if (
+	});
-						x
-							.overwriteChannel(channel.permission_overwrites!)
-							.has("VIEW_CHANNEL")
-					) {
-						this.events[channel.id] = await listenEvent(
-							channel.id,
-							consumer,
-							opts
-						);
-					}
-				}
-			})
-			.catch((e) =>
-				console.log("couldn't get permission for guild " + guild, e)
-			);
-	}
 
 	this.once("close", () => {
 		if (opts.channel) opts.channel.close();
-		else Object.values(this.events).forEach((x) => x());
+		else {
+			Object.values(this.events).forEach((x) => x());
+			Object.values(this.member_events).forEach((x) => x());
+		}
 	});
 }
 
@@ -97,10 +122,23 @@ async function consume(this: WebSocket, opts: EventOpts) {
 
 	const consumer = consume.bind(this);
 	const listenOpts = opts as ListenEventOpts;
+	opts.acknowledge?.();
 	// console.log("event", event);
 
 	// subscription managment
 	switch (event) {
+		case "GUILD_MEMBER_REMOVE":
+			this.member_events[data.user.id]?.();
+			delete this.member_events[data.user.id];
+		case "GUILD_MEMBER_ADD":
+			if (this.member_events[data.user.id]) break; // already subscribed
+			this.member_events[data.user.id] = await listenEvent(
+				data.user.id,
+				handlePresenceUpdate.bind(this),
+				this.listen_options
+			);
+			break;
+		case "RELATIONSHIP_REMOVE":
 		case "CHANNEL_DELETE":
 		case "GUILD_DELETE":
 			delete this.events[id];
@@ -178,7 +216,7 @@ async function consume(this: WebSocket, opts: EventOpts) {
 		case "CHANNEL_CREATE":
 		case "CHANNEL_DELETE":
 		case "CHANNEL_UPDATE":
-		case "GUILD_EMOJI_UPDATE":
+		case "GUILD_EMOJIS_UPDATE":
 		case "READY": // will be sent by the gateway
 		case "USER_UPDATE":
 		case "APPLICATION_COMMAND_CREATE":
@@ -196,5 +234,4 @@ async function consume(this: WebSocket, opts: EventOpts) {
 		d: data,
 		s: this.sequence++,
 	});
-	opts.acknowledge?.();
 }

gateway/src/opcodes/Identify.ts

@@ -12,6 +12,12 @@ import {
 	PublicUser,
 	PrivateUserProjection,
 	ReadState,
+	Application,
+	emitEvent,
+	SessionsReplace,
+	PrivateSessionProjection,
+	MemberPrivateProjection,
+	PresenceUpdateEvent,
 } from "@fosscord/util";
 import { Send } from "../util/Send";
 import { CLOSECODES, OPCODES } from "../util/Constants";
@@ -41,7 +47,61 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 		return this.close(CLOSECODES.Authentication_failed);
 	}
 	this.user_id = decoded.id;
-	if (!identify.intents) identify.intents = 0b11111111111111n;
+
+	const session_id = genSessionId();
+	this.session_id = session_id; //Set the session of the WebSocket object
+
+	const [user, read_states, members, recipients, session, application] =
+		await Promise.all([
+			User.findOneOrFail({
+				where: { id: this.user_id },
+				relations: ["relationships", "relationships.to"],
+				select: [...PrivateUserProjection, "relationships"],
+			}),
+			ReadState.find({ user_id: this.user_id }),
+			Member.find({
+				where: { id: this.user_id },
+				select: MemberPrivateProjection,
+				relations: [
+					"guild",
+					"guild.channels",
+					"guild.emojis",
+					"guild.emojis.user",
+					"guild.roles",
+					"guild.stickers",
+					"user",
+					"roles",
+				],
+			}),
+			Recipient.find({
+				where: { user_id: this.user_id, closed: false },
+				relations: [
+					"channel",
+					"channel.recipients",
+					"channel.recipients.user",
+				],
+				// TODO: public user selection
+			}),
+			// save the session and delete it when the websocket is closed
+			new Session({
+				user_id: this.user_id,
+				session_id: session_id,
+				// TODO: check if status is only one of: online, dnd, offline, idle
+				status: identify.presence?.status || "online", //does the session always start as online?
+				client_info: {
+					//TODO read from identity
+					client: "desktop",
+					os: identify.properties?.os,
+					version: 0,
+				},
+				activities: [],
+			}).save(),
+			Application.findOne({ id: this.user_id }),
+		]);
+
+	if (!user) return this.close(CLOSECODES.Authentication_failed);
+
+	if (!identify.intents) identify.intents = BigInt("0b11111111111111");
 	this.intents = new Intents(identify.intents);
 	if (identify.shard) {
 		this.shard_id = identify.shard[0];
@@ -59,18 +119,6 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 	}
 	var users: PublicUser[] = [];
 
-	const members = await Member.find({
-		where: { id: this.user_id },
-		relations: [
-			"guild",
-			"guild.channels",
-			"guild.emojis",
-			"guild.roles",
-			"guild.stickers",
-			"user",
-			"roles",
-		],
-	});
 	const merged_members = members.map((x: Member) => {
 		return [
 			{
@@ -81,19 +129,32 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 			},
 		];
 	}) as PublicMember[][];
-	const guilds = members.map((x) => ({ ...x.guild, joined_at: x.joined_at }));
+	let guilds = members.map((x) => ({ ...x.guild, joined_at: x.joined_at }));
+
+	// @ts-ignore
+	guilds = guilds.map((guild) => {
+		if (user.bot) {
+			setTimeout(() => {
+				Send(this, {
+					op: OPCODES.Dispatch,
+					t: EVENTEnum.GuildCreate,
+					s: this.sequence++,
+					d: guild,
+				});
+			}, 500);
+			return { id: guild.id, unavailable: true };
+		}
+
+		return guild;
+	});
+
 	const user_guild_settings_entries = members.map((x) => x.settings);
 
-	const recipients = await Recipient.find({
-		where: { user_id: this.user_id, closed: false },
-		relations: ["channel", "channel.recipients", "channel.recipients.user"],
-		// TODO: public user selection
-	});
 	const channels = recipients.map((x) => {
 		// @ts-ignore
 		x.channel.recipients = x.channel.recipients?.map((x) => x.user);
 		//TODO is this needed? check if users in group dm that are not friends are sent in the READY event
-		//users = users.concat(x.channel.recipients);
+		users = users.concat(x.channel.recipients as unknown as User[]);
 		if (x.channel.isDm()) {
 			x.channel.recipients = x.channel.recipients!.filter(
 				(x) => x.id !== this.user_id
@@ -101,12 +162,6 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 		}
 		return x.channel;
 	});
-	const user = await User.findOneOrFail({
-		where: { id: this.user_id },
-		relations: ["relationships", "relationships.to"],
-		select: [...PrivateUserProjection, "relationships"],
-	});
-	if (!user) return this.close(CLOSECODES.Authentication_failed);
 
 	for (let relation of user.relationships) {
 		const related_user = relation.to;
@@ -122,24 +177,28 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 		users.push(public_related_user);
 	}
 
-	const session_id = genSessionId();
+	setImmediate(async () => {
-	this.session_id = session_id; //Set the session of the WebSocket object
+		// run in seperate "promise context" because ready payload is not dependent on those events
-	const session = new Session({
+		emitEvent({
-		user_id: this.user_id,
+			event: "SESSIONS_REPLACE",
-		session_id: session_id,
+			user_id: this.user_id,
-		status: "online", //does the session always start as online?
+			data: await Session.find({
-		client_info: {
+				where: { user_id: this.user_id },
-			//TODO read from identity
+				select: PrivateSessionProjection,
-			client: "desktop",
+			}),
-			os: "linux",
+		} as SessionsReplace);
-			version: 0,
+		emitEvent({
-		},
+			event: "PRESENCE_UPDATE",
+			user_id: this.user_id,
+			data: {
+				user: await User.getPublicUser(this.user_id),
+				activities: session.activities,
+				client_status: session?.client_info,
+				status: session.status,
+			},
+		} as PresenceUpdateEvent);
 	});
 
-	//We save the session and we delete it when the websocket is closed
-	await session.save();
-
-	const read_states = await ReadState.find({ user_id: this.user_id });
 	read_states.forEach((s: any) => {
 		s.id = s.channel_id;
 		delete s.user_id;
@@ -170,6 +229,7 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 
 	const d: ReadyEventData = {
 		v: 8,
+		application,
 		user: privateUser,
 		user_settings: user.settings,
 		// @ts-ignore
@@ -178,6 +238,8 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 			x.guild_hashes = {}; // @ts-ignore
 			x.guild_scheduled_events = []; // @ts-ignore
 			x.threads = [];
+			x.premium_subscription_count = 30;
+			x.premium_tier = 3;
 			return x;
 		}),
 		guild_experiments: [], // TODO
@@ -207,14 +269,11 @@ export async function onIdentify(this: WebSocket, data: Payload) {
 		// @ts-ignore
 		experiments: experiments, // TODO
 		guild_join_requests: [], // TODO what is this?
-		users: users.unique(),
+		users: users.filter((x) => x).unique(),
 		merged_members: merged_members,
 		// shard // TODO: only for bots sharding
-		// application // TODO for applications
 	};
 
-	console.log("Send ready");
-
 	// TODO: send real proper data structure
 	await Send(this, {
 		op: OPCODES.Dispatch,

gateway/src/opcodes/LazyRequest.ts

@@ -1,46 +1,56 @@
 import {
+	EVENTEnum,
+	EventOpts,
 	getPermission,
+	listenEvent,
 	Member,
-	PublicMemberProjection,
 	Role,
 } from "@fosscord/util";
 import { LazyRequest } from "../schema/LazyRequest";
 import { Send } from "../util/Send";
 import { OPCODES } from "../util/Constants";
-import { WebSocket, Payload } from "@fosscord/gateway";
+import { WebSocket, Payload, handlePresenceUpdate } from "@fosscord/gateway";
 import { check } from "./instanceOf";
 import "missing-native-js-functions";
+import { getRepository } from "typeorm";
+import "missing-native-js-functions";
 
-// TODO: check permission and only show roles/members that have access to this channel
+// TODO: only show roles/members that have access to this channel
 // TODO: config: to list all members (even those who are offline) sorted by role, or just those who are online
 // TODO: rewrite typeorm
 
-export async function onLazyRequest(this: WebSocket, { d }: Payload) {
+async function getMembers(guild_id: string, range: [number, number]) {
-	// TODO: check data
+	if (!Array.isArray(range) || range.length !== 2) {
-	check.call(this, LazyRequest, d);
+		throw new Error("range is not a valid array");
-	const { guild_id, typing, channels, activities } = d as LazyRequest;
+	}
+	// TODO: wait for typeorm to implement ordering for .find queries https://github.com/typeorm/typeorm/issues/2620
 
-	const permissions = await getPermission(this.user_id, guild_id);
+	let members = await getRepository(Member)
-	permissions.hasThrow("VIEW_CHANNEL");
+		.createQueryBuilder("member")
-
+		.where("member.guild_id = :guild_id", { guild_id })
-	var members = await Member.find({
+		.leftJoinAndSelect("member.roles", "role")
-		where: { guild_id: guild_id },
+		.leftJoinAndSelect("member.user", "user")
-		relations: ["roles", "user"],
+		.leftJoinAndSelect("user.sessions", "session")
-		select: PublicMemberProjection,
+		.addSelect(
-	});
+			"CASE WHEN session.status = 'offline' THEN 0 ELSE 1 END",
-
+			"_status"
-	const roles = await Role.find({
+		)
-		where: { guild_id: guild_id },
+		.orderBy("role.position", "DESC")
-		order: {
+		.addOrderBy("_status", "DESC")
-			position: "DESC",
+		.addOrderBy("user.username", "ASC")
-		},
+		.offset(Number(range[0]) || 0)
-	});
+		.limit(Number(range[1]) || 100)
+		.getMany();
 
 	const groups = [] as any[];
-	var member_count = 0;
 	const items = [];
+	const member_roles = members
+		.map((m) => m.roles)
+		.flat()
+		.unique((r) => r.id);
 
-	for (const role of roles) {
+	for (const role of member_roles) {
+		// @ts-ignore
 		const [role_members, other_members] = partition(members, (m: Member) =>
 			m.roles.find((r) => r.id === role.id)
 		);
@@ -53,38 +63,94 @@ export async function onLazyRequest(this: WebSocket, { d }: Payload) {
 		groups.push(group);
 
 		for (const member of role_members) {
-			member.roles = member.roles.filter((x) => x.id !== guild_id);
+			const roles = member.roles
+				.filter((x: Role) => x.id !== guild_id)
+				.map((x: Role) => x.id);
+
+			const session = member.user.sessions.first();
+
+			// TODO: properly mock/hide offline/invisible status
 			items.push({
-				member: { ...member, roles: member.roles.map((x) => x.id) },
+				member: {
+					...member,
+					roles,
+					user: { ...member.user, sessions: undefined },
+					presence: {
+						...session,
+						activities: session?.activities || [],
+						user: { id: member.user.id },
+					},
+				},
 			});
 		}
 		members = other_members;
-		member_count += role_members.length;
 	}
 
+	return {
+		items,
+		groups,
+		range,
+		members: items.map((x) => x.member).filter((x) => x),
+	};
+}
+
+export async function onLazyRequest(this: WebSocket, { d }: Payload) {
+	// TODO: check data
+	check.call(this, LazyRequest, d);
+	const { guild_id, typing, channels, activities } = d as LazyRequest;
+
+	const channel_id = Object.keys(channels || {}).first();
+	if (!channel_id) return;
+
+	const permissions = await getPermission(this.user_id, guild_id, channel_id);
+	permissions.hasThrow("VIEW_CHANNEL");
+
+	const ranges = channels![channel_id];
+	if (!Array.isArray(ranges)) throw new Error("Not a valid Array");
+
+	const member_count = await Member.count({ guild_id });
+	const ops = await Promise.all(ranges.map((x) => getMembers(guild_id, x)));
+
+	// TODO: unsubscribe member_events that are not in op.members
+
+	ops.forEach((op) => {
+		op.members.forEach(async (member) => {
+			if (this.events[member.user.id]) return; // already subscribed as friend
+			if (this.member_events[member.user.id]) return; // already subscribed in member list
+			this.member_events[member.user.id] = await listenEvent(
+				member.user.id,
+				handlePresenceUpdate.bind(this),
+				this.listen_options
+			);
+		});
+	});
+
 	return Send(this, {
 		op: OPCODES.Dispatch,
 		s: this.sequence++,
 		t: "GUILD_MEMBER_LIST_UPDATE",
 		d: {
-			ops: [
+			ops: ops.map((x) => ({
-				{
+				items: x.items,
-					range: [0, 99],
+				op: "SYNC",
-					op: "SYNC",
+				range: x.range,
-					items,
+			})),
-				},
+			online_count: member_count,
-			],
-			online_count: member_count, // TODO count online count
 			member_count,
 			id: "everyone",
 			guild_id,
-			groups,
+			groups: ops
+				.map((x) => x.groups)
+				.flat()
+				.unique(),
 		},
 	});
 }
 
 function partition<T>(array: T[], isValid: Function) {
+	// @ts-ignore
 	return array.reduce(
+		// @ts-ignore
 		([pass, fail], elem) => {
 			return isValid(elem)
 				? [[...pass, elem], fail]

gateway/src/opcodes/PresenceUpdate.ts

@@ -1,5 +1,25 @@
 import { WebSocket, Payload } from "@fosscord/gateway";
+import { emitEvent, PresenceUpdateEvent, Session, User } from "@fosscord/util";
+import { ActivitySchema } from "../schema/Activity";
+import { check } from "./instanceOf";
 
-export function onPresenceUpdate(this: WebSocket, data: Payload) {
+export async function onPresenceUpdate(this: WebSocket, { d }: Payload) {
-	// return this.close(CLOSECODES.Unknown_error);
+	check.call(this, ActivitySchema, d);
+	const presence = d as ActivitySchema;
+
+	await Session.update(
+		{ session_id: this.session_id },
+		{ status: presence.status, activities: presence.activities }
+	);
+
+	await emitEvent({
+		event: "PRESENCE_UPDATE",
+		user_id: this.user_id,
+		data: {
+			user: await User.getPublicUser(this.user_id),
+			activities: presence.activities,
+			client_status: {}, // TODO:
+			status: presence.status,
+		},
+	} as PresenceUpdateEvent);
 }

gateway/src/schema/Activity.ts

@@ -1,4 +1,4 @@
-import { EmojiSchema } from "./Emoji";
+import { Activity, Status } from "@fosscord/util";
 
 export const ActivitySchema = {
 	afk: Boolean,
@@ -21,7 +21,7 @@ export const ActivitySchema = {
 			$emoji: {
 				$name: String,
 				$id: String,
-				$amimated: Boolean,
+				$animated: Boolean,
 			},
 			$party: {
 				$id: String,
@@ -47,40 +47,7 @@ export const ActivitySchema = {
 
 export interface ActivitySchema {
 	afk: boolean;
-	status: string;
+	status: Status;
-	activities?: [
+	activities?: Activity[];
-		{
-			name: string; // the activity's name
-			type: number; // activity type // TODO: check if its between range 0-5
-			url?: string; // stream url, is validated when type is 1
-			created_at?: number; // unix timestamp of when the activity was added to the user's session
-			timestamps?: {
-				// unix timestamps for start and/or end of the game
-				start: number;
-				end: number;
-			};
-			application_id?: string; // application id for the game
-			details?: string;
-			state?: string;
-			emoji?: EmojiSchema;
-			party?: {
-				id?: string;
-				size?: [number]; // used to show the party's current and maximum size // TODO: array length 2
-			};
-			assets?: {
-				large_image?: string; // the id for a large asset of the activity, usually a snowflake
-				large_text?: string; // text displayed when hovering over the large image of the activity
-				small_image?: string; // the id for a small asset of the activity, usually a snowflake
-				small_text?: string; // text displayed when hovering over the small image of the activity
-			};
-			secrets?: {
-				join?: string; // the secret for joining a party
-				spectate?: string; // the secret for spectating a game
-				match?: string; // the secret for a specific instanced match
-			};
-			instance?: boolean;
-			flags: string; // activity flags OR d together, describes what the payload includes
-		}
-	];
 	since?: number; // unix time (in milliseconds) of when the client went idle, or null if the client is not idle
 }

gateway/src/schema/Emoji.ts

@@ -1,11 +0,0 @@
-export const EmojiSchema = {
-	name: String, // the name of the emoji
-	$id: String, // the id of the emoji
-	animated: Boolean, // whether this emoji is animated
-};
-
-export interface EmojiSchema {
-	name: string;
-	id?: string;
-	animated: Boolean;
-}

gateway/src/schema/LazyRequest.ts

@@ -1,6 +1,6 @@
 export interface LazyRequest {
 	guild_id: string;
-	channels?: Record<string, [number, number]>;
+	channels?: Record<string, [number, number][]>;
 	activities?: boolean;
 	threads?: boolean;
 	typing?: true;

gateway/src/util/Send.ts

@@ -18,6 +18,9 @@ export async function Send(socket: WebSocket, data: Payload) {
 	}
 
 	return new Promise((res, rej) => {
+		if (socket.readyState !== 1) {
+			return rej("socket not open");
+		}
 		socket.send(buffer, (err: any) => {
 			if (err) return rej(err);
 			return res(null);

gateway/src/util/WebSocket.ts

@@ -17,4 +17,6 @@ export interface WebSocket extends WS {
 	sequence: number;
 	permissions: Record<string, Permissions>;
 	events: Record<string, Function>;
+	member_events: Record<string, Function>;
+	listen_options: any;
 }

gateway/tsconfig.json

@@ -27,7 +27,7 @@
 		// "isolatedModules": true,               /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
 
 		/* Strict Type-Checking Options */
-		"strict": false /* Enable all strict type-checking options. */,
+		"strict": true /* Enable all strict type-checking options. */,
 		"noImplicitAny": true /* Raise error on expressions and declarations with an implied 'any' type. */,
 		"strictNullChecks": true /* Enable strict null checks. */,
 		// "strictFunctionTypes": true,           /* Enable strict checking of function types. */

rtc/package-lock.json

@@ -0,0 +1,6 @@
+{
+  "name": "rtc",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {}
+}

rtc/package.json

@@ -0,0 +1 @@
+{}

util/ormconfig.json

@@ -0,0 +1,9 @@
+{
+	"type": "sqlite",
+	"database": "../bundle/database.db",
+	"migrations": ["src/migrations/*.ts"],
+	"entities": ["src/entities/*.ts"],
+	"cli": {
+		"migrationsDir": "src/migrations"
+	}
+}

util/package.json

@@ -8,7 +8,8 @@
 		"start": "npm run build && node dist/",
 		"test": "npm run build && jest",
 		"postinstall": "npm run build",
-		"build": "npx tsc -b ."
+		"build": "npx tsc -p .",
+		"typeorm": "node --require ts-node/register ./node_modules/typeorm/cli.js"
 	},
 	"repository": {
 		"type": "git",
@@ -28,25 +29,19 @@
 	},
 	"homepage": "https://docs.fosscord.com/",
 	"devDependencies": {
-		"@swc/cli": "^0.1.51",
-		"@swc/core": "^1.2.93",
 		"@types/amqplib": "^0.8.1",
 		"@types/jsonwebtoken": "^8.5.0",
-		"@types/mongoose-autopopulate": "^0.10.1",
 		"@types/multer": "^1.4.7",
 		"@types/node": "^14.17.9",
 		"@types/node-fetch": "^2.5.12",
-		"jest": "^27.0.6"
+		"jest": "^27.0.6",
+		"ts-node": "^10.2.1"
 	},
 	"dependencies": {
-		"ajv": "^8.6.2",
 		"amqplib": "^0.8.0",
-		"class-validator": "^0.13.1",
-		"dot-prop": "^6.0.1",
-		"env-paths": "^2.2.1",
 		"jsonwebtoken": "^8.5.1",
-		"lambert-server": "^1.2.11",
+		"lambert-server": "^1.2.12",
-		"missing-native-js-functions": "^1.2.17",
+		"missing-native-js-functions": "^1.2.18",
 		"multer": "^1.4.3",
 		"nanocolors": "^0.2.12",
 		"node-fetch": "^2.6.1",
@@ -54,8 +49,7 @@
 		"pg": "^8.7.1",
 		"reflect-metadata": "^0.1.13",
 		"sqlite3": "^5.0.2",
-		"tsconfig-paths": "^3.11.0",
+		"typeorm": "^0.2.38",
-		"typeorm": "^0.2.37",
 		"typescript": "^4.4.2",
 		"typescript-json-schema": "^0.50.1"
 	},

util/scripts/migrate_db_engine.js

@@ -0,0 +1,109 @@
+const { config } = require("dotenv");
+config();
+const { createConnection } = require("typeorm");
+const { initDatabase } = require("../../dist/util/Database");
+require("missing-native-js-functions");
+const {
+	Application,
+	Attachment,
+	Ban,
+	Channel,
+	ConfigEntity,
+	ConnectedAccount,
+	Emoji,
+	Guild,
+	Invite,
+	Member,
+	Message,
+	ReadState,
+	Recipient,
+	Relationship,
+	Role,
+	Sticker,
+	Team,
+	TeamMember,
+	Template,
+	User,
+	VoiceState,
+	Webhook,
+} = require("../../dist/entities/index");
+
+async function main() {
+	if (!process.env.TO) throw new Error("TO database env connection string not set");
+
+	// manually arrange them because of foreign keys
+	const entities = [
+		ConfigEntity,
+		User,
+		Guild,
+		Channel,
+		Invite,
+		Role,
+		Ban,
+		Application,
+		Emoji,
+		ConnectedAccount,
+		Member,
+		ReadState,
+		Recipient,
+		Relationship,
+		Sticker,
+		Team,
+		TeamMember,
+		Template,
+		VoiceState,
+		Webhook,
+		Message,
+		Attachment,
+	];
+
+	const oldDB = await initDatabase();
+
+	const type = process.env.TO.includes("://") ? process.env.TO.split(":")[0]?.replace("+srv", "") : "sqlite";
+	const isSqlite = type.includes("sqlite");
+
+	// @ts-ignore
+	const newDB = await createConnection({
+		type,
+		url: isSqlite ? undefined : process.env.TO,
+		database: isSqlite ? process.env.TO : undefined,
+		entities,
+		name: "new",
+		synchronize: true,
+	});
+	let i = 0;
+
+	try {
+		for (const entity of entities) {
+			const entries = await oldDB.manager.find(entity);
+
+			// @ts-ignore
+			console.log("migrating " + entries.length + " " + entity.name + " ...");
+
+			for (const entry of entries) {
+				console.log(i++);
+
+				try {
+					await newDB.manager.insert(entity, entry);
+				} catch (error) {
+					try {
+						if (!entry.id) throw new Error("object doesn't have a unique id: " + entry);
+						await newDB.manager.update(entity, { id: entry.id }, entry);
+					} catch (error) {
+						console.error("couldn't migrate " + i + " " + entity.name, error);
+					}
+				}
+			}
+
+			// @ts-ignore
+			console.log("migrated " + entries.length + " " + entity.name);
+		}
+	} catch (error) {
+		console.error(error.message);
+	}
+
+	console.log("SUCCESS migrated all data");
+	await newDB.close();
+}
+
+main().caught();

util/src/entities/AuditLog.ts

@@ -55,10 +55,7 @@ export class AuditLog extends BaseClass {
 	@ManyToOne(() => User, (user: User) => user.id)
 	user: User;
 
-	@Column({
+	@Column({ type: "int" })
-		type: "simple-enum",
-		enum: AuditLogEvents,
-	})
 	action_type: AuditLogEvents;
 
 	@Column({ type: "simple-json", nullable: true })

util/src/entities/BaseClass.ts

@@ -1,19 +1,8 @@
 import "reflect-metadata";
-import {
+import { BaseEntity, EntityMetadata, FindConditions, ObjectIdColumn, PrimaryColumn } from "typeorm";
-	BaseEntity,
-	BeforeInsert,
-	BeforeUpdate,
-	EntityMetadata,
-	FindConditions,
-	ObjectIdColumn,
-	PrimaryColumn,
-} from "typeorm";
 import { Snowflake } from "../util/Snowflake";
 import "missing-native-js-functions";
 
-// TODO use class-validator https://typeorm.io/#/validation with class annotators (isPhone/isEmail) combined with types from typescript-json-schema
-// btw. we don't use class-validator for everything, because we need to explicitly set the type instead of deriving it from typescript also it doesn't easily support nested objects
-
 export class BaseClassWithoutId extends BaseEntity {
 	constructor(props?: any) {
 		super();
@@ -42,7 +31,7 @@ export class BaseClassWithoutId extends BaseEntity {
 		for (const key in props) {
 			if (!properties.has(key)) continue;
 			// @ts-ignore
-			const setter = this[`set${key.capitalize()}`];
+			const setter = this[`set${key.capitalize()}`]; // use setter function if it exists
 
 			if (setter) {
 				setter.call(this, props[key]);
@@ -53,12 +42,6 @@ export class BaseClassWithoutId extends BaseEntity {
 		}
 	}
 
-	@BeforeUpdate()
-	@BeforeInsert()
-	validate() {
-		return this;
-	}
-
 	toJSON(): any {
 		return Object.fromEntries(
 			this.metadata.columns // @ts-ignore
@@ -76,42 +59,6 @@ export class BaseClassWithoutId extends BaseEntity {
 		const repository = this.getRepository();
 		return repository.decrement(conditions, propertyPath, value);
 	}
-
-	// static async delete<T>(criteria: FindConditions<T>, options?: RemoveOptions) {
-	// 	if (!criteria) throw new Error("You need to specify delete criteria");
-
-	// 	const repository = this.getRepository();
-	// 	const promises = repository.metadata.relations.map(async (x) => {
-	// 		if (x.orphanedRowAction !== "delete") return;
-
-	// 		const foreignKey =
-	// 			x.foreignKeys.find((key) => key.entityMetadata === repository.metadata) ||
-	// 			x.inverseRelation?.foreignKeys[0]; // find foreign key for this entity
-	// 		if (!foreignKey) {
-	// 			throw new Error(
-	// 				`Foreign key not found for entity ${repository.metadata.name} in relation ${x.propertyName}`
-	// 			);
-	// 		}
-	// 		const id = (criteria as any)[foreignKey.referencedColumnNames[0]];
-	// 		if (!id) throw new Error("id missing in criteria options " + foreignKey.referencedColumnNames);
-
-	// 		if (x.relationType === "many-to-many") {
-	// 			return getConnection()
-	// 				.createQueryBuilder()
-	// 				.relation(this, x.propertyName)
-	// 				.of(id)
-	// 				.remove({ [foreignKey.columnNames[0]]: id });
-	// 		} else if (
-	// 			x.relationType === "one-to-one" ||
-	// 			x.relationType === "many-to-one" ||
-	// 			x.relationType === "one-to-many"
-	// 		) {
-	// 			return (x.inverseEntityMetadata.target as any).delete({ [foreignKey.columnNames[0]]: id });
-	// 		}
-	// 	});
-	// 	await Promise.all(promises);
-	// 	return super.delete(criteria, options);
-	// }
 }
 
 export const PrimaryIdColumn = process.env.DATABASE?.startsWith("mongodb") ? ObjectIdColumn : PrimaryColumn;

util/src/entities/Channel.ts

@@ -39,7 +39,7 @@ export class Channel extends BaseClass {
 	@Column({ type: "text", nullable: true })
 	icon?: string | null;
 
-	@Column({ type: "simple-enum", enum: ChannelType })
+	@Column({ type: "int" })
 	type: ChannelType;
 
 	@OneToMany(() => Recipient, (recipient: Recipient) => recipient.channel, {

util/src/entities/Config.ts

@@ -51,11 +51,6 @@ export interface ConfigValue {
 	general: {
 		instanceId: string;
 	};
-	permissions: {
-		user: {
-			createGuilds: boolean;
-		};
-	};
 	limits: {
 		user: {
 			maxGuilds: number;
@@ -64,6 +59,7 @@ export interface ConfigValue {
 		};
 		guild: {
 			maxRoles: number;
+			maxEmojis: number;
 			maxMembers: number;
 			maxChannels: number;
 			maxChannelsInCategory: number;
@@ -153,6 +149,11 @@ export interface ConfigValue {
 			canLeave: boolean;
 		};
 	};
+	gif: {
+		enabled: boolean;
+		provider: "tenor"; // more coming soon
+		apiKey?: string;
+	};
 	rabbitmq: {
 		host: string | null;
 	};
@@ -175,11 +176,6 @@ export const DefaultConfigOptions: ConfigValue = {
 	general: {
 		instanceId: Snowflake.generate(),
 	},
-	permissions: {
-		user: {
-			createGuilds: true,
-		},
-	},
 	limits: {
 		user: {
 			maxGuilds: 100,
@@ -188,6 +184,7 @@ export const DefaultConfigOptions: ConfigValue = {
 		},
 		guild: {
 			maxRoles: 250,
+			maxEmojis: 50, // TODO: max emojis per guild per nitro level
 			maxMembers: 250000,
 			maxChannels: 500,
 			maxChannelsInCategory: 50,
@@ -305,7 +302,6 @@ export const DefaultConfigOptions: ConfigValue = {
 			},
 		],
 	},
-
 	guild: {
 		showAllGuildsInDiscovery: false,
 		autoJoin: {
@@ -314,6 +310,11 @@ export const DefaultConfigOptions: ConfigValue = {
 			guilds: [],
 		},
 	},
+	gif: {
+		enabled: true,
+		provider: "tenor",
+		apiKey: "LIVDSRZULELA",
+	},
 	rabbitmq: {
 		host: null,
 	},

util/src/entities/Emoji.ts

@@ -1,4 +1,5 @@
-import { Column, Entity, JoinColumn, ManyToOne } from "typeorm";
+import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm";
+import { User } from ".";
 import { BaseClass } from "./BaseClass";
 import { Guild } from "./Guild";
 import { Role } from "./Role";
@@ -20,6 +21,14 @@ export class Emoji extends BaseClass {
 	})
 	guild: Guild;
 
+	@Column({ nullable: true })
+	@RelationId((emoji: Emoji) => emoji.user)
+	user_id: string;
+
+	@JoinColumn({ name: "user_id" })
+	@ManyToOne(() => User)
+	user: User;
+
 	@Column()
 	managed: boolean;
 
@@ -28,4 +37,7 @@ export class Emoji extends BaseClass {
 
 	@Column()
 	require_colons: boolean;
+
+	@Column({ type: "simple-array" })
+	roles: string[]; // roles this emoji is whitelisted to (new discord feature?)
 }

util/src/entities/Guild.ts

@@ -257,14 +257,6 @@ export class Guild extends BaseClass {
 	@Column({ nullable: true })
 	unavailable?: boolean;
 
-	@Column({ nullable: true })
-	@RelationId((guild: Guild) => guild.vanity_url)
-	vanity_url_code?: string;
-
-	@JoinColumn({ name: "vanity_url_code" })
-	@ManyToOne(() => Invite)
-	vanity_url?: Invite;
-
 	@Column({ nullable: true })
 	verification_level?: number;
 

util/src/entities/Invite.ts

@@ -1,6 +1,6 @@
-import { Column, Entity, JoinColumn, ManyToOne, RelationId } from "typeorm";
+import { Column, Entity, JoinColumn, ManyToOne, RelationId, PrimaryColumn } from "typeorm";
 import { Member } from "./Member";
-import { BaseClass, PrimaryIdColumn } from "./BaseClass";
+import { BaseClassWithoutId } from "./BaseClass";
 import { Channel } from "./Channel";
 import { Guild } from "./Guild";
 import { User } from "./User";
@@ -8,8 +8,8 @@ import { User } from "./User";
 export const PublicInviteRelation = ["inviter", "guild", "channel"];
 
 @Entity("invites")
-export class Invite extends BaseClass {
+export class Invite extends BaseClassWithoutId {
-	@PrimaryIdColumn()
+	@PrimaryColumn()
 	code: string;
 
 	@Column()
@@ -71,6 +71,9 @@ export class Invite extends BaseClass {
 	@Column({ nullable: true })
 	target_user_type?: number;
 
+	@Column({ nullable: true})
+	vanity_url?: boolean;
+
 	static async joinGuild(user_id: string, code: string) {
 		const invite = await Invite.findOneOrFail({ code });
 		if (invite.uses++ >= invite.max_uses && invite.max_uses !== 0) await Invite.delete({ code });

util/src/entities/Member.ts

@@ -26,6 +26,22 @@ import { BaseClassWithoutId } from "./BaseClass";
 import { Ban, PublicGuildRelations } from ".";
 import { DiscordApiErrors } from "../util/Constants";
 
+export const MemberPrivateProjection: (keyof Member)[] = [
+	"id",
+	"guild",
+	"guild_id",
+	"deaf",
+	"joined_at",
+	"last_message_id",
+	"mute",
+	"nick",
+	"pending",
+	"premium_since",
+	"roles",
+	"settings",
+	"user",
+];
+
 @Entity("members")
 @Index(["id", "guild_id"], { unique: true })
 export class Member extends BaseClassWithoutId {
@@ -81,9 +97,12 @@ export class Member extends BaseClassWithoutId {
 	@Column()
 	pending: boolean;
 
-	@Column({ type: "simple-json" })
+	@Column({ type: "simple-json", select: false })
 	settings: UserGuildSettings;
 
+	@Column({ nullable: true })
+	last_message_id?: string;
+
 	// TODO: update
 	// @Column({ type: "simple-json" })
 	// read_state: ReadState;

util/src/entities/Message.ts

@@ -46,9 +46,6 @@ export enum MessageType {
 
 @Entity("messages")
 export class Message extends BaseClass {
-	@Column()
-	id: string;
-
 	@Column({ nullable: true })
 	@RelationId((message: Message) => message.channel)
 	channel_id: string;
@@ -130,7 +127,7 @@ export class Message extends BaseClass {
 	mention_channels: Channel[];
 
 	@JoinTable({ name: "message_stickers" })
-	@ManyToMany(() => Sticker)
+	@ManyToMany(() => Sticker, { cascade: true, onDelete: "CASCADE" })
 	sticker_items?: Sticker[];
 
 	@OneToMany(() => Attachment, (attachment: Attachment) => attachment.message, {
@@ -151,7 +148,7 @@ export class Message extends BaseClass {
 	@Column({ nullable: true })
 	pinned?: boolean;
 
-	@Column({ type: "simple-enum", enum: MessageType })
+	@Column({ type: "int" })
 	type: MessageType;
 
 	@Column({ type: "simple-json", nullable: true })