augh

util/src/util/Constants.ts

@@ -77,8 +77,9 @@ export const VoiceOPCodes = {
 	RESUME: 7,
 	HELLO: 8,
 	RESUMED: 9,
-	CLIENT_CONNECT: 12,
+	CLIENT_CONNECT: 12,		// incorrect, op 12 is probably used for video
-	CLIENT_DISCONNECT: 13,
+	CLIENT_DISCONNECT: 13,	// incorrect
+	VERSION: 16,	//not documented
 };
 
 export const Events = {

webrtc/src/Server.ts

@@ -6,6 +6,8 @@ import { setHeartbeat } from "./util";
 import * as mediasoup from "mediasoup";
 import { types as MediasoupTypes } from "mediasoup";
 
+import Net from "net";
+
 var port = Number(process.env.PORT);
 if (isNaN(port)) port = 3004;
 
@@ -13,7 +15,7 @@ export class Server {
 	public ws: WebSocketServer;
 	public mediasoupWorkers: MediasoupTypes.Worker[] = [];
 	public mediasoupRouters: MediasoupTypes.Router[] = [];
-	public mediasoupTransports: MediasoupTypes.Transport[] = [];
+	public mediasoupTransports: MediasoupTypes.WebRtcTransport[] = [];
 
 	constructor() {
 		this.ws = new WebSocketServer({
@@ -26,7 +28,7 @@ export class Server {
 			socket.on("message", async (message: string) => {
 				const payload: Payload = JSON.parse(message);
 
-				console.log(payload);
+				// console.log(payload);
 
 				if (OPCodeHandlers[payload.op])
 					try {
@@ -68,9 +70,13 @@ export class Server {
 
 				this.mediasoupRouters.push(router);
 
-				router.observer.on("newtransport", async (transport: MediasoupTypes.Transport) => {
+				router.observer.on("newtransport", async (transport: MediasoupTypes.WebRtcTransport) => {
 					console.log("new transport created [id:%s]", transport.id);
 
+					transport.observer.on("sctpstatechange", (state) => {
+						console.log(state)
+					});
+
 					await transport.enableTraceEvent();
 
 					transport.observer.on("newproducer", (producer: MediasoupTypes.Producer) => {

webrtc/src/opcodes/Heartbeat.ts

@@ -1,8 +1,8 @@
 import { WebSocket } from "@fosscord/gateway";
 import { Payload } from "./index";
-import { setHeartbeat } from "./../util";
+import { setHeartbeat } from "../util";
 import { Server } from "../Server"
 
 export async function onHeartbeat(this: Server, socket: WebSocket, data: Payload) {
-	await setHeartbeat(socket);
+	await setHeartbeat(socket, data.d);
 }

webrtc/src/opcodes/Identify.ts

@@ -28,12 +28,12 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify
 		}
 	);
 	const user = session.user;
-	const guild = await Guild.findOneOrFail({ id: data.d.server_id });
+	const guild = await Guild.findOneOrFail({ id: data.d.server_id }, { relations: ["members"] });
 
 	if (!guild.members.find(x => x.id === user.id))
 		return socket.close(CLOSECODES.Invalid_intent);
 
-	var transport = await this.mediasoupRouters[0].createWebRtcTransport({
+	var transport = this.mediasoupTransports[0] || await this.mediasoupRouters[0].createWebRtcTransport({
 		listenIps: [{ ip: "0.0.0.0", announcedIp: "127.0.0.1" }],
 		enableUdp: true,
 		enableTcp: true,
@@ -66,13 +66,39 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify
 		}
 	*/
 
+
+
+	/* 
+		{
+			"streams": [
+				{ "type": "video", "ssrc": 129861, "rtx_ssrc": 129862, "rid": "100", "quality": 100, "active": false }
+			],
+			"ssrc": 129860,
+			"port": 50003,
+			"modes": [
+				"aead_aes256_gcm_rtpsize",
+				"aead_aes256_gcm",
+				"xsalsa20_poly1305_lite_rtpsize",
+				"xsalsa20_poly1305_lite",
+				"xsalsa20_poly1305_suffix",
+				"xsalsa20_poly1305"
+			],
+			"ip": "109.200.213.251",
+			"experiments": [
+				"bwe_conservative_link_estimate",
+				"bwe_remote_locus_client",
+				"fixed_keyframe_interval"
+			];
+		};
+	*/
+
 	socket.send(JSON.stringify({
 		op: VoiceOPCodes.READY,
 		d: {
-			streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: 1311886, ssrc: 1311885, active: false, }))],
+			streams: [...data.d.streams.map(x => ({ ...x, rtx_ssrc: Math.floor(Math.random() * 10000), ssrc: Math.floor(Math.random() * 10000), active: false, }))],
-			ssrc: 1,
+			ssrc: Math.floor(Math.random() * 10000),
 			ip: transport.iceCandidates[0].ip,
-			port: transport.iceCandidates[0].port,
+			port: "50001",
 			modes: [
 				"aead_aes256_gcm_rtpsize",
 				"aead_aes256_gcm",
@@ -81,7 +107,6 @@ export async function onIdentify(this: Server, socket: WebSocket, data: Identify
 				"xsalsa20_poly1305_suffix",
 				"xsalsa20_poly1305"
 			],
-			heartbeat_interval: 1,
 			experiments: [],
 		},
 	}));

webrtc/src/opcodes/SelectProtocol.ts

@@ -87,42 +87,82 @@ export async function onSelectProtocol(this: Server, socket: WebSocket, data: Pa
 			})),
 	*/
 
+	const videoCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "video")?.mimeType
+	const audioCodec = this.mediasoupRouters[0].rtpCapabilities.codecs!.find((x: any) => x.kind === "audio")
+
 	if (!test_hasMadeProducer) {
 		const producer = await transport.produce({
 			kind: "audio",
 			rtpParameters: {
 				mid: "audio",
 				codecs: [{
-					clockRate: 48000,
+					clockRate: audioCodec!.clockRate,
-					payloadType: 111,
+					payloadType: audioCodec!.preferredPayloadType as number,
-					mimeType: "audio/opus",
+					mimeType: audioCodec!.mimeType,
-					channels: 2,
+					channels: audioCodec?.channels,
 				}],
 				headerExtensions: res.ext?.map(x => ({
 					id: x.value,
 					uri: x.uri,
-				}))
+				})),
 			},
 			paused: false,
 		});
 
 		const consumer = await transport.consume({
 			producerId: producer.id,
-			paused: false,
+			paused: true,
 			rtpCapabilities,
-		})
+		});
 		
 		test_hasMadeProducer = true;
 	}
 
+	/* server sends sdp:
+
+	m=audio 50021 ICE/SDP		//same port as sent in READY
+	a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
+	c=IN IP4 109.200.213.132	//same IP as sent in READY
+	a=rtcp:50021				//same port?
+	a=ice-ufrag:rTmX
+	a=ice-pwd:M+ncqWK6SEdHhirOjG2VFA
+	a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87
+	a=candidate:1 1 UDP 4261412862 109.200.213.132 50021 typ host	//same IP and PORT
+
+	*/
+
+
+	var test = {
+		"video_codec": "H264",
+		"sdp": `
+			m=audio 50011 ICE/SDP\n
+			a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
+			c=IN IP4 109.200.214.156\n
+			a=rtcp:50011\n
+			a=ice-ufrag:d0aZ\n
+			a=ice-pwd:51ubWYu7GSkQRqlH/apTSZ\n
+			a=fingerprint:sha-256 4A:79:94:16:44:3F:BD:05:41:5A:C7:20:F3:12:54:70:00:73:5D:33:00:2D:2C:80:9B:39:E1:9F:2D:A7:49:87\n
+			a=candidate:1 1 UDP 4261412862 109.200.214.156 50011 typ host\n`,
+		"media_session_id": "9e18c981687f2de5399edd5cb3f3babf",
+		"audio_codec": "opus"
+	};
+
+
 	socket.send(JSON.stringify({
 		op: VoiceOPCodes.SESSION_DESCRIPTION,
 		d: {
-			video_codec: data.d.codecs.find((x: any) => x.type === "video").name,
+			video_codec: videoCodec?.substring(6) || undefined,
-			secret_key: new Array(32).fill(null).map(x => Math.random() * 256),
+			// mode: "xsalsa20_poly1305",
-			mode: "xsalsa20_poly1305",
+			media_session_id: transport.id,
-			media_session_id: this.mediasoupTransports[0].id,
+			audio_codec: audioCodec?.mimeType.substring(6),
-			audio_codec: data.d.codecs.find((x: any) => x.type === "audio").name,
+			sdp: `m=audio ${transport.iceCandidates[0].port} ICE/SDP\n`
+				+ `a=fingerprint:sha-256 ${transport.dtlsParameters.fingerprints.find(x => x.algorithm === "sha-256")?.value}\n`
+				+ `c=IN IPV4 ${transport.iceCandidates[0].ip}\n`
+				+ `a=rtcp:${transport.iceCandidates[0].port}\n`
+				+ `a=ice-ufrag:${transport.iceParameters.usernameFragment}\n`
+				+ `a=ice-pwd:${transport.iceParameters.password}\n`
+				+ `a=fingerprint:sha-1 ${transport.dtlsParameters.fingerprints[0].value}\n`
+				+ `a=candidate:1 1 ${transport.iceCandidates[0].protocol} ${transport.iceCandidates[0].priority} ${transport.iceCandidates[0].ip} ${transport.iceCandidates[0].port} typ ${transport.iceCandidates[0].type}`
 		}
 	}));
 }

webrtc/src/opcodes/Version.ts

@@ -0,0 +1,14 @@
+import { WebSocket } from "@fosscord/gateway";
+import { Payload } from "./index";
+import { setHeartbeat } from "../util";
+import { Server } from "../Server"
+
+export async function onVersion(this: Server, socket: WebSocket, data: Payload) {
+	socket.send(JSON.stringify({
+		op: 16,
+		d: {
+			voice: "0.8.31",	//version numbers?
+			rtc_worker: "0.3.18",
+		}
+	}))
+}

webrtc/src/opcodes/index.ts

@@ -15,6 +15,8 @@ import { onSpeaking } from "./Speaking";
 import { onResume } from "./Resume";
 import { onConnect } from "./Connect";
 
+import { onVersion } from "./Version";
+
 export type OPCodeHandler = (this: WebSocket, data: Payload) => any;
 
 export default {
@@ -34,4 +36,5 @@ export default {
 	//op 13?
 	//op 15?
 	//op 16? empty data on client send but server sends {"voice":"0.8.24+bugfix.voice.streams.opt.branch-ffcefaff7","rtc_worker":"0.3.14-crypto-collision-copy"}
+	[VoiceOPCodes.VERSION]: onVersion,
 };

webrtc/src/start.ts

@@ -1,10 +1,10 @@
+//testing
+process.env.DATABASE = "../bundle/database.db";
+
 import { config } from "dotenv";
 config();
 
 import { Server } from "./Server";
 
-//testing
-process.env.DATABASE = "../bundle/database.db";
-
 const server = new Server();
 server.listen();

webrtc/src/util/Heartbeat.ts

@@ -1,18 +1,23 @@
 import { WebSocket, CLOSECODES } from "@fosscord/gateway";
 import { VoiceOPCodes } from "@fosscord/util";
 
-export async function setHeartbeat(socket: WebSocket) {
+export async function setHeartbeat(socket: WebSocket, nonce?: Number) {
 	if (socket.heartbeatTimeout) clearTimeout(socket.heartbeatTimeout);
 
 	socket.heartbeatTimeout = setTimeout(() => {
 		return socket.close(CLOSECODES.Session_timed_out);
 	}, 1000 * 45);
 
-	socket.send(JSON.stringify({
+	if (!nonce) {
-		op: VoiceOPCodes.HEARTBEAT_ACK,
+		socket.send(JSON.stringify({
-		d: {
+			op: VoiceOPCodes.HELLO,
-			v: 6,
+			d: {
-			heartbeat_interval: 13750,
+				v: 5,
-		}
+				heartbeat_interval: 13750,
-	}));
+			}
+		}));
+	}
+	else {
+		socket.send(JSON.stringify({ op: VoiceOPCodes.HEARTBEAT_ACK, d: nonce }));
+	}
 }