diff --git a/package.json b/package.json index acf8ca4d..5b1a91a4 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "husky": "^9.1.7", "prettier": "^3.5.3", "pretty-quick": "^4.1.1", + "spacebar-webrtc-types": "github:dank074/spacebar-webrtc-types", "typescript": "^5.8.3" }, "dependencies": { @@ -124,8 +125,6 @@ "optionalDependencies": { "@yukikaze-bot/erlpack": "^1.0.1", "jimp": "^1.6.0", - "@dank074/medooze-media-server": "1.156.4", - "semantic-sdp": "^3.31.1", "mysql": "^2.18.1", "nodemailer-mailgun-transport": "^2.1.5", "nodemailer-mailjet-transport": "github:n0script22/nodemailer-mailjet-transport", diff --git a/src/webrtc/Server.ts b/src/webrtc/Server.ts index 6127aef4..40c39658 100644 --- a/src/webrtc/Server.ts +++ b/src/webrtc/Server.ts @@ -21,7 +21,12 @@ import dotenv from "dotenv"; import http from "http"; import ws from "ws"; import { Connection } from "./events/Connection"; -import { mediaServer } from "./util/MediaServer"; +import { + mediaServer, + WRTC_PORT_MAX, + WRTC_PORT_MIN, + WRTC_PUBLIC_IP, +} from "./util/MediaServer"; import { green, yellow } from "picocolors"; dotenv.config(); @@ -77,7 +82,7 @@ export class Server { console.log(`[WebRTC] ${yellow("WEBRTC disabled")}`); return Promise.resolve(); } - await mediaServer.start(); + await mediaServer.start(WRTC_PUBLIC_IP, WRTC_PORT_MIN, WRTC_PORT_MAX); if (!this.server.listening) { this.server.listen(this.port); console.log(`[WebRTC] ${green(`online on 0.0.0.0:${this.port}`)}`); diff --git a/src/webrtc/medooze/MedoozeSignalingDelegate.ts b/src/webrtc/medooze/MedoozeSignalingDelegate.ts deleted file mode 100644 index 9e05fb45..00000000 --- a/src/webrtc/medooze/MedoozeSignalingDelegate.ts +++ /dev/null @@ -1,274 +0,0 @@ -import { CodecInfo, MediaInfo, SDPInfo } from "semantic-sdp"; -import { SignalingDelegate } from "../util/SignalingDelegate"; -import { Codec, WebRtcClient } from "../util/WebRtcClient"; -import { MediaServer, Endpoint } from "@dank074/medooze-media-server"; -import { VoiceRoom } from "./VoiceRoom"; -import { MedoozeWebRtcClient } from "./MedoozeWebRtcClient"; - -export class MedoozeSignalingDelegate implements SignalingDelegate { - private _rooms: Map = new Map(); - private _ip: string; - private _port: number; - private _endpoint: Endpoint; - - public start(): Promise { - MediaServer.enableLog(true); - - this._ip = process.env.PUBLIC_IP || "127.0.0.1"; - - try { - const range = process.env.WEBRTC_PORT_RANGE || "3690-3960"; - var ports = range.split("-"); - const min = Number(ports[0]); - const max = Number(ports[1]); - - MediaServer.setPortRange(min, max); - } catch (error) { - console.error( - "Invalid env var: WEBRTC_PORT_RANGE", - process.env.WEBRTC_PORT_RANGE, - error, - ); - process.exit(1); - } - - //MediaServer.setAffinity(2) - this._endpoint = MediaServer.createEndpoint(this._ip); - this._port = this._endpoint.getLocalPort(); - return Promise.resolve(); - } - - public join( - rtcServerId: string, - userId: string, - ws: any, - type: "guild-voice" | "dm-voice" | "stream", - ): WebRtcClient { - // make sure user isn't already in a room of the same type - // user can be in two simultanous rooms of different type though (can be in a voice channel and watching a stream for example) - const rooms = this.rooms - .values() - .filter((room) => - type === "stream" - ? room.type === "stream" - : room.type === "dm-voice" || room.type === "guild-voice", - ); - let existingClient; - - for (const room of rooms) { - let result = room.getClientById(userId); - if (result) { - existingClient = result; - break; - } - } - - if (existingClient) { - console.log("client already connected, disconnect.."); - this.onClientClose(existingClient); - } - - if (!this._rooms.has(rtcServerId)) { - console.debug("no channel created, creating one..."); - this.createRoom(rtcServerId, type); - } - - const room = this._rooms.get(rtcServerId)!; - - const client = new MedoozeWebRtcClient(userId, rtcServerId, ws, room); - - room?.onClientJoin(client); - - return client; - } - - public async onOffer( - client: WebRtcClient, - sdpOffer: string, - codecs: Codec[], - ): Promise { - const room = this._rooms.get(client.rtc_server_id); - - if (!room) { - console.error( - "error, client sent an offer but has not authenticated", - ); - Promise.reject(); - } - - const offer = SDPInfo.parse("m=audio\n" + sdpOffer); - - const rtpHeaders = new Map(offer.medias[0].extensions); - - const getIdForHeader = ( - rtpHeaders: Map, - headerUri: string, - ) => { - for (const [key, value] of rtpHeaders) { - if (value == headerUri) return key; - } - return -1; - }; - - const audioMedia = new MediaInfo("0", "audio"); - const audioCodec = new CodecInfo( - "opus", - codecs.find((val) => val.name == "opus")?.payload_type ?? 111, - ); - audioCodec.addParam("minptime", "10"); - audioCodec.addParam("usedtx", "1"); - audioCodec.addParam("useinbandfec", "1"); - audioCodec.setChannels(2); - audioMedia.addCodec(audioCodec); - - audioMedia.addExtension( - getIdForHeader( - rtpHeaders, - "urn:ietf:params:rtp-hdrext:ssrc-audio-level", - ), - "urn:ietf:params:rtp-hdrext:ssrc-audio-level", - ); - if (audioCodec.type === 111) - // if this is chromium, apply this header - audioMedia.addExtension( - getIdForHeader( - rtpHeaders, - "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", - ), - "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", - ); - - const videoMedia = new MediaInfo("1", "video"); - const videoCodec = new CodecInfo( - "H264", - codecs.find((val) => val.name == "H264")?.payload_type ?? 102, - ); - videoCodec.setRTX( - codecs.find((val) => val.name == "H264")?.rtx_payload_type ?? 103, - ); - videoCodec.addParam("level-asymmetry-allowed", "1"); - videoCodec.addParam("packetization-mode", "1"); - videoCodec.addParam("profile-level-id", "42e01f"); - videoCodec.addParam("x-google-max-bitrate", "2500"); - videoMedia.addCodec(videoCodec); - - videoMedia.addExtension( - getIdForHeader( - rtpHeaders, - "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time", - ), - "http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time", - ); - videoMedia.addExtension( - getIdForHeader(rtpHeaders, "urn:ietf:params:rtp-hdrext:toffset"), - "urn:ietf:params:rtp-hdrext:toffset", - ); - videoMedia.addExtension( - getIdForHeader( - rtpHeaders, - "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay", - ), - "http://www.webrtc.org/experiments/rtp-hdrext/playout-delay", - ); - videoMedia.addExtension( - getIdForHeader( - rtpHeaders, - "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", - ), - "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", - ); - - if (audioCodec.type === 111) - // if this is chromium, apply this header - videoMedia.addExtension( - getIdForHeader(rtpHeaders, "urn:3gpp:video-orientation"), - "urn:3gpp:video-orientation", - ); - - offer.medias = [audioMedia, videoMedia]; - - const transport = this._endpoint.createTransport(offer); - - transport.setRemoteProperties(offer); - - room?.onClientOffer(client, transport); - - const dtls = transport.getLocalDTLSInfo(); - const ice = transport.getLocalICEInfo(); - const fingerprint = dtls.getHash() + " " + dtls.getFingerprint(); - const candidates = transport.getLocalCandidates(); - const candidate = candidates[0]; - - const answer = - `m=audio ${this.port} ICE/SDP\n` + - `a=fingerprint:${fingerprint}\n` + - `c=IN IP4 ${this.ip}\n` + - `a=rtcp:${this.port}\n` + - `a=ice-ufrag:${ice.getUfrag()}\n` + - `a=ice-pwd:${ice.getPwd()}\n` + - `a=fingerprint:${fingerprint}\n` + - `a=candidate:1 1 ${candidate.getTransport()} ${candidate.getFoundation()} ${candidate.getAddress()} ${candidate.getPort()} typ host\n`; - - return Promise.resolve(answer); - } - - public onClientClose = (client: WebRtcClient) => { - this._rooms.get(client.rtc_server_id)?.onClientLeave(client); - }; - - public updateSDP(offer: string): void { - throw new Error("Method not implemented."); - } - - public createRoom( - rtcServerId: string, - type: "guild-voice" | "dm-voice" | "stream", - ): void { - this._rooms.set(rtcServerId, new VoiceRoom(rtcServerId, type, this)); - } - - public disposeRoom(rtcServerId: string): void { - const room = this._rooms.get(rtcServerId); - room?.dispose(); - this._rooms.delete(rtcServerId); - } - - get rooms(): Map { - return this._rooms; - } - - public getClientsForRtcServer(rtcServerId: string): Set> { - if (!this._rooms.has(rtcServerId)) { - return new Set(); - } - - return new Set(this._rooms.get(rtcServerId)?.clients.values())!; - } - - private getClientForUserId = ( - userId: string, - ): MedoozeWebRtcClient | undefined => { - for (const channel of this.rooms.values()) { - let result = channel.getClientById(userId); - if (result) { - return result; - } - } - return undefined; - }; - - get ip(): string { - return this._ip; - } - get port(): number { - return this._port; - } - - get endpoint(): Endpoint { - return this._endpoint; - } - - public stop(): Promise { - return Promise.resolve(); - } -} diff --git a/src/webrtc/medooze/MedoozeWebRtcClient.ts b/src/webrtc/medooze/MedoozeWebRtcClient.ts deleted file mode 100644 index c6defaa3..00000000 --- a/src/webrtc/medooze/MedoozeWebRtcClient.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { - IncomingStream, - OutgoingStream, - Transport, -} from "@dank074/medooze-media-server"; -import { SSRCs, WebRtcClient } from "webrtc/util"; -import { VoiceRoom } from "./VoiceRoom"; - -export class MedoozeWebRtcClient implements WebRtcClient { - websocket: any; - user_id: string; - rtc_server_id: string; - webrtcConnected: boolean; - public transport?: Transport; - public incomingStream?: IncomingStream; - public outgoingStream?: OutgoingStream; - public room?: VoiceRoom; - public isStopped?: boolean; - - constructor( - userId: string, - rtcServerId: string, - websocket: any, - room: VoiceRoom, - ) { - this.user_id = userId; - this.rtc_server_id = rtcServerId; - this.websocket = websocket; - this.room = room; - this.webrtcConnected = false; - this.isStopped = false; - } - - public isProducingAudio(): boolean { - if (!this.webrtcConnected) return false; - const audioTrack = this.incomingStream?.getTrack( - `audio-${this.user_id}`, - ); - - if (audioTrack) return true; - - return false; - } - - public isProducingVideo(): boolean { - if (!this.webrtcConnected) return false; - const videoTrack = this.incomingStream?.getTrack( - `video-${this.user_id}`, - ); - - if (videoTrack) return true; - - return false; - } - - public getIncomingStreamSSRCs(): SSRCs { - if (!this.webrtcConnected) - return { audio_ssrc: 0, video_ssrc: 0, rtx_ssrc: 0 }; - - const audioTrack = this.incomingStream?.getTrack( - `audio-${this.user_id}`, - ); - const audio_ssrc = - audioTrack?.getSSRCs()[audioTrack.getDefaultEncoding().id]; - const videoTrack = this.incomingStream?.getTrack( - `video-${this.user_id}`, - ); - const video_ssrc = - videoTrack?.getSSRCs()[videoTrack.getDefaultEncoding().id]; - - return { - audio_ssrc: audio_ssrc?.media ?? 0, - video_ssrc: video_ssrc?.media ?? 0, - rtx_ssrc: video_ssrc?.rtx ?? 0, - }; - } - - public getOutgoingStreamSSRCsForUser(user_id: string): SSRCs { - const outgoingStream = this.outgoingStream; - - const audioTrack = outgoingStream?.getTrack(`audio-${user_id}`); - const audio_ssrc = audioTrack?.getSSRCs(); - const videoTrack = outgoingStream?.getTrack(`video-${user_id}`); - const video_ssrc = videoTrack?.getSSRCs(); - - return { - audio_ssrc: audio_ssrc?.media ?? 0, - video_ssrc: video_ssrc?.media ?? 0, - rtx_ssrc: video_ssrc?.rtx ?? 0, - }; - } - - public publishTrack(type: "audio" | "video", ssrc: SSRCs) { - if (!this.transport) return; - - const id = `${type}-${this.user_id}`; - const existingTrack = this.incomingStream?.getTrack(id); - - if (existingTrack) { - console.error(`error: attempted to create duplicate track ${id}`); - return; - } - let ssrcs; - if (type === "audio") { - ssrcs = { media: ssrc.audio_ssrc! }; - } else { - ssrcs = { media: ssrc.video_ssrc!, rtx: ssrc.rtx_ssrc }; - } - const track = this.transport?.createIncomingStreamTrack( - type, - { id, ssrcs: ssrcs, media: type }, - this.incomingStream, - ); - - //this.channel?.onClientPublishTrack(this, track, ssrcs); - } - - public subscribeToTrack(user_id: string, type: "audio" | "video") { - if (!this.transport) return; - - const id = `${type}-${user_id}`; - - const otherClient = this.room?.getClientById(user_id); - const incomingStream = otherClient?.incomingStream; - const incomingTrack = incomingStream?.getTrack(id); - - if (!incomingTrack) { - console.error(`error subscribing, not track found ${id}`); - return; - } - - let ssrcs; - if (type === "audio") { - ssrcs = { - media: otherClient?.getIncomingStreamSSRCs().audio_ssrc!, - }; - } else { - ssrcs = { - media: otherClient?.getIncomingStreamSSRCs().video_ssrc!, - rtx: otherClient?.getIncomingStreamSSRCs().rtx_ssrc, - }; - } - - const outgoingTrack = this.transport?.createOutgoingStreamTrack( - incomingTrack.media, - { id, ssrcs, media: incomingTrack.media }, - this.outgoingStream, - ); - - outgoingTrack?.attachTo(incomingTrack); - } -} diff --git a/src/webrtc/medooze/VoiceRoom.ts b/src/webrtc/medooze/VoiceRoom.ts deleted file mode 100644 index 55eb6fea..00000000 --- a/src/webrtc/medooze/VoiceRoom.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { MedoozeSignalingDelegate } from "./MedoozeSignalingDelegate"; -import { - IncomingStreamTrack, - SSRCs, - Transport, -} from "@dank074/medooze-media-server"; -import { MedoozeWebRtcClient } from "./MedoozeWebRtcClient"; -import { StreamInfo } from "semantic-sdp"; - -export class VoiceRoom { - private _clients: Map; - private _id: string; - private _sfu: MedoozeSignalingDelegate; - private _type: "guild-voice" | "dm-voice" | "stream"; - - constructor( - id: string, - type: "guild-voice" | "dm-voice" | "stream", - sfu: MedoozeSignalingDelegate, - ) { - this._id = id; - this._type = type; - this._clients = new Map(); - this._sfu = sfu; - } - - onClientJoin = (client: MedoozeWebRtcClient) => { - // do shit here - this._clients.set(client.user_id, client); - }; - - onClientOffer = (client: MedoozeWebRtcClient, transport: Transport) => { - client.transport = transport; - - client.transport.on("dtlsstate", (state, self) => { - if (state === "connected") { - client.webrtcConnected = true; - console.log("connected"); - } - }); - - client.incomingStream = transport.createIncomingStream( - new StreamInfo(`in-${client.user_id}`), - ); - - client.outgoingStream = transport.createOutgoingStream( - new StreamInfo(`out-${client.user_id}`), - ); - - client.webrtcConnected = true; - - // subscribe to all current streams from this channel - // for(const otherClient of this._clients.values()) { - // const incomingStream = otherClient.incomingStream - - // if(!incomingStream) continue; - - // for(const track of (incomingStream.getTracks())) { - // client.subscribeToTrack(otherClient.user_id, track.media) - // } - // } - }; - - onClientLeave = (client: MedoozeWebRtcClient) => { - console.log("stopping client"); - this._clients.delete(client.user_id); - - // stop the client - if (!client.isStopped) { - client.isStopped = true; - - for (const otherClient of this.clients.values()) { - //remove outgoing track for this user - otherClient.outgoingStream - ?.getTrack(`audio-${client.user_id}`) - ?.stop(); - otherClient.outgoingStream - ?.getTrack(`video-${client.user_id}`) - ?.stop(); - } - - client.incomingStream?.stop(); - client.outgoingStream?.stop(); - - client.transport?.stop(); - client.room = undefined; - client.incomingStream = undefined; - client.outgoingStream = undefined; - client.transport = undefined; - client.websocket = undefined; - } - }; - - get clients(): Map { - return this._clients; - } - - getClientById = (id: string) => { - return this._clients.get(id); - }; - - get id(): string { - return this._id; - } - - get type(): "guild-voice" | "dm-voice" | "stream" { - return this._type; - } - - public dispose(): void { - const clients = this._clients.values(); - for (const client of clients) { - this.onClientLeave(client); - } - this._clients.clear(); - this._sfu = undefined!; - this._clients = undefined!; - } -} diff --git a/src/webrtc/opcodes/Video.ts b/src/webrtc/opcodes/Video.ts index 61be1e60..1f21be83 100644 --- a/src/webrtc/opcodes/Video.ts +++ b/src/webrtc/opcodes/Video.ts @@ -20,10 +20,10 @@ import { mediaServer, VoiceOPCodes, VoicePayload, - WebRtcClient, WebRtcWebSocket, Send, } from "@spacebar/webrtc"; +import type { WebRtcClient } from "spacebar-webrtc-types"; export async function onVideo(this: WebRtcWebSocket, payload: VoicePayload) { if (!this.webRtcClient || !this.webRtcClient.webrtcConnected) return; diff --git a/src/webrtc/util/MediaServer.ts b/src/webrtc/util/MediaServer.ts index befacbce..9842efb7 100644 --- a/src/webrtc/util/MediaServer.ts +++ b/src/webrtc/util/MediaServer.ts @@ -16,25 +16,49 @@ along with this program. If not, see . */ -//import { MedoozeSignalingDelegate } from "../medooze/MedoozeSignalingDelegate"; -import { SignalingDelegate } from "./SignalingDelegate"; +import type { SignalingDelegate } from "spacebar-webrtc-types"; import { green, red } from "picocolors"; export let mediaServer: SignalingDelegate; +export const WRTC_PUBLIC_IP = process.env.WRTC_PUBLIC_IP ?? "127.0.0.1"; +export const WRTC_PORT_MIN = process.env.WRTC_PORT_MIN + ? parseInt(process.env.WRTC_PORT_MIN) + : 2000; +export const WRTC_PORT_MAX = process.env.WRTC_PORT_MAX + ? parseInt(process.env.WRTC_PORT_MAX) + : 65000; + +const selectedWrtcLibrary = process.env.WRTC_LIBRARY; + +// could not find a way to hide stack trace from base Error object +class NoConfiguredLibraryError implements Error { + name: string; + message: string; + stack?: string | undefined; + cause?: unknown; + + constructor(message: string) { + this.name = "NoConfiguredLibraryError"; + this.message = message; + } +} + (async () => { try { - //mediaServer = require('../medooze/MedoozeSignalingDelegate'); - mediaServer = new ( - await import("../medooze/MedoozeSignalingDelegate") - ).MedoozeSignalingDelegate(); + //mediaServer = require('medooze-spacebar-wrtc'); + if (!selectedWrtcLibrary) + throw new NoConfiguredLibraryError("No library configured in .env"); + + mediaServer = new // @ts-ignore + (await import(selectedWrtcLibrary)).default(); console.log( - `[WebRTC] ${green("Succesfully loaded MedoozeSignalingDelegate")}`, + `[WebRTC] ${green(`Succesfully loaded ${selectedWrtcLibrary}`)}`, ); - } catch (e) { + } catch (error) { console.log( - `[WebRTC] ${red("Failed to import MedoozeSignalingDelegate")}`, + `[WebRTC] ${red(`Failed to import ${selectedWrtcLibrary}: ${error instanceof NoConfiguredLibraryError ? error.message : ""}`)}`, ); } })(); diff --git a/src/webrtc/util/SignalingDelegate.ts b/src/webrtc/util/SignalingDelegate.ts deleted file mode 100644 index e2f010d2..00000000 --- a/src/webrtc/util/SignalingDelegate.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { Codec, WebRtcClient } from "./WebRtcClient"; - -export interface SignalingDelegate { - start: () => Promise; - stop: () => Promise; - join( - rtcServerId: string, - userId: string, - ws: T, - type: "guild-voice" | "dm-voice" | "stream", - ): WebRtcClient; - onOffer( - client: WebRtcClient, - offer: string, - codecs: Codec[], - ): Promise; - onClientClose(client: WebRtcClient): void; - updateSDP(offer: string): void; - getClientsForRtcServer(rtcServerId: string): Set>; - get ip(): string; - get port(): number; -} diff --git a/src/webrtc/util/WebRtcClient.ts b/src/webrtc/util/WebRtcClient.ts deleted file mode 100644 index 3dcd0493..00000000 --- a/src/webrtc/util/WebRtcClient.ts +++ /dev/null @@ -1,31 +0,0 @@ -export interface WebRtcClient { - websocket: T; - user_id: string; - rtc_server_id: string; - webrtcConnected: boolean; - getIncomingStreamSSRCs: () => SSRCs; - getOutgoingStreamSSRCsForUser: (user_id: string) => SSRCs; - isProducingAudio: () => boolean; - isProducingVideo: () => boolean; - publishTrack: (type: "audio" | "video", ssrc: SSRCs) => void; - subscribeToTrack: (user_id: string, type: "audio" | "video") => void; -} - -export interface SSRCs { - audio_ssrc?: number; - video_ssrc?: number; - rtx_ssrc?: number; -} - -export interface RtpHeader { - uri: string; - id: number; -} - -export interface Codec { - name: "opus" | "VP8" | "VP9" | "H264"; - type: "audio" | "video"; - priority: number; - payload_type: number; - rtx_payload_type?: number; -} diff --git a/src/webrtc/util/WebRtcWebSocket.ts b/src/webrtc/util/WebRtcWebSocket.ts index 086e1247..5bb2da46 100644 --- a/src/webrtc/util/WebRtcWebSocket.ts +++ b/src/webrtc/util/WebRtcWebSocket.ts @@ -1,5 +1,5 @@ import { WebSocket } from "@spacebar/gateway"; -import { WebRtcClient } from "./WebRtcClient"; +import type { WebRtcClient } from "spacebar-webrtc-types"; export interface WebRtcWebSocket extends WebSocket { type: "guild-voice" | "dm-voice" | "stream"; diff --git a/src/webrtc/util/index.ts b/src/webrtc/util/index.ts index aa33ad2d..264f1ecd 100644 --- a/src/webrtc/util/index.ts +++ b/src/webrtc/util/index.ts @@ -18,6 +18,5 @@ export * from "./Constants"; export * from "./MediaServer"; -export * from "./WebRtcClient"; export * from "./WebRtcWebSocket"; export * from "./Send";