From 28dc43ab4a92cbd173cb988a04aede06d3a21f2f Mon Sep 17 00:00:00 2001 From: Alexander Zinchuk Date: Wed, 8 Feb 2023 00:47:26 +0100 Subject: [PATCH] [Dev] Calls: Move secret-sauce to repo (#2511) --- src/lib/secret-sauce/blacksilence.d.ts | 5 - src/lib/secret-sauce/blacksilence.ts | 18 + src/lib/secret-sauce/buildSdp.d.ts | 21 - src/lib/secret-sauce/buildSdp.ts | 182 +++++ src/lib/secret-sauce/colibriClass.d.ts | 36 - src/lib/secret-sauce/colibriClass.ts | 45 ++ src/lib/secret-sauce/index.d.ts | 5 - src/lib/secret-sauce/index.js | 2 - src/lib/secret-sauce/index.js.LICENSE.txt | 39 - src/lib/secret-sauce/index.ts | 14 + src/lib/secret-sauce/p2p.d.ts | 16 - src/lib/secret-sauce/p2p.ts | 478 ++++++++++++ src/lib/secret-sauce/p2pMessage.d.ts | 47 -- src/lib/secret-sauce/p2pMessage.ts | 58 ++ src/lib/secret-sauce/parseSdp.d.ts | 3 - src/lib/secret-sauce/parseSdp.ts | 140 ++++ src/lib/secret-sauce/secretsauce.d.ts | 19 - src/lib/secret-sauce/secretsauce.ts | 851 ++++++++++++++++++++++ src/lib/secret-sauce/types.d.ts | 126 ---- src/lib/secret-sauce/types.ts | 143 ++++ src/lib/secret-sauce/utils.d.ts | 10 - src/lib/secret-sauce/utils.ts | 47 ++ 22 files changed, 1976 insertions(+), 329 deletions(-) delete mode 100644 src/lib/secret-sauce/blacksilence.d.ts create mode 100644 src/lib/secret-sauce/blacksilence.ts delete mode 100644 src/lib/secret-sauce/buildSdp.d.ts create mode 100644 src/lib/secret-sauce/buildSdp.ts delete mode 100644 src/lib/secret-sauce/colibriClass.d.ts create mode 100644 src/lib/secret-sauce/colibriClass.ts delete mode 100644 src/lib/secret-sauce/index.d.ts delete mode 100644 src/lib/secret-sauce/index.js delete mode 100644 src/lib/secret-sauce/index.js.LICENSE.txt create mode 100644 src/lib/secret-sauce/index.ts delete mode 100644 src/lib/secret-sauce/p2p.d.ts create mode 100644 src/lib/secret-sauce/p2p.ts delete mode 100644 src/lib/secret-sauce/p2pMessage.d.ts create mode 100644 src/lib/secret-sauce/p2pMessage.ts delete mode 100644 src/lib/secret-sauce/parseSdp.d.ts create mode 100644 src/lib/secret-sauce/parseSdp.ts delete mode 100644 src/lib/secret-sauce/secretsauce.d.ts create mode 100644 src/lib/secret-sauce/secretsauce.ts delete mode 100644 src/lib/secret-sauce/types.d.ts create mode 100644 src/lib/secret-sauce/types.ts delete mode 100644 src/lib/secret-sauce/utils.d.ts create mode 100644 src/lib/secret-sauce/utils.ts diff --git a/src/lib/secret-sauce/blacksilence.d.ts b/src/lib/secret-sauce/blacksilence.d.ts deleted file mode 100644 index 828b76642..000000000 --- a/src/lib/secret-sauce/blacksilence.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -export declare const silence: (ctx: AudioContext) => MediaStream; -export declare const black: ({ width, height }?: { - width?: number | undefined; - height?: number | undefined; -}) => MediaStream; diff --git a/src/lib/secret-sauce/blacksilence.ts b/src/lib/secret-sauce/blacksilence.ts new file mode 100644 index 000000000..244c4ad0a --- /dev/null +++ b/src/lib/secret-sauce/blacksilence.ts @@ -0,0 +1,18 @@ +// https://blog.mozilla.org/webrtc/warm-up-with-replacetrack/ + +export const silence = (ctx: AudioContext) => { + const oscillator = ctx.createOscillator(); + const dst = oscillator.connect(ctx.createMediaStreamDestination()); + oscillator.start(); + return new MediaStream([Object.assign((dst as any).stream.getAudioTracks()[0], { enabled: false })]); +}; + +export const black = ({ width = 640, height = 480 } = {}) => { + const canvas = Object.assign(document.createElement('canvas'), { width, height }); + const ctx = canvas.getContext('2d'); + if (!ctx) throw Error('Cannot create canvas ctx'); + + ctx.fillRect(0, 0, width, height); + const stream = canvas.captureStream(); + return new MediaStream([Object.assign(stream.getVideoTracks()[0], { enabled: false })]); +}; diff --git a/src/lib/secret-sauce/buildSdp.d.ts b/src/lib/secret-sauce/buildSdp.d.ts deleted file mode 100644 index f0f674a9a..000000000 --- a/src/lib/secret-sauce/buildSdp.d.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { GroupCallTransport, PayloadType, RTPExtension, SsrcGroup } from './types'; -export declare type Conference = { - sessionId: number; - audioExtensions: RTPExtension[]; - videoExtensions: RTPExtension[]; - audioPayloadTypes: PayloadType[]; - videoPayloadTypes: PayloadType[]; - ssrcs: Ssrc[]; - transport: GroupCallTransport; -}; -export declare type Ssrc = { - userId: string; - endpoint: string; - isMain: boolean; - isRemoved?: boolean; - isVideo: boolean; - isPresentation?: boolean; - sourceGroups: SsrcGroup[]; -}; -declare const _default: (conference: Conference, isAnswer?: boolean, isPresentation?: boolean, isP2p?: boolean) => string; -export default _default; diff --git a/src/lib/secret-sauce/buildSdp.ts b/src/lib/secret-sauce/buildSdp.ts new file mode 100644 index 000000000..976bc1b76 --- /dev/null +++ b/src/lib/secret-sauce/buildSdp.ts @@ -0,0 +1,182 @@ +import type { + Candidate, GroupCallTransport, PayloadType, RTPExtension, SsrcGroup, +} from './types'; +import { fromTelegramSource } from './utils'; + +export type Conference = { + sessionId: number; + audioExtensions: RTPExtension[]; + videoExtensions: RTPExtension[]; + audioPayloadTypes: PayloadType[]; + videoPayloadTypes: PayloadType[]; + ssrcs: Ssrc[]; + transport: GroupCallTransport; +}; + +export type Ssrc = { + userId: string; + endpoint: string; + isMain: boolean; + isRemoved?: boolean; + isVideo: boolean; + isPresentation?: boolean; + sourceGroups: SsrcGroup[]; +}; + +export default (conference: Conference, isAnswer = false, isPresentation = false, isP2p = false) => { + const lines: string[] = []; + + const add = (value: string) => { + lines.push(value); + }; + + const { + sessionId, + ssrcs, + audioExtensions, + videoExtensions, + audioPayloadTypes, + videoPayloadTypes, + transport: { + ufrag, + pwd, + fingerprints, + candidates, + }, + } = conference; + + // Header + add('v=0'); // version + add(`o=- ${sessionId} 2 IN IP4 0.0.0.0`); // sessionId, 2=sessionVersion + add('s=-'); // name of the session + add('t=0 0'); // time when session is valid + add('a=ice-options:trickle'); + add('a=msid-semantic:WMS *'); + add(`a=group:BUNDLE ${ssrcs.map((ssrc) => ssrc.endpoint).join(' ')}${isPresentation ? '' : ` ${isP2p ? '3' : '2'}`}`); + // ice-lite: is a minimal version of the ICE specification, intended only for servers running on a public IP address + if (!isP2p) add('a=ice-lite'); + + const addCandidate = (c: Candidate) => { + if (c.sdpString) { + add(`a=${c.sdpString}`); + } else { + let str = ''; + str += 'a=candidate:'; + str += `${c.foundation} ${c.component} ${c.protocol} ${c.priority} ${c.ip} ${c.port} typ ${c.type}`; + if ('rel-addr' in c) { + str += ` raddr ${c['rel-addr']} rport ${c['rel-port']}`; + } + str += ` generation ${c.generation}`; + add(str); + } + }; + + const addTransport = () => { + add(`a=ice-ufrag:${ufrag}`); + add(`a=ice-pwd:${pwd}`); + fingerprints.forEach((fingerprint) => { + add(`a=fingerprint:${fingerprint.hash} ${fingerprint.fingerprint}`); + add(`a=setup:${isP2p ? (fingerprint.setup) : 'passive'}`); + }); + + candidates.forEach(addCandidate); + }; + + const addPayloadType = (payloadType: PayloadType) => { + const { + channels, id, name, clockrate, parameters, + } = payloadType; + + const channelsString = channels ? `/${channels}` : ''; + add(`a=rtpmap:${id} ${name}/${clockrate}${channelsString}`); + + if (parameters) { + const parametersString = Object.keys(parameters).map((key) => { + return `${key}=${parameters![key]};`; + }).join(' '); + + add(`a=fmtp:${id} ${parametersString}`); + } + + payloadType['rtcp-fbs']?.forEach((fbParam) => { + add(`a=rtcp-fb:${id} ${fbParam.type}${fbParam.subtype ? ` ${fbParam.subtype}` : ''}`); + }); + }; + + const addSsrcEntry = (entry: Ssrc) => { + const payloadTypes = entry.isVideo ? videoPayloadTypes : audioPayloadTypes; + + const type = entry.isVideo ? 'video' : 'audio'; + add(`m=${type} ${entry.isMain ? 1 : 0} RTP/SAVPF ${payloadTypes.map((l) => l.id).join(' ')}`); + add('c=IN IP4 0.0.0.0'); + add('b=AS:1300'); // 1300000 / 1000 + add(`a=mid:${entry.endpoint}`); + add('a=rtcp-mux'); + payloadTypes.forEach(addPayloadType); + + add('a=rtcp:1 IN IP4 0.0.0.0'); + if (entry.isVideo) { + add('a=rtcp-rsize'); + } + + (entry.isVideo ? videoExtensions : audioExtensions).forEach(({ id, uri }) => { + add(`a=extmap:${id} ${uri}`); + }); + + if (entry.isRemoved) { + add('a=inactive'); + return; + } + + addTransport(); + + if (isP2p) { + add('a=sendrecv'); + add('a=bundle-only'); + } else { + if (isAnswer) { + add('a=recvonly'); + return; + } + if (entry.isMain) { + add('a=sendrecv'); + } else { + add('a=sendonly'); + add('a=bundle-only'); + } + } + + entry.sourceGroups.forEach((sourceGroup) => { + add(`a=ssrc-group:${sourceGroup.semantics} ${sourceGroup.sources.map(fromTelegramSource).join(' ')}`); + sourceGroup.sources.forEach((ssrcTelegram) => { + const ssrc = fromTelegramSource(ssrcTelegram); + add(`a=ssrc:${ssrc} cname:${entry.endpoint}`); + add(`a=ssrc:${ssrc} msid:${entry.endpoint} ${entry.endpoint}`); + add(`a=ssrc:${ssrc} mslabel:${entry.endpoint}`); + add(`a=ssrc:${ssrc} label:${entry.endpoint}`); + }); + }); + }; + + if (!isP2p) { + ssrcs.filter((ssrc) => ssrc.endpoint === '0' || ssrc.endpoint === '1').map(addSsrcEntry); + } else { + ssrcs.filter(addSsrcEntry); + } + + if (!isPresentation) { + add('m=application 1 UDP/DTLS/SCTP webrtc-datachannel'); + add('c=IN IP4 0.0.0.0'); + addTransport(); + add('a=ice-options:trickle'); + add(`a=mid:${isP2p ? '3' : (isPresentation ? '1' : '2')}`); + add('a=sctp-port:5000'); + add('a=max-message-size:262144'); + } + + if (!isP2p) { + ssrcs.filter((ssrc) => ssrc.endpoint !== '0' && ssrc.endpoint !== '1').map(addSsrcEntry); + } + + return `${lines.join('\n')}\n`; +}; diff --git a/src/lib/secret-sauce/colibriClass.d.ts b/src/lib/secret-sauce/colibriClass.d.ts deleted file mode 100644 index 80bcc3aaa..000000000 --- a/src/lib/secret-sauce/colibriClass.d.ts +++ /dev/null @@ -1,36 +0,0 @@ -export declare type EndpointConnectivityStatusChangeEvent = { - colibriClass: 'EndpointConnectivityStatusChangeEvent'; - endpoint: string; - active: boolean; -}; -export declare type DominantSpeakerEndpointChangeEvent = { - colibriClass: 'DominantSpeakerEndpointChangeEvent'; - dominantSpeakerEndpoint: string; - previousSpeakers: string[]; -}; -export declare type SenderVideoConstraints = { - colibriClass: 'SenderVideoConstraints'; - videoConstraints: { - idealHeight: number; - }; -}; -export declare type DebugMessage = { - colibriClass: 'DebugMessage'; - message: string; -}; -export declare type LastNEndpointsChangeEvent = { - colibriClass: 'LastNEndpointsChangeEvent'; - lastNEndpoints: string[]; -}; -export declare type ReceiverVideoConstraints = { - colibriClass: 'ReceiverVideoConstraints'; - defaultConstraints: { - maxHeight: number; - }; - constraints: Record; - onStageEndpoints: string[]; -}; -export declare type ColibriClass = (LastNEndpointsChangeEvent | DebugMessage | EndpointConnectivityStatusChangeEvent | SenderVideoConstraints | DominantSpeakerEndpointChangeEvent | ReceiverVideoConstraints); diff --git a/src/lib/secret-sauce/colibriClass.ts b/src/lib/secret-sauce/colibriClass.ts new file mode 100644 index 000000000..d7f06d2ca --- /dev/null +++ b/src/lib/secret-sauce/colibriClass.ts @@ -0,0 +1,45 @@ +export type EndpointConnectivityStatusChangeEvent = { + colibriClass: 'EndpointConnectivityStatusChangeEvent'; + endpoint: string; + active: boolean; +}; + +export type DominantSpeakerEndpointChangeEvent = { + colibriClass: 'DominantSpeakerEndpointChangeEvent'; + dominantSpeakerEndpoint: string; + previousSpeakers: string[]; +}; + +export type SenderVideoConstraints = { + colibriClass: 'SenderVideoConstraints'; + videoConstraints: { + idealHeight: number; + }; +}; + +export type DebugMessage = { + colibriClass: 'DebugMessage'; + message: string; +}; + +export type LastNEndpointsChangeEvent = { + colibriClass: 'LastNEndpointsChangeEvent'; + lastNEndpoints: string[]; +}; + +export type ReceiverVideoConstraints = { + colibriClass: 'ReceiverVideoConstraints'; + defaultConstraints: { + maxHeight: number; + }; + constraints: Record; + onStageEndpoints: string[]; +}; + +export type ColibriClass = ( + LastNEndpointsChangeEvent | DebugMessage | EndpointConnectivityStatusChangeEvent | + SenderVideoConstraints | DominantSpeakerEndpointChangeEvent | ReceiverVideoConstraints +); diff --git a/src/lib/secret-sauce/index.d.ts b/src/lib/secret-sauce/index.d.ts deleted file mode 100644 index 098fe6c24..000000000 --- a/src/lib/secret-sauce/index.d.ts +++ /dev/null @@ -1,5 +0,0 @@ -export { handleUpdateGroupCallConnection, startSharingScreen, joinGroupCall, getDevices, getUserStreams, setVolume, isStreamEnabled, toggleStream, leaveGroupCall, handleUpdateGroupCallParticipants, switchCameraInput, toggleSpeaker, toggleNoiseSuppression, } from './secretsauce'; -export { joinPhoneCall, processSignalingMessage, getStreams, toggleStreamP2p, stopPhoneCall, switchCameraInputP2p, } from './p2p'; -export * from './p2pMessage'; -export { IS_SCREENSHARE_SUPPORTED, THRESHOLD, } from './utils'; -export * from './types'; diff --git a/src/lib/secret-sauce/index.js b/src/lib/secret-sauce/index.js deleted file mode 100644 index 42361cd8f..000000000 --- a/src/lib/secret-sauce/index.js +++ /dev/null @@ -1,2 +0,0 @@ -/*! For license information please see index.js.LICENSE.txt */ -(()=>{"use strict";var e={"./src/blacksilence.ts":(e,t,a)=>{a.r(t),a.d(t,{silence:()=>n,black:()=>s});const n=e=>{const t=e.createOscillator(),a=t.connect(e.createMediaStreamDestination());return t.start(),new MediaStream([Object.assign(a.stream.getAudioTracks()[0],{enabled:!1})])},s=({width:e=640,height:t=480}={})=>{const a=Object.assign(document.createElement("canvas"),{width:e,height:t}),n=a.getContext("2d");if(!n)throw Error("Cannot create canvas ctx");n.fillRect(0,0,e,t);const s=a.captureStream();return new MediaStream([Object.assign(s.getVideoTracks()[0],{enabled:!1})])}},"./src/buildSdp.ts":(e,t,a)=>{a.r(t),a.d(t,{default:()=>s});var n=a("./src/utils.ts");const s=(e,t=!1,a=!1,s=!1)=>{const i=[],r=e=>{i.push(e)},{sessionId:o,ssrcs:c,audioExtensions:d,videoExtensions:p,audioPayloadTypes:u,videoPayloadTypes:l,transport:{ufrag:m,pwd:f,fingerprints:g,candidates:v}}=e;r("v=0"),r(`o=- ${o} 2 IN IP4 0.0.0.0`),r("s=-"),r("t=0 0"),r("a=ice-options:trickle"),r("a=msid-semantic:WMS *"),r(`a=group:BUNDLE ${c.map((e=>e.endpoint)).join(" ")}${a?"":" "+(s?"3":"2")}`),s||r("a=ice-lite");const S=e=>{if(e.sdpString)r(`a=${e.sdpString}`);else{let t="";t+="a=candidate:",t+=`${e.foundation} ${e.component} ${e.protocol} ${e.priority} ${e.ip} ${e.port} typ ${e.type}`,"rel-addr"in e&&(t+=` raddr ${e["rel-addr"]} rport ${e["rel-port"]}`),t+=` generation ${e.generation}`,r(t)}},y=()=>{r(`a=ice-ufrag:${m}`),r(`a=ice-pwd:${f}`),g.forEach((e=>{r(`a=fingerprint:${e.hash} ${e.fingerprint}`),r(`a=setup:${s?e.setup:"passive"}`)})),v.forEach(S)},h=e=>{const{channels:t,id:a,name:n,clockrate:s,parameters:i}=e;var o=t?`/${t}`:"";r(`a=rtpmap:${a} ${n}/${s}${o}`),i&&(o=Object.keys(i).map((e=>`${e}=${i[e]};`)).join(" "),r(`a=fmtp:${a} ${o}`)),e["rtcp-fbs"]?.forEach((e=>{r(`a=rtcp-fb:${a} ${e.type}${e.subtype?` ${e.subtype}`:""}`)}))};return e=e=>{const a=e.isVideo?l:u;var i=e.isVideo?"video":"audio";if(r(`m=${i} ${e.isMain?1:0} RTP/SAVPF ${a.map((e=>e.id)).join(" ")}`),r("c=IN IP4 0.0.0.0"),r("b=AS:1300"),r(`a=mid:${e.endpoint}`),r("a=rtcp-mux"),a.forEach(h),r("a=rtcp:1 IN IP4 0.0.0.0"),e.isVideo&&r("a=rtcp-rsize"),(e.isVideo?p:d).forEach((({id:e,uri:t})=>{r(`a=extmap:${e} ${t}`)})),e.isRemoved)r("a=inactive");else{if(y(),s)r("a=sendrecv"),r("a=bundle-only");else{if(t)return void r("a=recvonly");e.isMain?r("a=sendrecv"):(r("a=sendonly"),r("a=bundle-only"))}e.sourceGroups.forEach((t=>{r(`a=ssrc-group:${t.semantics} ${t.sources.map(n.fromTelegramSource).join(" ")}`),t.sources.forEach((t=>{t=(0,n.fromTelegramSource)(t),r(`a=ssrc:${t} cname:${e.endpoint}`),r(`a=ssrc:${t} msid:${e.endpoint} ${e.endpoint}`),r(`a=ssrc:${t} mslabel:${e.endpoint}`),r(`a=ssrc:${t} label:${e.endpoint}`)}))}))}},s?c.filter(e):c.filter((e=>"0"===e.endpoint||"1"===e.endpoint)).map(e),a||(r("m=application 1 UDP/DTLS/SCTP webrtc-datachannel"),r("c=IN IP4 0.0.0.0"),y(),r("a=ice-options:trickle"),r("a=mid:"+(s?"3":a?"1":"2")),r("a=sctp-port:5000"),r("a=max-message-size:262144")),s||c.filter((e=>"0"!==e.endpoint&&"1"!==e.endpoint)).map(e),`${i.join("\n")}\n`}},"./src/p2p.ts":(e,t,a)=>{a.r(t),a.d(t,{getStreams:()=>function(){return o?.streams},switchCameraInputP2p:()=>async function(){if(o&&o.facingMode){const e=o.streams.ownVideo;if(e){const t=e.getTracks()[0];if(t){const e=o.connection.getSenders().find((e=>t.id===e.track?.id));if(e){o.facingMode="environment"===o.facingMode?"user":"environment";try{const t=await d("video",o.facingMode);await e.replaceTrack(t.getTracks()[0]),o.streams.ownVideo=t,c()}catch(e){}}}}}},toggleStreamP2p:()=>p,joinPhoneCall:()=>async function(e,t,a,i,r){const d=new RTCPeerConnection({iceServers:e.map((e=>({urls:[e.isTurn&&`turn:${e.ip}:${e.port}`,e.isStun&&`stun:${e.ip}:${e.port}`].filter(Boolean),username:e.username,credentialType:"password",credential:e.password}))),iceCandidatePoolSize:2}),u=(0,n.silence)(new AudioContext),l=(0,n.black)({width:640,height:480}),g=(0,n.black)({width:640,height:480});d.addTrack(u.getTracks()[0],u),d.addTrack(l.getTracks()[0],l),d.addTrack(g.getTracks()[0],g),d.onicecandidate=e=>{e.candidate&&t({"@type":"Candidates",candidates:[{sdpString:e.candidate.candidate}]})},d.onconnectionstatechange=()=>{r({"@type":"updatePhoneCallConnectionState",connectionState:d.connectionState})},d.ontrack=e=>{var t;o&&(t=e.streams[0],"audio"===e.track.kind?(o.audio.srcObject=t,o.audio.play().catch(),o.streams.audio=t):"1"===e.transceiver.mid?o.streams.video=t:o.streams.presentation=t,c())};const v=d.createDataChannel("data",{id:0,negotiated:!0});v.onmessage=e=>{f(JSON.parse(e.data))},e=new Audio,o={audio:e,connection:d,emitSignalingData:t,isOutgoing:a,candidates:[],onUpdate:r,streams:{ownVideo:l,ownAudio:u,ownPresentation:g},mediaState:{isBatteryLow:!1,screencastState:"inactive",videoState:"inactive",videoRotation:0,isMuted:!0},blackVideo:l,blackPresentation:g,silence:u,dataChannel:v};try{i&&p("video",!0),p("audio",!0)}catch(e){}a&&(a=await d.createOffer({offerToReceiveAudio:!0,offerToReceiveVideo:!0}),await d.setLocalDescription(a),m((0,s.default)(a,!0)))},stopPhoneCall:()=>function(){o&&(o.streams.ownVideo?.getTracks().forEach((e=>e.stop())),o.streams.ownPresentation?.getTracks().forEach((e=>e.stop())),o.streams.ownAudio?.getTracks().forEach((e=>e.stop())),o.dataChannel.close(),o.connection.close(),o=void 0)},processSignalingMessage:()=>f});var n=a("./src/blacksilence.ts"),s=a("./src/parseSdp.ts"),i=a("./src/utils.ts"),r=a("./src/buildSdp.ts");let o;function c(){o?.onUpdate({...o.mediaState,"@type":"updatePhoneCallMediaState"})}function d(e,t="user"){return"presentation"===e?navigator.mediaDevices.getDisplayMedia({audio:!1,video:!0}):navigator.mediaDevices.getUserMedia({audio:"audio"===e&&{...i.IS_ECHO_CANCELLATION_SUPPORTED&&{echoCancellation:!0},...i.IS_NOISE_SUPPRESSION_SUPPORTED&&{noiseSuppression:!0}},video:"video"===e&&{facingMode:t}})}async function p(e,t){if(o){const a="audio"===e?o.streams.ownAudio:"video"===e?o.streams.ownVideo:o.streams.ownPresentation;if(a){const n=a.getTracks()[0];if(n){const a=o.connection.getSenders().find((e=>n.id===e.track?.id));if(a){t=void 0===t?!n.enabled:t;try{if(t&&!n.enabled){const t=await d(e);t.getTracks()[0].onended=()=>{p(e,!1)},await a.replaceTrack(t.getTracks()[0]),"audio"===e?o.streams.ownAudio=t:"video"===e?(o.streams.ownVideo=t,o.facingMode="user"):o.streams.ownPresentation=t,"video"!==e&&"presentation"!==e||p("video"===e?"presentation":"video",!1)}else if(!t&&n.enabled){n.stop();const t="audio"===e?o.silence:"video"===e?o.blackVideo:o.blackPresentation;if(!t)return;await a.replaceTrack(t.getTracks()[0]),"audio"===e?o.streams.ownAudio=t:"video"===e?o.streams.ownVideo=t:o.streams.ownPresentation=t}c(),u()}catch(e){}}}}}}function u(){if(o){const{emitSignalingData:e,streams:t}=o;e({"@type":"MediaState",videoRotation:0,isMuted:!t.ownAudio?.getTracks()[0].enabled,isBatteryLow:!0,videoState:t.ownVideo?.getTracks()[0].enabled?"active":"inactive",screencastState:t.ownPresentation?.getTracks()[0].enabled?"active":"inactive"})}}function l(e){if(!o||o.isOutgoing)return e;const t=e.payloadTypes;var a=t.findIndex((e=>"VP8"===e.name));const n=t[a];var s=t.findIndex((e=>Number(e.parameters?.apt)===n.id));return e.payloadTypes=[t[a],t[s]],e}function m(e){if(o){const t=o.emitSignalingData;e.ssrc&&e["ssrc-groups"]&&e["ssrc-groups"][0]&&e["ssrc-groups"][1]&&t({"@type":"InitialSetup",fingerprints:e.fingerprints,ufrag:e.ufrag,pwd:e.pwd,audio:{ssrc:(0,i.fromTelegramSource)(e.ssrc).toString(),ssrcGroups:[],payloadTypes:e.audioPayloadTypes,rtpExtensions:e.audioExtmap},video:l({ssrc:(0,i.fromTelegramSource)(e["ssrc-groups"][0].sources[0]).toString(),ssrcGroups:[{semantics:e["ssrc-groups"][0].semantics,ssrcs:e["ssrc-groups"][0].sources.map(i.fromTelegramSource)}],payloadTypes:e.videoPayloadTypes,rtpExtensions:e.videoExtmap}),screencast:l({ssrc:(0,i.fromTelegramSource)(e["ssrc-groups"][1].sources[0]).toString(),ssrcGroups:[{semantics:e["ssrc-groups"][1].semantics,ssrcs:e["ssrc-groups"][1].sources.map(i.fromTelegramSource)}],payloadTypes:e.screencastPayloadTypes,rtpExtensions:e.screencastExtmap})})}}async function f(e){if(o&&o.connection)switch(e["@type"]){case"MediaState":o.mediaState=e,c(),u();break;case"Candidates":var{candidates:t,gotInitialSetup:a}=o;if(!t)return;e.candidates.forEach((e=>{o.candidates.push(e.sdpString)})),a&&await Promise.all(o.candidates.map((e=>o.connection.addIceCandidate({candidate:e,sdpMLineIndex:0}))));break;case"InitialSetup":{const{connection:t,isOutgoing:n}=o;if(!t)return;if(a={transport:{candidates:[],ufrag:e.ufrag,pwd:e.pwd,fingerprints:e.fingerprints,"rtcp-mux":!1,xmlns:""},sessionId:Date.now(),ssrcs:[e.audio&&{isVideo:!1,isMain:!1,userId:"123",endpoint:"0",sourceGroups:[{semantics:"FID",sources:[e.audio.ssrc]}]},e.video&&{isVideo:!0,isPresentation:!1,isMain:!1,userId:"123",endpoint:"1",sourceGroups:e.video.ssrcGroups.map((e=>({semantics:e.semantics,sources:e.ssrcs})))},e.screencast&&{isVideo:!0,isPresentation:!0,isMain:!1,userId:"123",endpoint:"2",sourceGroups:e.screencast.ssrcGroups.map((e=>({semantics:e.semantics,sources:e.ssrcs})))}],audioPayloadTypes:e.audio.payloadTypes?.map(i.p2pPayloadTypeToConference)||[],audioExtensions:e.audio.rtpExtensions,videoPayloadTypes:l(e.video).payloadTypes?.map(i.p2pPayloadTypeToConference)||[],videoExtensions:e.video.rtpExtensions},await t.setRemoteDescription({sdp:(0,r.default)(a,n,void 0,!0),type:n?"answer":"offer"}),o.conference=a,!n){if(a=await t.createAnswer(),!a)return;await t.setLocalDescription(a),m((0,s.default)(a,!0))}o.gotInitialSetup=!0,await Promise.all(o.candidates.map((e=>t.addIceCandidate({candidate:e,sdpMLineIndex:0}))));break}}}},"./src/p2pMessage.ts":(e,t,a)=>{a.r(t)},"./src/parseSdp.ts":(e,t,a)=>{a.r(t),a.d(t,{default:()=>s});var n=a("./src/utils.ts");const s=(e,t=!1)=>{if(!e||!e.sdp)throw Error("Failed parsing SDP: session description is null");const a=e.sdp.split("\r\nm=").map(((e,t)=>0===t?e:`m=${e}`)).reduce(((e,t)=>{var a=t.match(/^m=(.+?)\s/)?.[1]||"header";return e[e.hasOwnProperty(a)&&"video"===a?"screencast":a]=t.split("\r\n").filter(Boolean),e}),{});var s=(e,t)=>t?a[t]?.find((t=>t.startsWith(e)))?.substr(e.length):Object.values(a).map((t=>t.find((t=>t.startsWith(e)))?.substr(e.length))).filter(Boolean)[0],i=e=>a[e].filter((e=>e.startsWith("a=extmap"))).map((e=>{var[,t,e]=e.match(/extmap:(\d+)(?:\/.+)?\s(.+)/);return{id:Number(t),uri:e}})),r=e=>{const t=a[e].filter((e=>e.startsWith("a=rtpmap"))).map((e=>{const[,t,a]=e.match(/:(\d+)\s(.+)/)||[];var[n,s,e]=a.split("/");return{id:Number(t),name:n,clockrate:Number(s),...e&&{channels:Number(e)}}})),n=a[e].filter((e=>e.startsWith("a=rtcp-fb"))).map((e=>{const[,t,a]=e.match(/:(\d+)\s(.+)/)||[];var[n,e]=a.split(" ");return{id:Number(t),type:n,subtype:e||""}})),s=a[e].filter((e=>e.startsWith("a=fmtp"))).map((e=>{const[,t,a]=e.match(/:(\d+)\s(.+)/)||[];if(e=a.split(";").reduce(((e,t)=>{var[a,t]=t.split("=");return e[a]=t,e}),{}),!Object.values(e).some((e=>!e)))return{id:Number(t),data:e}})).filter(Boolean);return t.map((e=>{var t=s.filter((t=>t.id===e.id)).map((e=>e.data)).reduce(((e,t)=>Object.assign(e,t)),{}),a=n.filter((t=>t.id===e.id)).map((e=>({type:e.type,subtype:e.subtype})));return{...e,...0{a.r(t),a.d(t,{getDevices:()=>async function(e,t=!0){return(await navigator.mediaDevices.enumerateDevices()).filter((a=>a.kind===`${e}${t?"input":"output"}`))},toggleSpeaker:()=>function(){o&&(o.isSpeakerDisabled=!o.isSpeakerDisabled,o?.onUpdate?.({"@type":"updateGroupCallConnectionState",connectionState:"connected",isSpeakerDisabled:o.isSpeakerDisabled}),o.participantFunctions&&Object.values(o.participantFunctions).forEach((e=>{e.toggleMute?.(!!o?.isSpeakerDisabled)})))},toggleNoiseSuppression:()=>function(){if(o&&o.myId&&o.streams){const a=o.streams[o.myId].audio;if(a){const n=a.getTracks()[0];var e,t;n&&(({echoCancellation:e,noiseSuppression:t}=n.getConstraints()),n.applyConstraints({echoCancellation:!e,noiseSuppression:!t}))}}},getUserStreams:()=>d,setVolume:()=>function(e,t){const a=o?.participantFunctions?.[e];a&&a.setVolume?.(t)},isStreamEnabled:()=>p,switchCameraInput:()=>async function(){if(o?.myId&&o.connection&&o.streams&&o.facingMode){const e=d(o.myId)?.video;if(e){const t=e.getTracks()[0];if(t){const e=o.connection.getSenders().find((e=>t.id===e.track?.id));if(e){o.facingMode="environment"===o.facingMode?"user":"environment";try{const t=await l("video",o.facingMode);await e.replaceTrack(t.getTracks()[0]),o.streams[o.myId].video=t}catch(e){}}}}}},toggleStream:()=>m,leaveGroupCall:()=>g,handleUpdateGroupCallParticipants:()=>async function(e){if(o){const{participants:n,conference:i,connection:r,myId:c}=o;if(n&&i&&r&&i.ssrcs&&i.transport&&c)if(e.find((e=>e.isSelf&&e.source!==o?.conference?.ssrcs?.find((e=>e.isMain&&!e.isVideo))?.sourceGroups[0].sources[0])))g();else{const n=[];if(e.forEach((e=>{if(e.isSelf)e.isMuted&&!e.canSelfUnmute&&(m("audio",!1),m("video",!1),m("presentation",!1));else{var t=e.isLeft;const a=e.isMuted||e.isMutedByMe,s=!e.isVideoJoined||!e.video||t,r=!e.presentation||t;let o=!1,c=!1,d=!1;i.ssrcs.filter((t=>t.userId===e.id)).forEach((t=>{t.isVideo||(t.sourceGroups[0].sources[0]===e.source&&(c=!0),t.isRemoved=a),t.isVideo&&(t.isPresentation||(e.video&&t.endpoint===e.video.endpoint&&(o=!0),t.isRemoved=s),t.isPresentation&&(e.presentation&&t.endpoint===e.presentation.endpoint&&(d=!0),t.isRemoved=r))})),a||c||i.ssrcs.push({userId:e.id,isMain:!1,endpoint:`audio${e.source}`,isVideo:!1,sourceGroups:[{semantics:"FID",sources:[e.source]}]}),s||o||!e.video||(n.push(e.video.endpoint),i.ssrcs.push({userId:e.id,isMain:!1,endpoint:e.video.endpoint,isVideo:!0,sourceGroups:e.video.sourceGroups})),r||d||!e.presentation||i.ssrcs.push({isPresentation:!0,userId:e.id,isMain:!1,endpoint:e.presentation.endpoint,isVideo:!0,sourceGroups:e.presentation.sourceGroups})}})),o.updatingParticipantsQueue)o.updatingParticipantsQueue.push(i);else{o.updatingParticipantsQueue=[],e=(0,s.default)(i),await r.setRemoteDescription({type:"offer",sdp:e});try{var t=await r.createAnswer();if(await r.setLocalDescription(t),u(c),0async function(e,t){if(o){var a=t?o.screenshareConference:o.conference;const i=t?o.screenshareConnection:o.connection;if(a&&i&&a.ssrcs){var n=Date.now();e={...a,transport:e.transport,sessionId:n,audioExtensions:e.audio?.["rtp-hdrexts"],audioPayloadTypes:e.audio?.["payload-types"],videoExtensions:e.video?.["rtp-hdrexts"],videoPayloadTypes:e.video?.["payload-types"]};o={...o,...t?{screenshareConference:e}:{conference:e}};try{await i.setRemoteDescription({type:"answer",sdp:(0,s.default)(e,!0,t)})}catch(e){console.error(e)}}}},startSharingScreen:()=>async function(){if(o)try{const e=await l("presentation");return e?(e.getTracks()[0].onended=()=>{o&&o.myId&&(o.streams?.[o.myId].presentation,u(o.myId),c())},new Promise((t=>{var{connection:a,dataChannel:t}=y([e],t,!0);o={...o,screenshareConnection:a,screenshareDataChannel:t}}))):void 0}catch(e){return}},joinGroupCall:()=>function(e,t,a,n){if(o)throw Error("Already in call");f("connecting");var s=new MediaStream;return a.srcObject=s,a.play().catch((e=>console.warn(e))),o={onUpdate:n,participants:[],myId:e,speaking:{},silence:(0,i.silence)(t),black:(0,i.black)({width:640,height:480}),analyserInterval:setInterval(v,1e3),audioElement:a,audioContext:t,mediaStream:s},new Promise((e=>{o={...o,...y([o.silence,o.black],e)}}))}});var n=a("./src/parseSdp.ts"),s=a("./src/buildSdp.ts"),i=a("./src/blacksilence.ts"),r=a("./src/utils.ts");let o;function c(e){o&&(o.screenshareDataChannel?.close(),o.screenshareConnection?.close(),e||o.onUpdate?.({"@type":"updateGroupCallLeavePresentation"}))}function d(e){return o?.streams?.[e]}function p(e,t){const a=(t=t||o?.myId)&&d(t)?.[e];return!!a&&a.getTracks()[0]?.enabled}function u(e){o?.onUpdate?.({"@type":"updateGroupCallStreams",userId:e,hasAudioStream:p("audio",e),hasVideoStream:p("video",e),hasPresentationStream:p("presentation",e),amplitude:o.speaking?.[e]})}function l(e,t="user"){return"presentation"===e?navigator.mediaDevices.getDisplayMedia({audio:!1,video:!0}):navigator.mediaDevices.getUserMedia({audio:"audio"===e&&{...r.IS_ECHO_CANCELLATION_SUPPORTED&&{echoCancellation:!0},...r.IS_NOISE_SUPPRESSION_SUPPORTED&&{noiseSuppression:!0}},video:"video"===e&&{facingMode:t}})}async function m(e,t){if(o&&o.myId&&o.connection&&o.streams){const a=d(o.myId)?.[e];if(a){const n=a.getTracks()[0];if(n){const a=[...o.connection.getSenders(),...o.screenshareConnection?.getSenders()||[]].find((e=>n.id===e.track?.id));if(a){t=void 0===t?!n.enabled:t;try{if(t&&!n.enabled){const t=await l(e);if(await a.replaceTrack(t.getTracks()[0]),o.streams[o.myId][e]=t,"video"===e)o.facingMode="user";else if("audio"===e){const e=o.audioContext;if(!e)return;const a=e.createMediaStreamSource(t),n=e.createAnalyser();n.minDecibels=-100,n.maxDecibels=-30,n.smoothingTimeConstant=.05,n.fftSize=1024,a.connect(n),o={...o,participantFunctions:{...o.participantFunctions,[o.myId]:{...o.participantFunctions?.[o.myId],getCurrentAmplitude:()=>{var e=new Uint8Array(n.frequencyBinCount);return n.getByteFrequencyData(e),(0,r.getAmplitude)(e,1.5)}}}}}}else if(!t&&n.enabled){n.stop();const t="audio"===e?o.silence:o.black;if(!t)return;await a.replaceTrack(t.getTracks()[0]),o.streams[o.myId][e]=t,"video"===e&&(o.facingMode=void 0)}u(o.myId),"presentation"!==e||t||c(!0)}catch(e){}}}}}}function f(e){o?.onUpdate?.({"@type":"updateGroupCallConnectionState",connectionState:e})}function g(){o&&(o.myId&&o.streams?.[o.myId]&&Object.values(o.streams[o.myId]||{}).forEach((e=>{e?.getTracks().forEach((e=>{e.stop()}))})),c(!0),o.dataChannel?.close(),o.connection?.close(),f("disconnected"),o.analyserInterval&&clearInterval(o.analyserInterval),o=void 0)}function v(){o&&o.participantFunctions&&Object.keys(o.participantFunctions).forEach((e=>{const t=o.participantFunctions[Number(e)].getCurrentAmplitude;var a,n;t&&(a=t(),n=o.speaking[e]||0,((o.speaking[e]=a)>r.THRESHOLD&&n<=r.THRESHOLD||a<=r.THRESHOLD&&n>r.THRESHOLD)&&u(e))}))}function S(e){if(o&&o.audioElement&&o.audioContext&&o.mediaStream){var t=o.conference?.ssrcs?.find((t=>t.endpoint===e.track.id));if(t&&t.userId){const{userId:n,isPresentation:s}=t;var a=o.participants?.find((e=>e.id===n));const i="video"===e.track.kind?s?"presentation":"video":"audio";if(e.track.onended=()=>{o?.streams?.[n][i],u(n)},t=e.streams[0],"audio"===e.track.kind){const e=o.mediaStream,s=new window.AudioContext,i=s.createMediaStreamSource(t),c=s.createGain();c.gain.value=(a?.volume||1e4)/1e4;const d=s.createGain();c.gain.value=1;const p=s.createAnalyser();p.minDecibels=-100,p.maxDecibels=-30,p.smoothingTimeConstant=.05,p.fftSize=1024,i.connect(p).connect(d).connect(c).connect(s.destination),e.addTrack(i.mediaStream.getAudioTracks()[0]);const u=new Audio;u.srcObject=t,u.muted=!0,u.remove(),o={...o,participantFunctions:{...o.participantFunctions,[n]:{...o.participantFunctions?.[n],setVolume:e=>{c.gain.value=1{d.gain.value=e?0:1},getCurrentAmplitude:()=>{var e=new Uint8Array(p.frequencyBinCount);return p.getByteFrequencyData(e),(0,r.getAmplitude)(e,1.5)}}}}}o={...o,streams:{...o.streams,[n]:{...o.streams?.[n],[i]:t}}},u(n)}}}function y(e,t,a=!1){const s=new RTCPeerConnection;var i=a?void 0:function(e){const t=e.createDataChannel("data",{id:0});return t.onopen=()=>{},t.onmessage=e=>{JSON.parse(e.data).colibriClass},t.onerror=e=>{console.log("%conerror","background: green; font-size: 5em"),console.error(e)},t}(s);return e.forEach((e=>e.getTracks().forEach((t=>{s.addTrack(t,e)})))),a||(s.oniceconnectionstatechange=()=>{var e=s.iceConnectionState;"connected"===e||"completed"===e?f("connected"):"checking"===e||"new"===e?f("connecting"):"disconnected"===s.iceConnectionState&&f("reconnecting")}),s.ontrack=S,s.onnegotiationneeded=async()=>{if(o){var i=o.myId;if(i){var r=await s.createOffer({offerToReceiveVideo:!0,offerToReceiveAudio:!a});if(await s.setLocalDescription(r),r.sdp){var c=(0,n.default)(r),d=a?void 0:{userId:"",sourceGroups:[{semantics:"FID",sources:[c.ssrc||0]}],isRemoved:a,isMain:!0,isVideo:!1,isPresentation:a,endpoint:a?"1":"0"},p=c["ssrc-groups"]&&{isPresentation:a,userId:"",sourceGroups:c["ssrc-groups"],isMain:!0,isVideo:!0,endpoint:a?"0":"1"};r=a?o.screenshareConference:o.conference;const s=[];a?(p&&s.push(p),d&&s.push(d)):(d&&s.push(d),p&&s.push(p)),d=e.find((e=>"audio"===e.getTracks()[0].kind)),p=e.find((e=>"video"===e.getTracks()[0].kind)),o={...o,...a?{screenshareConference:{...r,ssrcs:s}}:{conference:{...r,ssrcs:s}},streams:{...o.streams,[i]:{...o.streams?.[i],...d&&{audio:d},...!a&&p?{video:p}:{presentation:p}}}},u(i),t(c)}}}},{connection:s,dataChannel:i}}},"./src/types.ts":(e,t,a)=>{a.r(t)},"./src/utils.ts":(e,t,a)=>{a.r(t),a.d(t,{toTelegramSource:()=>function(e){return e<<0},fromTelegramSource:()=>function(e){return e>>>0},getAmplitude:()=>function(e,t=3){if(!e)return 0;var a=e.length;let n=0;for(let t=0;tfunction(e){return{id:e.id,name:e.name,"rtcp-fbs":e.feedbackTypes,clockrate:e.clockrate,parameters:e.parameters,channels:e.channels}},THRESHOLD:()=>n,IS_SCREENSHARE_SUPPORTED:()=>s,IS_ECHO_CANCELLATION_SUPPORTED:()=>i,IS_NOISE_SUPPRESSION_SUPPORTED:()=>r});const n=.1,s="getDisplayMedia"in(navigator?.mediaDevices||{}),i=navigator?.mediaDevices?.getSupportedConstraints().echoCancellation,r=navigator?.mediaDevices?.getSupportedConstraints().noiseSuppression}},t={};function a(n){var s=t[n];return void 0!==s||(s=t[n]={exports:{}},e[n](s,s.exports,a)),s.exports}a.d=(e,t)=>{for(var n in t)a.o(t,n)&&!a.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},a.o=(e,t)=>Object.prototype.hasOwnProperty.call(e,t),a.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})};var n={};(()=>{a.r(n),a.d(n,{handleUpdateGroupCallConnection:()=>e.handleUpdateGroupCallConnection,startSharingScreen:()=>e.startSharingScreen,joinGroupCall:()=>e.joinGroupCall,getDevices:()=>e.getDevices,getUserStreams:()=>e.getUserStreams,setVolume:()=>e.setVolume,isStreamEnabled:()=>e.isStreamEnabled,toggleStream:()=>e.toggleStream,leaveGroupCall:()=>e.leaveGroupCall,handleUpdateGroupCallParticipants:()=>e.handleUpdateGroupCallParticipants,switchCameraInput:()=>e.switchCameraInput,toggleSpeaker:()=>e.toggleSpeaker,toggleNoiseSuppression:()=>e.toggleNoiseSuppression,joinPhoneCall:()=>t.joinPhoneCall,processSignalingMessage:()=>t.processSignalingMessage,getStreams:()=>t.getStreams,toggleStreamP2p:()=>t.toggleStreamP2p,stopPhoneCall:()=>t.stopPhoneCall,switchCameraInputP2p:()=>t.switchCameraInputP2p,IS_SCREENSHARE_SUPPORTED:()=>s.IS_SCREENSHARE_SUPPORTED,THRESHOLD:()=>s.THRESHOLD});var e=a("./src/secretsauce.ts"),t=a("./src/p2p.ts"),s=(a("./src/p2pMessage.ts"),a("./src/utils.ts"));a("./src/types.ts")})();var s,i=exports;for(s in n)i[s]=n[s];n.__esModule&&Object.defineProperty(i,"__esModule",{value:!0})})(); \ No newline at end of file diff --git a/src/lib/secret-sauce/index.js.LICENSE.txt b/src/lib/secret-sauce/index.js.LICENSE.txt deleted file mode 100644 index 1d5cd7da2..000000000 --- a/src/lib/secret-sauce/index.js.LICENSE.txt +++ /dev/null @@ -1,39 +0,0 @@ -/*! ./blacksilence */ - -/*! ./buildSdp */ - -/*! ./parseSdp */ - -/*! ./secretsauce */ - -/*! ./types */ - -/*! ./utils */ - -/*!**********************!*\ - !*** ./src/index.ts ***! - \**********************/ - -/*!**********************!*\ - !*** ./src/types.ts ***! - \**********************/ - -/*!**********************!*\ - !*** ./src/utils.ts ***! - \**********************/ - -/*!*************************!*\ - !*** ./src/buildSdp.ts ***! - \*************************/ - -/*!*************************!*\ - !*** ./src/parseSdp.ts ***! - \*************************/ - -/*!****************************!*\ - !*** ./src/secretsauce.ts ***! - \****************************/ - -/*!*****************************!*\ - !*** ./src/blacksilence.ts ***! - \*****************************/ diff --git a/src/lib/secret-sauce/index.ts b/src/lib/secret-sauce/index.ts new file mode 100644 index 000000000..aa56b38ab --- /dev/null +++ b/src/lib/secret-sauce/index.ts @@ -0,0 +1,14 @@ +export { + handleUpdateGroupCallConnection, startSharingScreen, joinGroupCall, + getDevices, getUserStreams, setVolume, isStreamEnabled, toggleStream, + leaveGroupCall, handleUpdateGroupCallParticipants, switchCameraInput, + toggleSpeaker, toggleNoiseSuppression, +} from './secretsauce'; +export { + joinPhoneCall, processSignalingMessage, getStreams, toggleStreamP2p, stopPhoneCall, switchCameraInputP2p, +} from './p2p'; +export * from './p2pMessage'; +export { + IS_SCREENSHARE_SUPPORTED, THRESHOLD, +} from './utils'; +export * from './types'; diff --git a/src/lib/secret-sauce/p2p.d.ts b/src/lib/secret-sauce/p2p.d.ts deleted file mode 100644 index f6865d859..000000000 --- a/src/lib/secret-sauce/p2p.d.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { ApiPhoneCallConnection } from './types'; -import { P2pMessage } from './p2pMessage'; -import { StreamType } from './secretsauce'; -export declare function getStreams(): { - video?: MediaStream | undefined; - audio?: MediaStream | undefined; - presentation?: MediaStream | undefined; - ownAudio?: MediaStream | undefined; - ownVideo?: MediaStream | undefined; - ownPresentation?: MediaStream | undefined; -} | undefined; -export declare function switchCameraInputP2p(): Promise; -export declare function toggleStreamP2p(streamType: StreamType, value?: boolean | undefined): Promise; -export declare function joinPhoneCall(connections: ApiPhoneCallConnection[], emitSignalingData: (data: P2pMessage) => void, isOutgoing: boolean, shouldStartVideo: boolean, onUpdate: (...args: any[]) => void): Promise; -export declare function stopPhoneCall(): void; -export declare function processSignalingMessage(message: P2pMessage): Promise; diff --git a/src/lib/secret-sauce/p2p.ts b/src/lib/secret-sauce/p2p.ts new file mode 100644 index 000000000..13f4ba996 --- /dev/null +++ b/src/lib/secret-sauce/p2p.ts @@ -0,0 +1,478 @@ +import { black, silence } from './blacksilence'; +import type { ApiPhoneCallConnection, P2pParsedSdp } from './types'; +import parseSdp from './parseSdp'; +import type { MediaContent, MediaStateMessage, P2pMessage } from './p2pMessage'; +import { + fromTelegramSource, + IS_ECHO_CANCELLATION_SUPPORTED, + IS_NOISE_SUPPRESSION_SUPPORTED, + p2pPayloadTypeToConference, +} from './utils'; +import buildSdp, { Conference } from './buildSdp'; +import { getUserStreams, StreamType } from './secretsauce'; + +type P2pState = { + connection: RTCPeerConnection; + dataChannel: RTCDataChannel; + emitSignalingData: (data: P2pMessage) => void; + onUpdate: (...args: any[]) => void; + conference?: Partial; + isOutgoing: boolean; + candidates: string[]; + streams: { + video?: MediaStream; + audio?: MediaStream; + presentation?: MediaStream; + ownAudio?: MediaStream; + ownVideo?: MediaStream; + ownPresentation?: MediaStream; + }; + silence: MediaStream; + blackVideo: MediaStream; + blackPresentation: MediaStream; + mediaState: Omit; + audio: HTMLAudioElement; + gotInitialSetup?: boolean; + facingMode?: VideoFacingModeEnum; +}; + +let state: P2pState | undefined; + +export function getStreams() { + return state?.streams; +} + +function updateStreams() { + state?.onUpdate({ + ...state.mediaState, + '@type': 'updatePhoneCallMediaState', + }); +} + +function getUserStream(streamType: StreamType, facing: VideoFacingModeEnum = 'user') { + if (streamType === 'presentation') { + return (navigator.mediaDevices as any).getDisplayMedia({ + audio: false, + video: true, + }); + } + + return navigator.mediaDevices.getUserMedia({ + audio: streamType === 'audio' ? { + // @ts-ignore + ...(IS_ECHO_CANCELLATION_SUPPORTED && { echoCancellation: true }), + ...(IS_NOISE_SUPPRESSION_SUPPORTED && { noiseSuppression: true }), + } : false, + video: streamType === 'video' ? { + facingMode: facing, + } : false, + }); +} + +export async function switchCameraInputP2p() { + if (!state || !state.facingMode) { + return; + } + + const stream = state.streams.ownVideo; + + if (!stream) return; + + const track = stream.getTracks()[0]; + + if (!track) { + return; + } + + const sender = state.connection.getSenders().find((l) => track.id === l.track?.id); + + if (!sender) { + return; + } + + state.facingMode = state.facingMode === 'environment' ? 'user' : 'environment'; + try { + const newStream = await getUserStream('video', state.facingMode); + + await sender.replaceTrack(newStream.getTracks()[0]); + state.streams.ownVideo = newStream; + updateStreams(); + } catch (e) { + + } +} + +export async function toggleStreamP2p(streamType: StreamType, value: boolean | undefined = undefined) { + if (!state) return; + const stream = streamType === 'audio' ? state.streams.ownAudio + : (streamType === 'video' ? state.streams.ownVideo : state.streams.ownPresentation); + + if (!stream) return; + const track = stream.getTracks()[0]; + + if (!track) { + return; + } + + const sender = state.connection.getSenders().find((l) => track.id === l.track?.id); + + if (!sender) { + return; + } + + value = value === undefined ? !track.enabled : value; + + try { + if (value && !track.enabled) { + const newStream = await getUserStream(streamType); + newStream.getTracks()[0].onended = () => { + toggleStreamP2p(streamType, false); + }; + await sender.replaceTrack(newStream.getTracks()[0]); + if (streamType === 'audio') { + state.streams.ownAudio = newStream; + } else if (streamType === 'video') { + state.streams.ownVideo = newStream; + state.facingMode = 'user'; + } else { + state.streams.ownPresentation = newStream; + } + if (streamType === 'video' || streamType === 'presentation') { + toggleStreamP2p(streamType === 'video' ? 'presentation' : 'video', false); + } + // if (streamType === 'video') { + // state.facingMode = 'user'; + // } + } else if (!value && track.enabled) { + track.stop(); + const newStream = streamType === 'audio' ? state.silence + : (streamType === 'video' ? state.blackVideo : state.blackPresentation); + if (!newStream) return; + + await sender.replaceTrack(newStream.getTracks()[0]); + + if (streamType === 'audio') { + state.streams.ownAudio = newStream; + } else if (streamType === 'video') { + state.streams.ownVideo = newStream; + } else { + state.streams.ownPresentation = newStream; + } + // if (streamType === 'video') { + // state.facingMode = undefined; + // } + } + updateStreams(); + sendMediaState(); + } catch (e) { + + } +} + +export async function joinPhoneCall( + connections: ApiPhoneCallConnection[], + emitSignalingData: (data: P2pMessage) => void, + isOutgoing: boolean, + shouldStartVideo: boolean, + onUpdate: (...args: any[]) => void, +) { + const conn = new RTCPeerConnection({ + iceServers: connections.map((connection) => ( + { + urls: [ + connection.isTurn && `turn:${connection.ip}:${connection.port}`, + connection.isStun && `stun:${connection.ip}:${connection.port}`, + ].filter(Boolean) as string[], + username: connection.username, + credentialType: 'password', + credential: connection.password, + } + )), + iceCandidatePoolSize: 2, + }); + const slnc = silence(new AudioContext()); + const video = black({ width: 640, height: 480 }); + const screenshare = black({ width: 640, height: 480 }); + conn.addTrack(slnc.getTracks()[0], slnc); + conn.addTrack(video.getTracks()[0], video); + conn.addTrack(screenshare.getTracks()[0], screenshare); + + conn.onicecandidate = (e) => { + if (!e.candidate) return; + + emitSignalingData({ + '@type': 'Candidates', + candidates: [{ + sdpString: e.candidate.candidate, + }], + }); + }; + + conn.onconnectionstatechange = () => { + onUpdate({ + '@type': 'updatePhoneCallConnectionState', + connectionState: conn.connectionState, + }); + }; + + conn.ontrack = (e) => { + if (!state) return; + + const stream = e.streams[0]; + if (e.track.kind === 'audio') { + state.audio.srcObject = stream; + state.audio.play().catch(); + state.streams.audio = stream; + } else if (e.transceiver.mid === '1') { + state.streams.video = stream; + } else { + state.streams.presentation = stream; + } + + updateStreams(); + }; + + const dc = conn.createDataChannel('data', { + id: 0, + negotiated: true, + }); + + dc.onmessage = (e) => { + processSignalingMessage(JSON.parse(e.data)); + }; + + const audio = new Audio(); + + state = { + audio, + connection: conn, + emitSignalingData, + isOutgoing, + candidates: [], + onUpdate, + streams: { + ownVideo: video, + ownAudio: slnc, + ownPresentation: screenshare, + }, + mediaState: { + isBatteryLow: false, + screencastState: 'inactive', + videoState: 'inactive', + videoRotation: 0, + isMuted: true, + }, + blackVideo: video, + blackPresentation: screenshare, + silence: slnc, + dataChannel: dc, + }; + + try { + if (shouldStartVideo) { + toggleStreamP2p('video', true); + } + toggleStreamP2p('audio', true); + } catch (e) { + + } + + if (isOutgoing) { + const offer = await conn.createOffer({ + offerToReceiveAudio: true, + offerToReceiveVideo: true, + }); + + await conn.setLocalDescription(offer); + + sendInitialSetup(parseSdp(offer, true) as P2pParsedSdp); + } +} + +export function stopPhoneCall() { + if (!state) return; + + state.streams.ownVideo?.getTracks().forEach((track) => track.stop()); + state.streams.ownPresentation?.getTracks().forEach((track) => track.stop()); + state.streams.ownAudio?.getTracks().forEach((track) => track.stop()); + state.dataChannel.close(); + state.connection.close(); + state = undefined; +} + +function sendMediaState() { + if (!state) return; + const { emitSignalingData, streams } = state; + + emitSignalingData({ + '@type': 'MediaState', + videoRotation: 0, + isMuted: !streams.ownAudio?.getTracks()[0].enabled, + isBatteryLow: true, + videoState: streams.ownVideo?.getTracks()[0].enabled ? 'active' : 'inactive', + screencastState: streams.ownPresentation?.getTracks()[0].enabled ? 'active' : 'inactive', + }); +} + +function filterVP8(mediaContent: MediaContent) { + if (!state || state.isOutgoing) return mediaContent; + + const { payloadTypes } = mediaContent!; + const idx = payloadTypes.findIndex((payloadType) => payloadType.name === 'VP8'); + const vp8PayloadType = payloadTypes[idx]; + const rtxIdx = payloadTypes.findIndex((payloadType) => Number(payloadType.parameters?.apt) === vp8PayloadType.id); + mediaContent.payloadTypes = [payloadTypes[idx], payloadTypes[rtxIdx]]; + + return mediaContent; +} + +function sendInitialSetup(sdp: P2pParsedSdp) { + if (!state) return; + const { emitSignalingData } = state; + + if (!sdp.ssrc || !sdp['ssrc-groups'] || !sdp['ssrc-groups'][0] || !sdp['ssrc-groups'][1]) return; + + emitSignalingData({ + '@type': 'InitialSetup', + fingerprints: sdp.fingerprints, + ufrag: sdp.ufrag, + pwd: sdp.pwd, + audio: { + ssrc: fromTelegramSource(sdp.ssrc).toString(), + ssrcGroups: [], + payloadTypes: sdp.audioPayloadTypes, + rtpExtensions: sdp.audioExtmap, + }, + video: filterVP8({ + ssrc: fromTelegramSource(sdp['ssrc-groups'][0].sources[0]).toString(), + ssrcGroups: [{ + semantics: sdp['ssrc-groups'][0].semantics, + ssrcs: sdp['ssrc-groups'][0].sources.map(fromTelegramSource), + }], + payloadTypes: sdp.videoPayloadTypes, + rtpExtensions: sdp.videoExtmap, + }), + screencast: filterVP8({ + ssrc: fromTelegramSource(sdp['ssrc-groups'][1].sources[0]).toString(), + ssrcGroups: [{ + semantics: sdp['ssrc-groups'][1].semantics, + ssrcs: sdp['ssrc-groups'][1].sources.map(fromTelegramSource), + }], + payloadTypes: sdp.screencastPayloadTypes, + rtpExtensions: sdp.screencastExtmap, + }), + }); +} + +export async function processSignalingMessage(message: P2pMessage) { + if (!state || !state.connection) return; + + console.log(message); + + switch (message['@type']) { + case 'MediaState': { + state.mediaState = message; + updateStreams(); + sendMediaState(); + break; + } + case 'Candidates': { + const { candidates, gotInitialSetup } = state; + + if (!candidates) return; + + message.candidates.forEach((candidate) => { + state!.candidates.push(candidate.sdpString); + }); + + if (gotInitialSetup) { + await Promise.all(state.candidates.map((c) => state!.connection.addIceCandidate({ + candidate: c, + sdpMLineIndex: 0, + }))); + } + + break; + } + case 'InitialSetup': { + const { + connection, isOutgoing, + } = state; + if (!connection) return; + + const newConference = { + transport: { + candidates: [], + ufrag: message.ufrag, + pwd: message.pwd, + fingerprints: message.fingerprints, + 'rtcp-mux': false, + xmlns: '', + }, + sessionId: Date.now(), + ssrcs: [ + message.audio && { + isVideo: false, + isMain: false, + userId: '123', + endpoint: '0', + sourceGroups: [{ + semantics: 'FID', + sources: [message.audio.ssrc], + }], + }, + message.video && { + isVideo: true, + isPresentation: false, + isMain: false, + userId: '123', + endpoint: '1', + sourceGroups: message.video.ssrcGroups.map((l) => ({ + semantics: l.semantics, + sources: l.ssrcs, + })), + }, + message.screencast && { + isVideo: true, + isPresentation: true, + isMain: false, + userId: '123', + endpoint: '2', + sourceGroups: message.screencast.ssrcGroups.map((l) => ({ + semantics: l.semantics, + sources: l.ssrcs, + })), + }, + ], + audioPayloadTypes: message.audio!.payloadTypes?.map(p2pPayloadTypeToConference) || [], + audioExtensions: message.audio!.rtpExtensions, + videoPayloadTypes: filterVP8(message.video!).payloadTypes?.map(p2pPayloadTypeToConference) || [], + videoExtensions: message.video!.rtpExtensions, + } as Conference; + + await connection.setRemoteDescription({ + sdp: buildSdp(newConference, isOutgoing, undefined, true), + type: isOutgoing ? 'answer' : 'offer', + }); + + state.conference = newConference; + + if (!isOutgoing) { + const answer = await connection.createAnswer(); + if (!answer) return; + + await connection.setLocalDescription(answer); + + sendInitialSetup(parseSdp(answer, true) as P2pParsedSdp); + } + + state.gotInitialSetup = true; + await Promise.all(state.candidates.map((c) => connection.addIceCandidate({ + candidate: c, + sdpMLineIndex: 0, + }))); + + break; + } + } +} diff --git a/src/lib/secret-sauce/p2pMessage.d.ts b/src/lib/secret-sauce/p2pMessage.d.ts deleted file mode 100644 index 2cab48ac6..000000000 --- a/src/lib/secret-sauce/p2pMessage.d.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { Fingerprint, RTCPFeedbackParam, RTPExtension } from './types'; -export declare type VideoState = 'inactive' | 'active' | 'suspended'; -export declare type VideoRotation = 0 | 90 | 180 | 270; -export declare type MediaStateMessage = { - '@type': 'MediaState'; - isMuted: boolean; - videoState: VideoState; - videoRotation: VideoRotation; - screencastState: VideoState; - isBatteryLow: boolean; -}; -declare type CandidatesMessage = { - '@type': 'Candidates'; - candidates: P2pCandidate[]; -}; -export declare type InitialSetupMessage = { - '@type': 'InitialSetup'; - ufrag: string; - pwd: string; - fingerprints: Fingerprint[]; - audio?: MediaContent; - video?: MediaContent; - screencast?: MediaContent; -}; -export declare type MediaContent = { - ssrc: string; - ssrcGroups: P2pSsrcGroup[]; - payloadTypes: P2PPayloadType[]; - rtpExtensions: RTPExtension[]; -}; -export interface P2PPayloadType { - id: number; - name: string; - clockrate: number; - channels: number; - parameters?: Record; - feedbackTypes?: RTCPFeedbackParam[]; -} -declare type P2pSsrcGroup = { - semantics: string; - ssrcs: number[]; -}; -declare type P2pCandidate = { - sdpString: string; -}; -export declare type P2pMessage = CandidatesMessage | InitialSetupMessage | MediaStateMessage; -export {}; diff --git a/src/lib/secret-sauce/p2pMessage.ts b/src/lib/secret-sauce/p2pMessage.ts new file mode 100644 index 000000000..bd574442d --- /dev/null +++ b/src/lib/secret-sauce/p2pMessage.ts @@ -0,0 +1,58 @@ +import type { + Fingerprint, RTCPFeedbackParam, RTPExtension, +} from './types'; + +export type VideoState = 'inactive' | 'active' | 'suspended'; + +export type VideoRotation = 0 | 90 | 180 | 270; + +export type MediaStateMessage = { + '@type': 'MediaState'; + isMuted: boolean; + videoState: VideoState; + videoRotation: VideoRotation; + screencastState: VideoState; + isBatteryLow: boolean; +}; + +type CandidatesMessage = { + '@type': 'Candidates'; + candidates: P2pCandidate[]; +}; + +export type InitialSetupMessage = { + '@type': 'InitialSetup'; + ufrag: string; + pwd: string; + fingerprints: Fingerprint[]; + audio?: MediaContent; + video?: MediaContent; + screencast?: MediaContent; +}; + +export type MediaContent = { + ssrc: string; + ssrcGroups: P2pSsrcGroup[]; + payloadTypes: P2PPayloadType[]; + rtpExtensions: RTPExtension[]; +}; + +export interface P2PPayloadType { + id: number; + name: string; + clockrate: number; + channels: number; + parameters?: Record; + feedbackTypes?: RTCPFeedbackParam[]; +} + +type P2pSsrcGroup = { + semantics: string; + ssrcs: number[]; +}; + +type P2pCandidate = { + sdpString: string; +}; + +export type P2pMessage = CandidatesMessage | InitialSetupMessage | MediaStateMessage; diff --git a/src/lib/secret-sauce/parseSdp.d.ts b/src/lib/secret-sauce/parseSdp.d.ts deleted file mode 100644 index bf0f5e4f7..000000000 --- a/src/lib/secret-sauce/parseSdp.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { JoinGroupCallPayload } from './types'; -declare const _default: (sessionDescription: RTCSessionDescriptionInit, isP2p?: boolean) => JoinGroupCallPayload; -export default _default; diff --git a/src/lib/secret-sauce/parseSdp.ts b/src/lib/secret-sauce/parseSdp.ts new file mode 100644 index 000000000..2083a02c6 --- /dev/null +++ b/src/lib/secret-sauce/parseSdp.ts @@ -0,0 +1,140 @@ +import { toTelegramSource } from './utils'; +import type { JoinGroupCallPayload, SsrcGroup } from './types'; + +export default (sessionDescription: RTCSessionDescriptionInit, isP2p = false): JoinGroupCallPayload => { + if (!sessionDescription || !sessionDescription.sdp) { + throw Error('Failed parsing SDP: session description is null'); + } + + const sections = sessionDescription + .sdp + .split('\r\nm=') + .map((s, i) => (i === 0 ? s : `m=${s}`)) + .reduce((acc: Record, el) => { + const name = el.match(/^m=(.+?)\s/)?.[1] || 'header'; + acc[acc.hasOwnProperty(name) && name === 'video' ? 'screencast' : name] = el.split('\r\n').filter(Boolean); + return acc; + }, {}); + + const lookup = (prefix: string, sectionName?: string) => { + if (!sectionName) { + return Object.values(sections).map((section) => { + return section.find((line) => line.startsWith(prefix))?.substr(prefix.length); + }).filter(Boolean)[0]; + } else { + return sections[sectionName]?.find((line) => line.startsWith(prefix))?.substr(prefix.length); + } + }; + + const parseExtmaps = (sectionName: string) => { + return sections[sectionName].filter((l) => l.startsWith('a=extmap')).map((l) => { + const [, id, uri] = l.match(/extmap:(\d+)(?:\/.+)?\s(.+)/)!; + return { id: Number(id), uri }; + }); + }; + + const parsePayloadTypes = (sectionName: string) => { + const payloads = sections[sectionName].filter((l) => l.startsWith('a=rtpmap')).map((l) => { + const [, id, data] = l.match(/:(\d+)\s(.+)/) || []; + const [name, clockrate, channels] = data.split('/'); + return { + id: Number(id), name, clockrate: Number(clockrate), ...(channels && { channels: Number(channels) }), + }; + }); + + const fbParams = sections[sectionName].filter((l) => l.startsWith('a=rtcp-fb')).map((l) => { + const [, id, data] = l.match(/:(\d+)\s(.+)/) || []; + const [type, subtype] = data.split(' '); + return { id: Number(id), type, subtype: subtype || '' }; + }); + + const parameters = sections[sectionName].filter((l) => l.startsWith('a=fmtp')).map((l) => { + const [, id, data] = l.match(/:(\d+)\s(.+)/) || []; + const d = data.split(';').reduce((acc: Record, q) => { + const [name, value] = q.split('='); + acc[name] = value; + return acc; + }, {}); + if (Object.values(d).some((z) => !z)) return undefined; + return { id: Number(id), data: d }; + }).filter(Boolean); + + return payloads.map((payload) => { + const p = parameters.filter((l) => l!.id === payload.id).map((q) => q!.data).reduce((acc, el) => { + return Object.assign(acc, el); + }, {}); + const f = fbParams.filter((l) => l.id === payload.id).map((l) => { + return { + type: l.type, + subtype: l.subtype, + }; + }); + + return { + ...payload, + ...(Object.keys(p).length > 0 && { parameters: p }), + ...(f.length > 0 && { feedbackTypes: f }), + }; + }); + }; + + const rawSource = lookup('a=ssrc:', 'audio'); + const sourceAudio = rawSource && Number(rawSource.split(' ')[0]); + + // TODO multiple source groups + const rawSourceVideo = lookup('a=ssrc-group:', 'video')?.split(' ') || undefined; + const rawSourceScreencast = lookup('a=ssrc-group:', 'screencast')?.split(' ') || undefined; + + if (!rawSourceVideo) { + throw Error('Failed parsing SDP: no video ssrc'); + } + + const [hash, fingerprint] = lookup('a=fingerprint:')?.split(' ') || []; + + const setup = lookup('a=setup:'); + if (!hash || !fingerprint) { + throw Error('Failed parsing SDP: no fingerprint'); + } + + console.log(sections); + + const ufrag = lookup('a=ice-ufrag:'); + const pwd = lookup('a=ice-pwd:'); + + if (!ufrag || !pwd) { + throw Error('Failed parsing SDP: no ICE ufrag or pwd'); + } + + return { + fingerprints: [ + { + fingerprint, + hash, + setup: isP2p ? setup! : 'active', + }, + ], + pwd, + ufrag, + ...(sourceAudio && { ssrc: toTelegramSource(sourceAudio) }), + ...(rawSourceVideo && { + 'ssrc-groups': [ + { + semantics: rawSourceVideo[0], + sources: rawSourceVideo.slice(1, rawSourceVideo.length).map(Number).map(toTelegramSource), + }, + (isP2p && rawSourceScreencast && { + semantics: rawSourceScreencast[0], + sources: rawSourceScreencast.slice(1, rawSourceScreencast.length).map(Number).map(toTelegramSource), + }), + ].filter(Boolean) as SsrcGroup[], + }), + ...(isP2p && { + audioExtmap: parseExtmaps('audio'), + videoExtmap: parseExtmaps('video'), + screencastExtmap: parseExtmaps('screencast'), + audioPayloadTypes: parsePayloadTypes('audio'), + videoPayloadTypes: parsePayloadTypes('video'), + screencastPayloadTypes: parsePayloadTypes('screencast'), + }), + }; +}; diff --git a/src/lib/secret-sauce/secretsauce.d.ts b/src/lib/secret-sauce/secretsauce.d.ts deleted file mode 100644 index 211376510..000000000 --- a/src/lib/secret-sauce/secretsauce.d.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { GroupCallConnectionData, GroupCallParticipant, JoinGroupCallPayload } from './types'; -export declare type StreamType = 'audio' | 'video' | 'presentation'; -export declare function getDevices(streamType: StreamType, isInput?: boolean): Promise; -export declare function toggleSpeaker(): void; -export declare function toggleNoiseSuppression(): void; -export declare function getUserStreams(userId: string): { - audio?: MediaStream | undefined; - video?: MediaStream | undefined; - presentation?: MediaStream | undefined; -} | undefined; -export declare function setVolume(userId: string, volume: number): void; -export declare function isStreamEnabled(streamType: StreamType, userId?: string): boolean; -export declare function switchCameraInput(): Promise; -export declare function toggleStream(streamType: StreamType, value?: boolean | undefined): Promise; -export declare function leaveGroupCall(): void; -export declare function handleUpdateGroupCallParticipants(updatedParticipants: GroupCallParticipant[]): Promise; -export declare function handleUpdateGroupCallConnection(data: GroupCallConnectionData, isPresentation: boolean): Promise; -export declare function startSharingScreen(): Promise; -export declare function joinGroupCall(myId: string, audioContext: AudioContext, audioElement: HTMLAudioElement, onUpdate: (...args: any[]) => void): Promise; diff --git a/src/lib/secret-sauce/secretsauce.ts b/src/lib/secret-sauce/secretsauce.ts new file mode 100644 index 000000000..de640fcbf --- /dev/null +++ b/src/lib/secret-sauce/secretsauce.ts @@ -0,0 +1,851 @@ +import parseSdp from './parseSdp'; +import { ColibriClass } from './colibriClass'; +import type { + GroupCallConnectionData, GroupCallConnectionState, GroupCallParticipant, JoinGroupCallPayload, +} from './types'; +import buildSdp, { Conference, Ssrc } from './buildSdp'; +import { black, silence } from './blacksilence'; +import { + getAmplitude, + IS_ECHO_CANCELLATION_SUPPORTED, + IS_NOISE_SUPPRESSION_SUPPORTED, + THRESHOLD, +} from './utils'; + +export type StreamType = 'audio' | 'video' | 'presentation'; + +type GroupCallState = { + connection?: RTCPeerConnection; + screenshareConnection?: RTCPeerConnection; + dataChannel?: RTCDataChannel; + screenshareDataChannel?: RTCDataChannel; + participants?: GroupCallParticipant[]; + conference?: Partial; + screenshareConference?: Partial; + streams?: Record; + participantFunctions?: Record void; + toggleMute?: (muted: boolean) => void; + getCurrentAmplitude?: () => number; + }>; + onUpdate?: (...args: any[]) => void; + myId?: string; + black?: MediaStream; + silence?: MediaStream; + updatingParticipantsQueue?: any[]; + facingMode?: VideoFacingModeEnum; + isSpeakerDisabled?: boolean; + analyserInterval?: number; + speaking?: Record; + audioElement?: HTMLAudioElement; + destination?: MediaStreamAudioDestinationNode; + audioContext?: AudioContext; + mediaStream?: MediaStream; +}; + +let state: GroupCallState | undefined; + +export async function getDevices(streamType: StreamType, isInput = true) { + return (await navigator.mediaDevices.enumerateDevices()) + .filter((l) => l.kind === `${streamType}${isInput ? 'input' : 'output'}`); +} + +export function toggleSpeaker() { + if (!state) { + return; + } + + state.isSpeakerDisabled = !state.isSpeakerDisabled; + state?.onUpdate?.({ + '@type': 'updateGroupCallConnectionState', + connectionState: 'connected', + isSpeakerDisabled: state.isSpeakerDisabled, + }); + if (state.participantFunctions) { + Object.values(state.participantFunctions).forEach((l) => { + l.toggleMute?.(!!state?.isSpeakerDisabled); + }); + } +} + +function leavePresentation(isFromToggle?: boolean) { + if (!state) { + return; + } + state.screenshareDataChannel?.close(); + state.screenshareConnection?.close(); + + if (!isFromToggle) { + state.onUpdate?.({ + '@type': 'updateGroupCallLeavePresentation', + }); + } +} + +export function toggleNoiseSuppression() { + if (!state || !state.myId || !state.streams) { + return; + } + + const audioStream = state.streams[state.myId].audio; + if (!audioStream) { + return; + } + + const track = audioStream.getTracks()[0]; + + if (!track) { + return; + } + + // @ts-ignore + const { echoCancellation, noiseSuppression } = track.getConstraints(); + + track.applyConstraints({ + echoCancellation: !echoCancellation, + // @ts-ignore + noiseSuppression: !noiseSuppression, + }); +} + +export function getUserStreams(userId: string) { + return state?.streams?.[userId]; +} + +export function setVolume(userId: string, volume: number) { + const participantFunctions = state?.participantFunctions?.[userId]; + if (!participantFunctions) return; + participantFunctions.setVolume?.(volume); +} + +export function isStreamEnabled(streamType: StreamType, userId?: string) { + const id = userId || state?.myId; + const stream = id && getUserStreams(id)?.[streamType]; + if (!stream) return false; + + return stream.getTracks()[0]?.enabled; +} + +function updateGroupCallStreams(userId: string) { + state?.onUpdate?.({ + '@type': 'updateGroupCallStreams', + userId, + hasAudioStream: isStreamEnabled('audio', userId), + hasVideoStream: isStreamEnabled('video', userId), + hasPresentationStream: isStreamEnabled('presentation', userId), + amplitude: state.speaking?.[userId], + }); +} + +function getUserStream(streamType: StreamType, facing: VideoFacingModeEnum = 'user') { + if (streamType === 'presentation') { + return (navigator.mediaDevices as any).getDisplayMedia({ + audio: false, + video: true, + }); + } + + return navigator.mediaDevices.getUserMedia({ + audio: streamType === 'audio' ? { + // @ts-ignore + ...(IS_ECHO_CANCELLATION_SUPPORTED && { echoCancellation: true }), + ...(IS_NOISE_SUPPRESSION_SUPPORTED && { noiseSuppression: true }), + } : false, + video: streamType === 'video' ? { + facingMode: facing, + } : false, + + }); +} + +export async function switchCameraInput() { + if (!state?.myId || !state.connection || !state.streams || !state.facingMode) { + return; + } + + const stream = getUserStreams(state.myId)?.video; + + if (!stream) return; + + const track = stream.getTracks()[0]; + + if (!track) { + return; + } + + const sender = state.connection.getSenders().find((l) => track.id === l.track?.id); + + if (!sender) { + return; + } + + state.facingMode = state.facingMode === 'environment' ? 'user' : 'environment'; + try { + const newStream = await getUserStream('video', state.facingMode); + + await sender.replaceTrack(newStream.getTracks()[0]); + state.streams[state.myId].video = newStream; + } catch (e) { + + } +} + +export async function toggleStream(streamType: StreamType, value: boolean | undefined = undefined) { + if (!state || !state.myId || !state.connection || !state.streams) { + return; + } + + const stream = getUserStreams(state.myId)?.[streamType]; + if (!stream) return; + + const track = stream.getTracks()[0]; + + if (!track) { + return; + } + + const sender = [ + ...state.connection.getSenders(), + ...(state.screenshareConnection?.getSenders() || []), + ].find((l) => track.id === l.track?.id); + + if (!sender) { + return; + } + + value = value === undefined ? !track.enabled : value; + + try { + if (value && !track.enabled) { + const newStream = await getUserStream(streamType); + await sender.replaceTrack(newStream.getTracks()[0]); + state.streams[state.myId][streamType] = newStream; + if (streamType === 'video') { + state.facingMode = 'user'; + } else if (streamType === 'audio') { + const { audioContext } = state; + if (!audioContext) return; + const source = audioContext.createMediaStreamSource(newStream); + + const analyser = audioContext.createAnalyser(); + analyser.minDecibels = -100; + analyser.maxDecibels = -30; + analyser.smoothingTimeConstant = 0.05; + analyser.fftSize = 1024; + + source.connect(analyser); + + state = { + ...state, + participantFunctions: { + ...state.participantFunctions, + [state.myId]: { + ...state.participantFunctions?.[state.myId], + getCurrentAmplitude: () => { + const array = new Uint8Array(analyser.frequencyBinCount); + analyser.getByteFrequencyData(array); + return getAmplitude(array, 1.5); + }, + }, + }, + }; + } + } else if (!value && track.enabled) { + track.stop(); + const newStream = streamType === 'audio' ? state.silence : state.black; + if (!newStream) return; + + await sender.replaceTrack(newStream.getTracks()[0]); + state.streams[state.myId][streamType] = newStream; + if (streamType === 'video') { + state.facingMode = undefined; + } + } + updateGroupCallStreams(state.myId!); + if (streamType === 'presentation' && !value) leavePresentation(true); + } catch (e) { + + } +} + +function updateConnectionState(connectionState: GroupCallConnectionState) { + state?.onUpdate?.({ + '@type': 'updateGroupCallConnectionState', + connectionState, + }); +} + +export function leaveGroupCall() { + if (!state) { + return; + } + + if (state.myId && state.streams?.[state.myId]) { + Object.values(state.streams[state.myId] || {}).forEach((stream) => { + stream?.getTracks().forEach((track) => { + track.stop(); + }); + }); + } + leavePresentation(true); + state.dataChannel?.close(); + state.connection?.close(); + updateConnectionState('disconnected'); + + if (state.analyserInterval) { + clearInterval(state.analyserInterval); + } + + state = undefined; +} + +function analyzeAmplitudes() { + if (!state || !state.participantFunctions) return; + + Object.keys(state.participantFunctions).forEach((id) => { + const { getCurrentAmplitude } = state!.participantFunctions![Number(id)]; + + if (getCurrentAmplitude) { + const amplitude = getCurrentAmplitude(); + const prevAmplitude = state!.speaking![id] || 0; + state!.speaking![id] = amplitude; + if ((amplitude > THRESHOLD && prevAmplitude <= THRESHOLD) + || (amplitude <= THRESHOLD && prevAmplitude > THRESHOLD)) { + updateGroupCallStreams(id); + } + } + }); +} + +function createDataChannel(connection: RTCPeerConnection) { + const dataChannel = connection.createDataChannel('data', { + id: 0, + }); + + dataChannel.onopen = () => { + // console.log('Data channel open!'); + }; + + dataChannel.onmessage = (e) => { + // console.log('onmessage'); + const data = JSON.parse(e.data) as ColibriClass; + // console.log(data); + switch (data.colibriClass) { + case 'DominantSpeakerEndpointChangeEvent': + break; + case 'SenderVideoConstraints': + + break; + case 'EndpointConnectivityStatusChangeEvent': + + break; + } + }; + + dataChannel.onerror = (e) => { + console.log('%conerror', 'background: green; font-size: 5em'); + console.error(e); + }; + + return dataChannel; +} + +export async function handleUpdateGroupCallParticipants(updatedParticipants: GroupCallParticipant[]) { + if (!state) { + return; + } + + const { + participants, conference, connection, myId, + } = state; + + if (!participants || !conference || !connection || !conference.ssrcs || !conference.transport || !myId) { + return; + } + + // Joined from another client + if (updatedParticipants.find((participant) => { + return participant.isSelf + && participant.source + !== state?.conference?.ssrcs?.find((l) => l.isMain && !l.isVideo)?.sourceGroups[0].sources[0]; + })) { + leaveGroupCall(); + return; + } + + const newEndpoints: string[] = []; + updatedParticipants.forEach((participant) => { + console.log('handleUpdateGroupCallParticipants', participant); + if (participant.isSelf) { + if (participant.isMuted && !participant.canSelfUnmute) { + // Muted by admin + toggleStream('audio', false); + toggleStream('video', false); + toggleStream('presentation', false); + } + return; + } + + const { isLeft } = participant; + const isAudioLeft = participant.isMuted || participant.isMutedByMe; + const isVideoLeft = !participant.isVideoJoined || !participant.video || isLeft; + const isPresentationLeft = !participant.presentation || isLeft; + + let hasVideo = false; + let hasAudio = false; + let hasPresentation = false; + + conference.ssrcs!.filter((l) => l.userId === participant.id).forEach((ssrc) => { + if (!ssrc.isVideo) { + if (ssrc.sourceGroups[0].sources[0] === participant.source) { + hasAudio = true; + } + // console.log('has audio, removed=', isAudioLeft); + ssrc.isRemoved = isAudioLeft; + } + + if (ssrc.isVideo) { + if (!ssrc.isPresentation) { + if (!!participant.video && ssrc.endpoint === participant.video.endpoint) { + hasVideo = true; + } + // console.log('has video = ', hasVideo, ' removed=', isVideoLeft); + ssrc.isRemoved = isVideoLeft; + } + + if (ssrc.isPresentation) { + if (!!participant.presentation && ssrc.endpoint === participant.presentation.endpoint) { + hasPresentation = true; + } + // console.log('has presentation, removed=', isPresentationLeft); + ssrc.isRemoved = isPresentationLeft; + } + } + }); + + if (!isAudioLeft && !hasAudio) { + // console.log('add audio'); + conference.ssrcs!.push({ + userId: participant.id, + isMain: false, + endpoint: `audio${participant.source}`, + isVideo: false, + sourceGroups: [{ + semantics: 'FID', + sources: [participant.source], + }], + }); + } + + if (!isVideoLeft && !hasVideo && participant.video) { + // console.log('add video', participant.video); + newEndpoints.push(participant.video.endpoint); + conference.ssrcs!.push({ + userId: participant.id, + isMain: false, + endpoint: participant.video.endpoint, + isVideo: true, + sourceGroups: participant.video.sourceGroups, + }); + } + + if (!isPresentationLeft && !hasPresentation && participant.presentation) { + // console.log('add presentation'); + conference.ssrcs!.push({ + isPresentation: true, + userId: participant.id, + isMain: false, + endpoint: participant.presentation.endpoint, + isVideo: true, + sourceGroups: participant.presentation.sourceGroups, + }); + } + }); + + if (state.updatingParticipantsQueue) { + state.updatingParticipantsQueue.push(conference); + return; + } else { + state.updatingParticipantsQueue = []; + } + + const sdp = buildSdp(conference as Conference); + console.log('build sdp!', sdp); + await connection.setRemoteDescription({ + type: 'offer', + sdp, + }); + + try { + const answer = await connection.createAnswer(); + await connection.setLocalDescription(answer); + + updateGroupCallStreams(myId); + if (state.updatingParticipantsQueue.length > 0) { + // eslint-disable-next-line no-restricted-syntax + for (const newConference of state.updatingParticipantsQueue) { + await connection.setRemoteDescription({ + type: 'offer', + sdp: buildSdp(newConference as Conference), + }); + const answerNew = await connection.createAnswer(); + await connection.setLocalDescription(answerNew); + updateGroupCallStreams(myId); + + // if (newEndpoints.length > 0) { + // sendDataChannelMessage({ + // colibriClass: 'ReceiverVideoConstraints', + // defaultConstraints: { + // maxHeight: 0, + // }, + // constraints: { + // ...(newEndpoints.reduce((acc: Record, el) => { + // acc[el] = { + // minHeight: 0, + // maxHeight: 1080, + // }; + // return acc; + // }, {})), + // }, + // onStageEndpoints: [], + // }); + // } + } + } + state.updatingParticipantsQueue = undefined; + } catch (e) { + console.error(e); + } +} +// +// function sendDataChannelMessage(message: ColibriClass) { +// if (!state || !state.dataChannel || state.dataChannel.readyState !== 'open') { +// return; +// } +// +// // console.log('SEND!', message); +// state.dataChannel.send(JSON.stringify(message)); +// } + +export async function handleUpdateGroupCallConnection(data: GroupCallConnectionData, isPresentation: boolean) { + if (!state) { + return; + } + + const conference = isPresentation ? state.screenshareConference : state.conference; + const connection = isPresentation ? state.screenshareConnection : state.connection; + + if (!conference || !connection || !conference.ssrcs) { + return; + } + + const sessionId = Date.now(); + const newConference = { + ...conference, + transport: data.transport, + sessionId, + audioExtensions: data.audio?.['rtp-hdrexts'], + audioPayloadTypes: data.audio?.['payload-types'], + videoExtensions: data.video?.['rtp-hdrexts'], + videoPayloadTypes: data.video?.['payload-types'], + } as Conference; + + state = { + ...state, + ...(!isPresentation ? { conference: newConference } : { screenshareConference: newConference }), + }; + + console.warn('update remote description', newConference, buildSdp(newConference, true, isPresentation)); + + try { + await connection.setRemoteDescription({ + type: 'answer', + sdp: buildSdp(newConference, true, isPresentation), + }); + + // state.resolveTest(); + // state.test = true; + } catch (e) { + console.error(e); + } +} + +function handleTrack(e: RTCTrackEvent) { + if (!state || !state.audioElement || !state.audioContext || !state.mediaStream) { + return; + } + const ssrc = state.conference?.ssrcs?.find((l) => l.endpoint === e.track.id); + if (!ssrc || !ssrc.userId) { + return; + } + + const { userId, isPresentation } = ssrc; + const participant = state.participants?.find((p) => p.id === userId); + + const streamType = (e.track.kind === 'video' ? (isPresentation ? 'presentation' : 'video') : 'audio') as StreamType; + + e.track.onended = () => { + delete state?.streams?.[userId][streamType]; + updateGroupCallStreams(userId); + }; + + const stream = e.streams[0]; + + if (e.track.kind === 'audio') { + const { mediaStream } = state; + const audioContext = new (window.AudioContext)(); + const source = audioContext.createMediaStreamSource(stream); + + const gainNode = audioContext.createGain(); + gainNode.gain.value = (participant?.volume || 10000) / 10000; + + const muteNode = audioContext.createGain(); + gainNode.gain.value = 1; + + const analyser = audioContext.createAnalyser(); + analyser.minDecibels = -100; + analyser.maxDecibels = -30; + analyser.smoothingTimeConstant = 0.05; + analyser.fftSize = 1024; + + source.connect(analyser).connect(muteNode).connect(gainNode).connect(audioContext.destination); + + mediaStream!.addTrack(source.mediaStream.getAudioTracks()[0]); + + // https://stackoverflow.com/questions/41784137/webrtc-doesnt-work-with-audiocontext#comment117600018_41784241 + const test = new Audio(); + test.srcObject = stream; + // test.srcObject = source.mediaStream; + test.muted = true; + test.remove(); + + state = { + ...state, + participantFunctions: { + ...state.participantFunctions, + [userId]: { + ...state.participantFunctions?.[userId], + setVolume: (volume: number) => { + gainNode.gain.value = volume > 1 ? volume * 2 : volume; + }, + toggleMute: (muted?: boolean) => { + muteNode.gain.value = muted ? 0 : 1; + }, + getCurrentAmplitude: () => { + const array = new Uint8Array(analyser.frequencyBinCount); + analyser.getByteFrequencyData(array); + return getAmplitude(array, 1.5); + }, + }, + }, + }; + } + + state = { + ...state, + streams: { + ...state.streams, + [userId]: { + ...state.streams?.[userId], + [streamType]: stream, + }, + }, + }; + + updateGroupCallStreams(userId); +} + +function initializeConnection( + streams: MediaStream[], + resolve: (payload: JoinGroupCallPayload) => void, + isPresentation = false, +) { + const connection = new RTCPeerConnection(); + + const dataChannel = isPresentation ? undefined : createDataChannel(connection); + + streams.forEach((stream) => stream.getTracks().forEach((track) => { + connection.addTrack(track, stream); + })); + + if (!isPresentation) { + connection.oniceconnectionstatechange = () => { + const connectionState = connection.iceConnectionState; + console.log('ice', connectionState); + if (connectionState === 'connected' || connectionState === 'completed') { + updateConnectionState('connected'); + } else if (connectionState === 'checking' || connectionState === 'new') { + updateConnectionState('connecting'); + } else if (connection.iceConnectionState === 'disconnected') { + updateConnectionState('reconnecting'); + } + }; + } + connection.ontrack = handleTrack; + connection.onnegotiationneeded = async () => { + if (!state) return; + + const { myId } = state; + + if (!myId) { + return; + } + const offer = await connection.createOffer({ + offerToReceiveVideo: true, + offerToReceiveAudio: !isPresentation, + }); + + console.log('created offer!', offer); + + await connection.setLocalDescription(offer); + + if (!offer.sdp) { + return; + } + + const sdp = parseSdp(offer); + console.log('parsed sdp', sdp); + const audioSsrc: Ssrc | undefined = !isPresentation ? { + userId: '', + sourceGroups: [ + { + semantics: 'FID', + sources: [sdp.ssrc || 0], + }, + ], + isRemoved: isPresentation, + isMain: true, + isVideo: false, + isPresentation, + endpoint: isPresentation ? '1' : '0', + } : undefined; + + const videoSsrc: Ssrc | undefined = sdp['ssrc-groups'] && { + isPresentation, + userId: '', + sourceGroups: sdp['ssrc-groups'], + isMain: true, + isVideo: true, + endpoint: isPresentation ? '0' : '1', + }; + + const conference = isPresentation ? state.screenshareConference : state.conference; + + const ssrcs: Ssrc[] = []; + if (isPresentation) { + if (videoSsrc) ssrcs.push(videoSsrc); + if (audioSsrc) ssrcs.push(audioSsrc); + } else { + if (audioSsrc) ssrcs.push(audioSsrc); + if (videoSsrc) ssrcs.push(videoSsrc); + } + + const audioStream = streams.find((l) => l.getTracks()[0].kind === 'audio'); + const videoStream = streams.find((l) => l.getTracks()[0].kind === 'video'); + + state = { + ...state, + ...(!isPresentation ? { + conference: { + ...conference, + ssrcs, + }, + } : { + screenshareConference: { + ...conference, + ssrcs, + }, + }), + streams: { + ...state.streams, + [myId]: { + ...state.streams?.[myId], + ...(audioStream && { audio: audioStream }), + ...(!isPresentation && videoStream ? { video: videoStream } : { presentation: videoStream }), + }, + }, + }; + + updateGroupCallStreams(myId); + + resolve(sdp); + }; + + return { connection, dataChannel }; +} + +export async function startSharingScreen(): Promise { + if (!state) { + return undefined; + } + + try { + const stream: MediaStream | undefined = await getUserStream('presentation'); + + if (!stream) { + return undefined; + } + + stream.getTracks()[0].onended = () => { + if (state && state.myId) { + delete state.streams?.[state.myId].presentation; + updateGroupCallStreams(state.myId); + leavePresentation(); + } + }; + + return await new Promise((resolve) => { + const { connection, dataChannel } = initializeConnection([stream], resolve, true); + state = { + ...state, + screenshareConnection: connection, + screenshareDataChannel: dataChannel, + }; + }); + } catch (e) { + return undefined; + } +} + +export function joinGroupCall( + myId: string, + audioContext: AudioContext, + audioElement: HTMLAudioElement, + onUpdate: (...args: any[]) => void, +): Promise { + if (state) { + throw Error('Already in call'); + } + + updateConnectionState('connecting'); + + const mediaStream = new MediaStream(); + audioElement.srcObject = mediaStream; + audioElement.play().catch((l) => console.warn(l)); + + state = { + onUpdate, + participants: [], + myId, + speaking: {}, + silence: silence(audioContext), + black: black({ width: 640, height: 480 }), + // @ts-ignore + analyserInterval: setInterval(analyzeAmplitudes, 1000), + audioElement, + // destination, + audioContext, + mediaStream, + }; + + return new Promise((resolve) => { + state = { + ...state, + ...initializeConnection([state!.silence!, state!.black!], resolve), + }; + }); +} diff --git a/src/lib/secret-sauce/types.d.ts b/src/lib/secret-sauce/types.d.ts deleted file mode 100644 index 88f5af42f..000000000 --- a/src/lib/secret-sauce/types.d.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { P2PPayloadType } from './p2pMessage'; -export interface GroupCallParticipant { - isSelf?: boolean; - isMuted?: boolean; - isLeft?: boolean; - isUser?: boolean; - canSelfUnmute?: boolean; - hasJustJoined?: boolean; - isVideoJoined?: boolean; - isMutedByMe?: boolean; - isVolumeByAdmin?: boolean; - isMin?: boolean; - isVersioned?: boolean; - source: number; - date: Date; - id: string; - volume?: number; - about?: string; - video?: GroupCallParticipantVideo; - presentation?: GroupCallParticipantVideo; - raiseHandRating?: string; - hasAudioStream?: boolean; - hasVideoStream?: boolean; - hasPresentationStream?: boolean; - amplitude?: number; -} -export declare type GroupCallConnectionState = 'connecting' | 'connected' | 'disconnected' | 'reconnecting' | 'discarded'; -export interface GroupCallParticipantVideo { - endpoint: string; - isPaused?: boolean; - sourceGroups: SsrcGroup[]; - audioSource?: number; -} -export declare type Fingerprint = { - hash: string; - setup: string; - fingerprint: string; -}; -export declare type SsrcGroup = { - semantics: string; - sources: number[]; -}; -export declare type Candidate = { - generation: string; - component: string; - protocol: string; - port: string; - ip: string; - foundation: string; - id: string; - priority: string; - type: string; - network: string; - 'rel-addr': string; - 'rel-port': string; - sdpString?: string; -}; -export declare type JoinGroupCallPayload = { - ufrag: string; - pwd: string; - fingerprints: Fingerprint[]; - ssrc?: number; - 'ssrc-groups'?: SsrcGroup[]; -}; -export declare type P2pParsedSdp = JoinGroupCallPayload & { - audioExtmap: RTPExtension[]; - videoExtmap: RTPExtension[]; - screencastExtmap: RTPExtension[]; - audioPayloadTypes: P2PPayloadType[]; - videoPayloadTypes: P2PPayloadType[]; - screencastPayloadTypes: P2PPayloadType[]; -}; -export interface RTPExtension { - id: number; - uri: string; -} -export interface RTCPFeedbackParam { - type: string; - subtype?: string; -} -export interface PayloadType { - id: number; - name: string; - clockrate: number; - channels: number; - parameters?: Record; - 'rtcp-fbs'?: RTCPFeedbackParam[]; -} -export interface GroupCallTransport { - candidates: Candidate[]; - pwd: string; - ufrag: string; - fingerprints: Fingerprint[]; - 'rtcp-mux': boolean; - xmlns: string; -} -export interface GroupCallConnectionData { - transport: GroupCallTransport; - audio: { - 'payload-types': PayloadType[]; - 'rtp-hdrexts': RTPExtension[]; - }; - video: { - endpoint: string; - 'payload-types': PayloadType[]; - 'rtp-hdrexts': RTPExtension[]; - server_sources: number[]; - }; - stream?: boolean; -} -export interface ApiPhoneCallConnection { - username: string; - password: string; - isTurn?: boolean; - isStun?: boolean; - ip: string; - ipv6: string; - port: number; -} -export interface ApiCallProtocol { - libraryVersions: string[]; - minLayer: number; - maxLayer: number; - isUdpP2p?: boolean; - isUdpReflector?: boolean; -} diff --git a/src/lib/secret-sauce/types.ts b/src/lib/secret-sauce/types.ts new file mode 100644 index 000000000..e5389a448 --- /dev/null +++ b/src/lib/secret-sauce/types.ts @@ -0,0 +1,143 @@ +import { P2PPayloadType } from './p2pMessage'; + +export interface GroupCallParticipant { + isSelf?: boolean; + isMuted?: boolean; + isLeft?: boolean; + isUser?: boolean; + canSelfUnmute?: boolean; + hasJustJoined?: boolean; + isVideoJoined?: boolean; + isMutedByMe?: boolean; + isVolumeByAdmin?: boolean; + isMin?: boolean; + isVersioned?: boolean; + source: number; + date: Date; + id: string; + volume?: number; + about?: string; + video?: GroupCallParticipantVideo; + presentation?: GroupCallParticipantVideo; + raiseHandRating?: string; + + hasAudioStream?: boolean; + hasVideoStream?: boolean; + hasPresentationStream?: boolean; + amplitude?: number; +} + +export type GroupCallConnectionState = 'connecting' | 'connected' | 'disconnected' | 'reconnecting' | 'discarded'; + +export interface GroupCallParticipantVideo { + endpoint: string; + isPaused?: boolean; + sourceGroups: SsrcGroup[]; + audioSource?: number; +} + +export type Fingerprint = { + hash: string; + setup: string; + fingerprint: string; +}; + +export type SsrcGroup = { + semantics: string; + sources: number[]; +}; + +export type Candidate = { + generation: string; + component: string; + protocol: string; + port: string; + ip: string; + foundation: string; + id: string; + priority: string; + type: string; + network: string; + 'rel-addr': string; + 'rel-port': string; + + sdpString?: string; // Used for P2P +}; + +export type JoinGroupCallPayload = { + ufrag: string; + pwd: string; + fingerprints: Fingerprint[]; + ssrc?: number; + 'ssrc-groups'?: SsrcGroup[]; +}; + +export type P2pParsedSdp = JoinGroupCallPayload & { + audioExtmap: RTPExtension[]; + videoExtmap: RTPExtension[]; + screencastExtmap: RTPExtension[]; + audioPayloadTypes: P2PPayloadType[]; + videoPayloadTypes: P2PPayloadType[]; + screencastPayloadTypes: P2PPayloadType[]; +}; + +export interface RTPExtension { + id: number; + uri: string; +} + +export interface RTCPFeedbackParam { + type: string; + subtype?: string; +} + +export interface PayloadType { + id: number; + name: string; + clockrate: number; + channels: number; + parameters?: Record; + 'rtcp-fbs'?: RTCPFeedbackParam[]; +} + +export interface GroupCallTransport { + candidates: Candidate[]; + pwd: string; + ufrag: string; + fingerprints: Fingerprint[]; + 'rtcp-mux': boolean; + xmlns: string; +} + +export interface GroupCallConnectionData { + transport: GroupCallTransport; + audio: { + 'payload-types': PayloadType[]; + 'rtp-hdrexts': RTPExtension[]; + }; + video: { + endpoint: string; + 'payload-types': PayloadType[]; + 'rtp-hdrexts': RTPExtension[]; + server_sources: number[]; + }; + stream?: boolean; +} + +export interface ApiPhoneCallConnection { + username: string; + password: string; + isTurn?: boolean; + isStun?: boolean; + ip: string; + ipv6: string; + port: number; +} + +export interface ApiCallProtocol { + libraryVersions: string[]; + minLayer: number; + maxLayer: number; + isUdpP2p?: boolean; + isUdpReflector?: boolean; +} diff --git a/src/lib/secret-sauce/utils.d.ts b/src/lib/secret-sauce/utils.d.ts deleted file mode 100644 index 31601f9f8..000000000 --- a/src/lib/secret-sauce/utils.d.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { P2PPayloadType } from './p2pMessage'; -import type { PayloadType } from './types'; -export declare function toTelegramSource(source: number): number; -export declare function fromTelegramSource(source: number): number; -export declare function getAmplitude(array: Uint8Array, scale?: number): number; -export declare function p2pPayloadTypeToConference(p: P2PPayloadType): PayloadType; -export declare const THRESHOLD = 0.1; -export declare const IS_SCREENSHARE_SUPPORTED: boolean; -export declare const IS_ECHO_CANCELLATION_SUPPORTED: boolean | undefined; -export declare const IS_NOISE_SUPPRESSION_SUPPORTED: any; diff --git a/src/lib/secret-sauce/utils.ts b/src/lib/secret-sauce/utils.ts new file mode 100644 index 000000000..63c73b900 --- /dev/null +++ b/src/lib/secret-sauce/utils.ts @@ -0,0 +1,47 @@ +import type { P2PPayloadType } from './p2pMessage'; +import type { PayloadType } from './types'; + +/// NOTE: telegram returns sign source, while webrtc uses unsign source internally +/// unsign => sign +export function toTelegramSource(source: number) { + // eslint-disable-next-line no-bitwise + return source << 0; +} + +/// NOTE: telegram returns sign source, while webrtc uses unsign source internally +/// sign => unsign +export function fromTelegramSource(source: number) { + // eslint-disable-next-line no-bitwise + return source >>> 0; +} + +export function getAmplitude(array: Uint8Array, scale = 3) { + if (!array) return 0; + + const { length } = array; + let total = 0; + for (let i = 0; i < length; i++) { + total += array[i] * array[i]; + } + const rms = Math.sqrt(total / length) / 255; + + return Math.min(1, rms * scale); +} + +export function p2pPayloadTypeToConference(p: P2PPayloadType): PayloadType { + return { + id: p.id, + name: p.name, + 'rtcp-fbs': p.feedbackTypes, + clockrate: p.clockrate, + parameters: p.parameters, + channels: p.channels, + }; +} + +export const THRESHOLD = 0.1; + +export const IS_SCREENSHARE_SUPPORTED = 'getDisplayMedia' in (navigator?.mediaDevices || {}); +export const IS_ECHO_CANCELLATION_SUPPORTED = navigator?.mediaDevices?.getSupportedConstraints().echoCancellation; +// @ts-ignore +export const IS_NOISE_SUPPRESSION_SUPPORTED = navigator?.mediaDevices?.getSupportedConstraints().noiseSuppression;