[Dev] Calls: Move secret-sauce to repo (#2511)
This commit is contained in:
Alexander Zinchuk
parent
eed6241f42
commit
28dc43ab4a
5
src/lib/secret-sauce/blacksilence.d.ts
vendored
5
src/lib/secret-sauce/blacksilence.d.ts
vendored
@ -1,5 +0,0 @@
|
||||
export declare const silence: (ctx: AudioContext) => MediaStream;
|
||||
export declare const black: ({ width, height }?: {
|
||||
width?: number | undefined;
|
||||
height?: number | undefined;
|
||||
}) => MediaStream;
|
||||
18
src/lib/secret-sauce/blacksilence.ts
Normal file
18
src/lib/secret-sauce/blacksilence.ts
Normal file
@ -0,0 +1,18 @@
|
||||
// https://blog.mozilla.org/webrtc/warm-up-with-replacetrack/
|
||||
|
||||
export const silence = (ctx: AudioContext) => {
|
||||
const oscillator = ctx.createOscillator();
|
||||
const dst = oscillator.connect(ctx.createMediaStreamDestination());
|
||||
oscillator.start();
|
||||
return new MediaStream([Object.assign((dst as any).stream.getAudioTracks()[0], { enabled: false })]);
|
||||
};
|
||||
|
||||
export const black = ({ width = 640, height = 480 } = {}) => {
|
||||
const canvas = Object.assign(document.createElement('canvas'), { width, height });
|
||||
const ctx = canvas.getContext('2d');
|
||||
if (!ctx) throw Error('Cannot create canvas ctx');
|
||||
|
||||
ctx.fillRect(0, 0, width, height);
|
||||
const stream = canvas.captureStream();
|
||||
return new MediaStream([Object.assign(stream.getVideoTracks()[0], { enabled: false })]);
|
||||
};
|
||||
21
src/lib/secret-sauce/buildSdp.d.ts
vendored
21
src/lib/secret-sauce/buildSdp.d.ts
vendored
@ -1,21 +0,0 @@
|
||||
import type { GroupCallTransport, PayloadType, RTPExtension, SsrcGroup } from './types';
|
||||
export declare type Conference = {
|
||||
sessionId: number;
|
||||
audioExtensions: RTPExtension[];
|
||||
videoExtensions: RTPExtension[];
|
||||
audioPayloadTypes: PayloadType[];
|
||||
videoPayloadTypes: PayloadType[];
|
||||
ssrcs: Ssrc[];
|
||||
transport: GroupCallTransport;
|
||||
};
|
||||
export declare type Ssrc = {
|
||||
userId: string;
|
||||
endpoint: string;
|
||||
isMain: boolean;
|
||||
isRemoved?: boolean;
|
||||
isVideo: boolean;
|
||||
isPresentation?: boolean;
|
||||
sourceGroups: SsrcGroup[];
|
||||
};
|
||||
declare const _default: (conference: Conference, isAnswer?: boolean, isPresentation?: boolean, isP2p?: boolean) => string;
|
||||
export default _default;
|
||||
182
src/lib/secret-sauce/buildSdp.ts
Normal file
182
src/lib/secret-sauce/buildSdp.ts
Normal file
@ -0,0 +1,182 @@
|
||||
import type {
|
||||
Candidate, GroupCallTransport, PayloadType, RTPExtension, SsrcGroup,
|
||||
} from './types';
|
||||
import { fromTelegramSource } from './utils';
|
||||
|
||||
export type Conference = {
|
||||
sessionId: number;
|
||||
audioExtensions: RTPExtension[];
|
||||
videoExtensions: RTPExtension[];
|
||||
audioPayloadTypes: PayloadType[];
|
||||
videoPayloadTypes: PayloadType[];
|
||||
ssrcs: Ssrc[];
|
||||
transport: GroupCallTransport;
|
||||
};
|
||||
|
||||
export type Ssrc = {
|
||||
userId: string;
|
||||
endpoint: string;
|
||||
isMain: boolean;
|
||||
isRemoved?: boolean;
|
||||
isVideo: boolean;
|
||||
isPresentation?: boolean;
|
||||
sourceGroups: SsrcGroup[];
|
||||
};
|
||||
|
||||
export default (conference: Conference, isAnswer = false, isPresentation = false, isP2p = false) => {
|
||||
const lines: string[] = [];
|
||||
|
||||
const add = (value: string) => {
|
||||
lines.push(value);
|
||||
};
|
||||
|
||||
const {
|
||||
sessionId,
|
||||
ssrcs,
|
||||
audioExtensions,
|
||||
videoExtensions,
|
||||
audioPayloadTypes,
|
||||
videoPayloadTypes,
|
||||
transport: {
|
||||
ufrag,
|
||||
pwd,
|
||||
fingerprints,
|
||||
candidates,
|
||||
},
|
||||
} = conference;
|
||||
|
||||
// Header
|
||||
add('v=0'); // version
|
||||
add(`o=- ${sessionId} 2 IN IP4 0.0.0.0`); // sessionId, 2=sessionVersion
|
||||
add('s=-'); // name of the session
|
||||
add('t=0 0'); // time when session is valid
|
||||
add('a=ice-options:trickle');
|
||||
add('a=msid-semantic:WMS *');
|
||||
add(`a=group:BUNDLE ${ssrcs.map((ssrc) => ssrc.endpoint).join(' ')}${isPresentation ? '' : ` ${isP2p ? '3' : '2'}`}`);
|
||||
// ice-lite: is a minimal version of the ICE specification, intended only for servers running on a public IP address
|
||||
if (!isP2p) add('a=ice-lite');
|
||||
|
||||
const addCandidate = (c: Candidate) => {
|
||||
if (c.sdpString) {
|
||||
add(`a=${c.sdpString}`);
|
||||
} else {
|
||||
let str = '';
|
||||
str += 'a=candidate:';
|
||||
str += `${c.foundation} ${c.component} ${c.protocol} ${c.priority} ${c.ip} ${c.port} typ ${c.type}`;
|
||||
if ('rel-addr' in c) {
|
||||
str += ` raddr ${c['rel-addr']} rport ${c['rel-port']}`;
|
||||
}
|
||||
str += ` generation ${c.generation}`;
|
||||
add(str);
|
||||
}
|
||||
};
|
||||
|
||||
const addTransport = () => {
|
||||
add(`a=ice-ufrag:${ufrag}`);
|
||||
add(`a=ice-pwd:${pwd}`);
|
||||
fingerprints.forEach((fingerprint) => {
|
||||
add(`a=fingerprint:${fingerprint.hash} ${fingerprint.fingerprint}`);
|
||||
add(`a=setup:${isP2p ? (fingerprint.setup) : 'passive'}`);
|
||||
});
|
||||
|
||||
candidates.forEach(addCandidate);
|
||||
};
|
||||
|
||||
const addPayloadType = (payloadType: PayloadType) => {
|
||||
const {
|
||||
channels, id, name, clockrate, parameters,
|
||||
} = payloadType;
|
||||
|
||||
const channelsString = channels ? `/${channels}` : '';
|
||||
add(`a=rtpmap:${id} ${name}/${clockrate}${channelsString}`);
|
||||
|
||||
if (parameters) {
|
||||
const parametersString = Object.keys(parameters).map((key) => {
|
||||
return `${key}=${parameters![key]};`;
|
||||
}).join(' ');
|
||||
|
||||
add(`a=fmtp:${id} ${parametersString}`);
|
||||
}
|
||||
|
||||
payloadType['rtcp-fbs']?.forEach((fbParam) => {
|
||||
add(`a=rtcp-fb:${id} ${fbParam.type}${fbParam.subtype ? ` ${fbParam.subtype}` : ''}`);
|
||||
});
|
||||
};
|
||||
|
||||
const addSsrcEntry = (entry: Ssrc) => {
|
||||
const payloadTypes = entry.isVideo ? videoPayloadTypes : audioPayloadTypes;
|
||||
|
||||
const type = entry.isVideo ? 'video' : 'audio';
|
||||
add(`m=${type} ${entry.isMain ? 1 : 0} RTP/SAVPF ${payloadTypes.map((l) => l.id).join(' ')}`);
|
||||
add('c=IN IP4 0.0.0.0');
|
||||
add('b=AS:1300'); // 1300000 / 1000
|
||||
add(`a=mid:${entry.endpoint}`);
|
||||
add('a=rtcp-mux');
|
||||
payloadTypes.forEach(addPayloadType);
|
||||
|
||||
add('a=rtcp:1 IN IP4 0.0.0.0');
|
||||
if (entry.isVideo) {
|
||||
add('a=rtcp-rsize');
|
||||
}
|
||||
|
||||
(entry.isVideo ? videoExtensions : audioExtensions).forEach(({ id, uri }) => {
|
||||
add(`a=extmap:${id} ${uri}`);
|
||||
});
|
||||
|
||||
if (entry.isRemoved) {
|
||||
add('a=inactive');
|
||||
return;
|
||||
}
|
||||
|
||||
addTransport();
|
||||
|
||||
if (isP2p) {
|
||||
add('a=sendrecv');
|
||||
add('a=bundle-only');
|
||||
} else {
|
||||
if (isAnswer) {
|
||||
add('a=recvonly');
|
||||
return;
|
||||
}
|
||||
if (entry.isMain) {
|
||||
add('a=sendrecv');
|
||||
} else {
|
||||
add('a=sendonly');
|
||||
add('a=bundle-only');
|
||||
}
|
||||
}
|
||||
|
||||
entry.sourceGroups.forEach((sourceGroup) => {
|
||||
add(`a=ssrc-group:${sourceGroup.semantics} ${sourceGroup.sources.map(fromTelegramSource).join(' ')}`);
|
||||
sourceGroup.sources.forEach((ssrcTelegram) => {
|
||||
const ssrc = fromTelegramSource(ssrcTelegram);
|
||||
add(`a=ssrc:${ssrc} cname:${entry.endpoint}`);
|
||||
add(`a=ssrc:${ssrc} msid:${entry.endpoint} ${entry.endpoint}`);
|
||||
add(`a=ssrc:${ssrc} mslabel:${entry.endpoint}`);
|
||||
add(`a=ssrc:${ssrc} label:${entry.endpoint}`);
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
if (!isP2p) {
|
||||
ssrcs.filter((ssrc) => ssrc.endpoint === '0' || ssrc.endpoint === '1').map(addSsrcEntry);
|
||||
} else {
|
||||
ssrcs.filter(addSsrcEntry);
|
||||
}
|
||||
|
||||
if (!isPresentation) {
|
||||
add('m=application 1 UDP/DTLS/SCTP webrtc-datachannel');
|
||||
add('c=IN IP4 0.0.0.0');
|
||||
addTransport();
|
||||
add('a=ice-options:trickle');
|
||||
add(`a=mid:${isP2p ? '3' : (isPresentation ? '1' : '2')}`);
|
||||
add('a=sctp-port:5000');
|
||||
add('a=max-message-size:262144');
|
||||
}
|
||||
|
||||
if (!isP2p) {
|
||||
ssrcs.filter((ssrc) => ssrc.endpoint !== '0' && ssrc.endpoint !== '1').map(addSsrcEntry);
|
||||
}
|
||||
|
||||
return `${lines.join('\n')}\n`;
|
||||
};
|
||||
36
src/lib/secret-sauce/colibriClass.d.ts
vendored
36
src/lib/secret-sauce/colibriClass.d.ts
vendored
@ -1,36 +0,0 @@
|
||||
export declare type EndpointConnectivityStatusChangeEvent = {
|
||||
colibriClass: 'EndpointConnectivityStatusChangeEvent';
|
||||
endpoint: string;
|
||||
active: boolean;
|
||||
};
|
||||
export declare type DominantSpeakerEndpointChangeEvent = {
|
||||
colibriClass: 'DominantSpeakerEndpointChangeEvent';
|
||||
dominantSpeakerEndpoint: string;
|
||||
previousSpeakers: string[];
|
||||
};
|
||||
export declare type SenderVideoConstraints = {
|
||||
colibriClass: 'SenderVideoConstraints';
|
||||
videoConstraints: {
|
||||
idealHeight: number;
|
||||
};
|
||||
};
|
||||
export declare type DebugMessage = {
|
||||
colibriClass: 'DebugMessage';
|
||||
message: string;
|
||||
};
|
||||
export declare type LastNEndpointsChangeEvent = {
|
||||
colibriClass: 'LastNEndpointsChangeEvent';
|
||||
lastNEndpoints: string[];
|
||||
};
|
||||
export declare type ReceiverVideoConstraints = {
|
||||
colibriClass: 'ReceiverVideoConstraints';
|
||||
defaultConstraints: {
|
||||
maxHeight: number;
|
||||
};
|
||||
constraints: Record<string, {
|
||||
minHeight: number;
|
||||
maxHeight: number;
|
||||
}>;
|
||||
onStageEndpoints: string[];
|
||||
};
|
||||
export declare type ColibriClass = (LastNEndpointsChangeEvent | DebugMessage | EndpointConnectivityStatusChangeEvent | SenderVideoConstraints | DominantSpeakerEndpointChangeEvent | ReceiverVideoConstraints);
|
||||
45
src/lib/secret-sauce/colibriClass.ts
Normal file
45
src/lib/secret-sauce/colibriClass.ts
Normal file
@ -0,0 +1,45 @@
|
||||
export type EndpointConnectivityStatusChangeEvent = {
|
||||
colibriClass: 'EndpointConnectivityStatusChangeEvent';
|
||||
endpoint: string;
|
||||
active: boolean;
|
||||
};
|
||||
|
||||
export type DominantSpeakerEndpointChangeEvent = {
|
||||
colibriClass: 'DominantSpeakerEndpointChangeEvent';
|
||||
dominantSpeakerEndpoint: string;
|
||||
previousSpeakers: string[];
|
||||
};
|
||||
|
||||
export type SenderVideoConstraints = {
|
||||
colibriClass: 'SenderVideoConstraints';
|
||||
videoConstraints: {
|
||||
idealHeight: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type DebugMessage = {
|
||||
colibriClass: 'DebugMessage';
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type LastNEndpointsChangeEvent = {
|
||||
colibriClass: 'LastNEndpointsChangeEvent';
|
||||
lastNEndpoints: string[];
|
||||
};
|
||||
|
||||
export type ReceiverVideoConstraints = {
|
||||
colibriClass: 'ReceiverVideoConstraints';
|
||||
defaultConstraints: {
|
||||
maxHeight: number;
|
||||
};
|
||||
constraints: Record<string, {
|
||||
minHeight: number;
|
||||
maxHeight: number;
|
||||
}>;
|
||||
onStageEndpoints: string[];
|
||||
};
|
||||
|
||||
export type ColibriClass = (
|
||||
LastNEndpointsChangeEvent | DebugMessage | EndpointConnectivityStatusChangeEvent |
|
||||
SenderVideoConstraints | DominantSpeakerEndpointChangeEvent | ReceiverVideoConstraints
|
||||
);
|
||||
5
src/lib/secret-sauce/index.d.ts
vendored
5
src/lib/secret-sauce/index.d.ts
vendored
@ -1,5 +0,0 @@
|
||||
export { handleUpdateGroupCallConnection, startSharingScreen, joinGroupCall, getDevices, getUserStreams, setVolume, isStreamEnabled, toggleStream, leaveGroupCall, handleUpdateGroupCallParticipants, switchCameraInput, toggleSpeaker, toggleNoiseSuppression, } from './secretsauce';
|
||||
export { joinPhoneCall, processSignalingMessage, getStreams, toggleStreamP2p, stopPhoneCall, switchCameraInputP2p, } from './p2p';
|
||||
export * from './p2pMessage';
|
||||
export { IS_SCREENSHARE_SUPPORTED, THRESHOLD, } from './utils';
|
||||
export * from './types';
|
||||
File diff suppressed because one or more lines are too long
@ -1,39 +0,0 @@
|
||||
/*! ./blacksilence */
|
||||
|
||||
/*! ./buildSdp */
|
||||
|
||||
/*! ./parseSdp */
|
||||
|
||||
/*! ./secretsauce */
|
||||
|
||||
/*! ./types */
|
||||
|
||||
/*! ./utils */
|
||||
|
||||
/*!**********************!*\
|
||||
!*** ./src/index.ts ***!
|
||||
\**********************/
|
||||
|
||||
/*!**********************!*\
|
||||
!*** ./src/types.ts ***!
|
||||
\**********************/
|
||||
|
||||
/*!**********************!*\
|
||||
!*** ./src/utils.ts ***!
|
||||
\**********************/
|
||||
|
||||
/*!*************************!*\
|
||||
!*** ./src/buildSdp.ts ***!
|
||||
\*************************/
|
||||
|
||||
/*!*************************!*\
|
||||
!*** ./src/parseSdp.ts ***!
|
||||
\*************************/
|
||||
|
||||
/*!****************************!*\
|
||||
!*** ./src/secretsauce.ts ***!
|
||||
\****************************/
|
||||
|
||||
/*!*****************************!*\
|
||||
!*** ./src/blacksilence.ts ***!
|
||||
\*****************************/
|
||||
14
src/lib/secret-sauce/index.ts
Normal file
14
src/lib/secret-sauce/index.ts
Normal file
@ -0,0 +1,14 @@
|
||||
export {
|
||||
handleUpdateGroupCallConnection, startSharingScreen, joinGroupCall,
|
||||
getDevices, getUserStreams, setVolume, isStreamEnabled, toggleStream,
|
||||
leaveGroupCall, handleUpdateGroupCallParticipants, switchCameraInput,
|
||||
toggleSpeaker, toggleNoiseSuppression,
|
||||
} from './secretsauce';
|
||||
export {
|
||||
joinPhoneCall, processSignalingMessage, getStreams, toggleStreamP2p, stopPhoneCall, switchCameraInputP2p,
|
||||
} from './p2p';
|
||||
export * from './p2pMessage';
|
||||
export {
|
||||
IS_SCREENSHARE_SUPPORTED, THRESHOLD,
|
||||
} from './utils';
|
||||
export * from './types';
|
||||
16
src/lib/secret-sauce/p2p.d.ts
vendored
16
src/lib/secret-sauce/p2p.d.ts
vendored
@ -1,16 +0,0 @@
|
||||
import type { ApiPhoneCallConnection } from './types';
|
||||
import { P2pMessage } from './p2pMessage';
|
||||
import { StreamType } from './secretsauce';
|
||||
export declare function getStreams(): {
|
||||
video?: MediaStream | undefined;
|
||||
audio?: MediaStream | undefined;
|
||||
presentation?: MediaStream | undefined;
|
||||
ownAudio?: MediaStream | undefined;
|
||||
ownVideo?: MediaStream | undefined;
|
||||
ownPresentation?: MediaStream | undefined;
|
||||
} | undefined;
|
||||
export declare function switchCameraInputP2p(): Promise<void>;
|
||||
export declare function toggleStreamP2p(streamType: StreamType, value?: boolean | undefined): Promise<void>;
|
||||
export declare function joinPhoneCall(connections: ApiPhoneCallConnection[], emitSignalingData: (data: P2pMessage) => void, isOutgoing: boolean, shouldStartVideo: boolean, onUpdate: (...args: any[]) => void): Promise<void>;
|
||||
export declare function stopPhoneCall(): void;
|
||||
export declare function processSignalingMessage(message: P2pMessage): Promise<void>;
|
||||
478
src/lib/secret-sauce/p2p.ts
Normal file
478
src/lib/secret-sauce/p2p.ts
Normal file
@ -0,0 +1,478 @@
|
||||
import { black, silence } from './blacksilence';
|
||||
import type { ApiPhoneCallConnection, P2pParsedSdp } from './types';
|
||||
import parseSdp from './parseSdp';
|
||||
import type { MediaContent, MediaStateMessage, P2pMessage } from './p2pMessage';
|
||||
import {
|
||||
fromTelegramSource,
|
||||
IS_ECHO_CANCELLATION_SUPPORTED,
|
||||
IS_NOISE_SUPPRESSION_SUPPORTED,
|
||||
p2pPayloadTypeToConference,
|
||||
} from './utils';
|
||||
import buildSdp, { Conference } from './buildSdp';
|
||||
import { getUserStreams, StreamType } from './secretsauce';
|
||||
|
||||
type P2pState = {
|
||||
connection: RTCPeerConnection;
|
||||
dataChannel: RTCDataChannel;
|
||||
emitSignalingData: (data: P2pMessage) => void;
|
||||
onUpdate: (...args: any[]) => void;
|
||||
conference?: Partial<Conference>;
|
||||
isOutgoing: boolean;
|
||||
candidates: string[];
|
||||
streams: {
|
||||
video?: MediaStream;
|
||||
audio?: MediaStream;
|
||||
presentation?: MediaStream;
|
||||
ownAudio?: MediaStream;
|
||||
ownVideo?: MediaStream;
|
||||
ownPresentation?: MediaStream;
|
||||
};
|
||||
silence: MediaStream;
|
||||
blackVideo: MediaStream;
|
||||
blackPresentation: MediaStream;
|
||||
mediaState: Omit<MediaStateMessage, '@type'>;
|
||||
audio: HTMLAudioElement;
|
||||
gotInitialSetup?: boolean;
|
||||
facingMode?: VideoFacingModeEnum;
|
||||
};
|
||||
|
||||
let state: P2pState | undefined;
|
||||
|
||||
export function getStreams() {
|
||||
return state?.streams;
|
||||
}
|
||||
|
||||
function updateStreams() {
|
||||
state?.onUpdate({
|
||||
...state.mediaState,
|
||||
'@type': 'updatePhoneCallMediaState',
|
||||
});
|
||||
}
|
||||
|
||||
function getUserStream(streamType: StreamType, facing: VideoFacingModeEnum = 'user') {
|
||||
if (streamType === 'presentation') {
|
||||
return (navigator.mediaDevices as any).getDisplayMedia({
|
||||
audio: false,
|
||||
video: true,
|
||||
});
|
||||
}
|
||||
|
||||
return navigator.mediaDevices.getUserMedia({
|
||||
audio: streamType === 'audio' ? {
|
||||
// @ts-ignore
|
||||
...(IS_ECHO_CANCELLATION_SUPPORTED && { echoCancellation: true }),
|
||||
...(IS_NOISE_SUPPRESSION_SUPPORTED && { noiseSuppression: true }),
|
||||
} : false,
|
||||
video: streamType === 'video' ? {
|
||||
facingMode: facing,
|
||||
} : false,
|
||||
});
|
||||
}
|
||||
|
||||
export async function switchCameraInputP2p() {
|
||||
if (!state || !state.facingMode) {
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = state.streams.ownVideo;
|
||||
|
||||
if (!stream) return;
|
||||
|
||||
const track = stream.getTracks()[0];
|
||||
|
||||
if (!track) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sender = state.connection.getSenders().find((l) => track.id === l.track?.id);
|
||||
|
||||
if (!sender) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.facingMode = state.facingMode === 'environment' ? 'user' : 'environment';
|
||||
try {
|
||||
const newStream = await getUserStream('video', state.facingMode);
|
||||
|
||||
await sender.replaceTrack(newStream.getTracks()[0]);
|
||||
state.streams.ownVideo = newStream;
|
||||
updateStreams();
|
||||
} catch (e) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
export async function toggleStreamP2p(streamType: StreamType, value: boolean | undefined = undefined) {
|
||||
if (!state) return;
|
||||
const stream = streamType === 'audio' ? state.streams.ownAudio
|
||||
: (streamType === 'video' ? state.streams.ownVideo : state.streams.ownPresentation);
|
||||
|
||||
if (!stream) return;
|
||||
const track = stream.getTracks()[0];
|
||||
|
||||
if (!track) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sender = state.connection.getSenders().find((l) => track.id === l.track?.id);
|
||||
|
||||
if (!sender) {
|
||||
return;
|
||||
}
|
||||
|
||||
value = value === undefined ? !track.enabled : value;
|
||||
|
||||
try {
|
||||
if (value && !track.enabled) {
|
||||
const newStream = await getUserStream(streamType);
|
||||
newStream.getTracks()[0].onended = () => {
|
||||
toggleStreamP2p(streamType, false);
|
||||
};
|
||||
await sender.replaceTrack(newStream.getTracks()[0]);
|
||||
if (streamType === 'audio') {
|
||||
state.streams.ownAudio = newStream;
|
||||
} else if (streamType === 'video') {
|
||||
state.streams.ownVideo = newStream;
|
||||
state.facingMode = 'user';
|
||||
} else {
|
||||
state.streams.ownPresentation = newStream;
|
||||
}
|
||||
if (streamType === 'video' || streamType === 'presentation') {
|
||||
toggleStreamP2p(streamType === 'video' ? 'presentation' : 'video', false);
|
||||
}
|
||||
// if (streamType === 'video') {
|
||||
// state.facingMode = 'user';
|
||||
// }
|
||||
} else if (!value && track.enabled) {
|
||||
track.stop();
|
||||
const newStream = streamType === 'audio' ? state.silence
|
||||
: (streamType === 'video' ? state.blackVideo : state.blackPresentation);
|
||||
if (!newStream) return;
|
||||
|
||||
await sender.replaceTrack(newStream.getTracks()[0]);
|
||||
|
||||
if (streamType === 'audio') {
|
||||
state.streams.ownAudio = newStream;
|
||||
} else if (streamType === 'video') {
|
||||
state.streams.ownVideo = newStream;
|
||||
} else {
|
||||
state.streams.ownPresentation = newStream;
|
||||
}
|
||||
// if (streamType === 'video') {
|
||||
// state.facingMode = undefined;
|
||||
// }
|
||||
}
|
||||
updateStreams();
|
||||
sendMediaState();
|
||||
} catch (e) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
export async function joinPhoneCall(
|
||||
connections: ApiPhoneCallConnection[],
|
||||
emitSignalingData: (data: P2pMessage) => void,
|
||||
isOutgoing: boolean,
|
||||
shouldStartVideo: boolean,
|
||||
onUpdate: (...args: any[]) => void,
|
||||
) {
|
||||
const conn = new RTCPeerConnection({
|
||||
iceServers: connections.map((connection) => (
|
||||
{
|
||||
urls: [
|
||||
connection.isTurn && `turn:${connection.ip}:${connection.port}`,
|
||||
connection.isStun && `stun:${connection.ip}:${connection.port}`,
|
||||
].filter(Boolean) as string[],
|
||||
username: connection.username,
|
||||
credentialType: 'password',
|
||||
credential: connection.password,
|
||||
}
|
||||
)),
|
||||
iceCandidatePoolSize: 2,
|
||||
});
|
||||
const slnc = silence(new AudioContext());
|
||||
const video = black({ width: 640, height: 480 });
|
||||
const screenshare = black({ width: 640, height: 480 });
|
||||
conn.addTrack(slnc.getTracks()[0], slnc);
|
||||
conn.addTrack(video.getTracks()[0], video);
|
||||
conn.addTrack(screenshare.getTracks()[0], screenshare);
|
||||
|
||||
conn.onicecandidate = (e) => {
|
||||
if (!e.candidate) return;
|
||||
|
||||
emitSignalingData({
|
||||
'@type': 'Candidates',
|
||||
candidates: [{
|
||||
sdpString: e.candidate.candidate,
|
||||
}],
|
||||
});
|
||||
};
|
||||
|
||||
conn.onconnectionstatechange = () => {
|
||||
onUpdate({
|
||||
'@type': 'updatePhoneCallConnectionState',
|
||||
connectionState: conn.connectionState,
|
||||
});
|
||||
};
|
||||
|
||||
conn.ontrack = (e) => {
|
||||
if (!state) return;
|
||||
|
||||
const stream = e.streams[0];
|
||||
if (e.track.kind === 'audio') {
|
||||
state.audio.srcObject = stream;
|
||||
state.audio.play().catch();
|
||||
state.streams.audio = stream;
|
||||
} else if (e.transceiver.mid === '1') {
|
||||
state.streams.video = stream;
|
||||
} else {
|
||||
state.streams.presentation = stream;
|
||||
}
|
||||
|
||||
updateStreams();
|
||||
};
|
||||
|
||||
const dc = conn.createDataChannel('data', {
|
||||
id: 0,
|
||||
negotiated: true,
|
||||
});
|
||||
|
||||
dc.onmessage = (e) => {
|
||||
processSignalingMessage(JSON.parse(e.data));
|
||||
};
|
||||
|
||||
const audio = new Audio();
|
||||
|
||||
state = {
|
||||
audio,
|
||||
connection: conn,
|
||||
emitSignalingData,
|
||||
isOutgoing,
|
||||
candidates: [],
|
||||
onUpdate,
|
||||
streams: {
|
||||
ownVideo: video,
|
||||
ownAudio: slnc,
|
||||
ownPresentation: screenshare,
|
||||
},
|
||||
mediaState: {
|
||||
isBatteryLow: false,
|
||||
screencastState: 'inactive',
|
||||
videoState: 'inactive',
|
||||
videoRotation: 0,
|
||||
isMuted: true,
|
||||
},
|
||||
blackVideo: video,
|
||||
blackPresentation: screenshare,
|
||||
silence: slnc,
|
||||
dataChannel: dc,
|
||||
};
|
||||
|
||||
try {
|
||||
if (shouldStartVideo) {
|
||||
toggleStreamP2p('video', true);
|
||||
}
|
||||
toggleStreamP2p('audio', true);
|
||||
} catch (e) {
|
||||
|
||||
}
|
||||
|
||||
if (isOutgoing) {
|
||||
const offer = await conn.createOffer({
|
||||
offerToReceiveAudio: true,
|
||||
offerToReceiveVideo: true,
|
||||
});
|
||||
|
||||
await conn.setLocalDescription(offer);
|
||||
|
||||
sendInitialSetup(parseSdp(offer, true) as P2pParsedSdp);
|
||||
}
|
||||
}
|
||||
|
||||
export function stopPhoneCall() {
|
||||
if (!state) return;
|
||||
|
||||
state.streams.ownVideo?.getTracks().forEach((track) => track.stop());
|
||||
state.streams.ownPresentation?.getTracks().forEach((track) => track.stop());
|
||||
state.streams.ownAudio?.getTracks().forEach((track) => track.stop());
|
||||
state.dataChannel.close();
|
||||
state.connection.close();
|
||||
state = undefined;
|
||||
}
|
||||
|
||||
function sendMediaState() {
|
||||
if (!state) return;
|
||||
const { emitSignalingData, streams } = state;
|
||||
|
||||
emitSignalingData({
|
||||
'@type': 'MediaState',
|
||||
videoRotation: 0,
|
||||
isMuted: !streams.ownAudio?.getTracks()[0].enabled,
|
||||
isBatteryLow: true,
|
||||
videoState: streams.ownVideo?.getTracks()[0].enabled ? 'active' : 'inactive',
|
||||
screencastState: streams.ownPresentation?.getTracks()[0].enabled ? 'active' : 'inactive',
|
||||
});
|
||||
}
|
||||
|
||||
function filterVP8(mediaContent: MediaContent) {
|
||||
if (!state || state.isOutgoing) return mediaContent;
|
||||
|
||||
const { payloadTypes } = mediaContent!;
|
||||
const idx = payloadTypes.findIndex((payloadType) => payloadType.name === 'VP8');
|
||||
const vp8PayloadType = payloadTypes[idx];
|
||||
const rtxIdx = payloadTypes.findIndex((payloadType) => Number(payloadType.parameters?.apt) === vp8PayloadType.id);
|
||||
mediaContent.payloadTypes = [payloadTypes[idx], payloadTypes[rtxIdx]];
|
||||
|
||||
return mediaContent;
|
||||
}
|
||||
|
||||
function sendInitialSetup(sdp: P2pParsedSdp) {
|
||||
if (!state) return;
|
||||
const { emitSignalingData } = state;
|
||||
|
||||
if (!sdp.ssrc || !sdp['ssrc-groups'] || !sdp['ssrc-groups'][0] || !sdp['ssrc-groups'][1]) return;
|
||||
|
||||
emitSignalingData({
|
||||
'@type': 'InitialSetup',
|
||||
fingerprints: sdp.fingerprints,
|
||||
ufrag: sdp.ufrag,
|
||||
pwd: sdp.pwd,
|
||||
audio: {
|
||||
ssrc: fromTelegramSource(sdp.ssrc).toString(),
|
||||
ssrcGroups: [],
|
||||
payloadTypes: sdp.audioPayloadTypes,
|
||||
rtpExtensions: sdp.audioExtmap,
|
||||
},
|
||||
video: filterVP8({
|
||||
ssrc: fromTelegramSource(sdp['ssrc-groups'][0].sources[0]).toString(),
|
||||
ssrcGroups: [{
|
||||
semantics: sdp['ssrc-groups'][0].semantics,
|
||||
ssrcs: sdp['ssrc-groups'][0].sources.map(fromTelegramSource),
|
||||
}],
|
||||
payloadTypes: sdp.videoPayloadTypes,
|
||||
rtpExtensions: sdp.videoExtmap,
|
||||
}),
|
||||
screencast: filterVP8({
|
||||
ssrc: fromTelegramSource(sdp['ssrc-groups'][1].sources[0]).toString(),
|
||||
ssrcGroups: [{
|
||||
semantics: sdp['ssrc-groups'][1].semantics,
|
||||
ssrcs: sdp['ssrc-groups'][1].sources.map(fromTelegramSource),
|
||||
}],
|
||||
payloadTypes: sdp.screencastPayloadTypes,
|
||||
rtpExtensions: sdp.screencastExtmap,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function processSignalingMessage(message: P2pMessage) {
|
||||
if (!state || !state.connection) return;
|
||||
|
||||
console.log(message);
|
||||
|
||||
switch (message['@type']) {
|
||||
case 'MediaState': {
|
||||
state.mediaState = message;
|
||||
updateStreams();
|
||||
sendMediaState();
|
||||
break;
|
||||
}
|
||||
case 'Candidates': {
|
||||
const { candidates, gotInitialSetup } = state;
|
||||
|
||||
if (!candidates) return;
|
||||
|
||||
message.candidates.forEach((candidate) => {
|
||||
state!.candidates.push(candidate.sdpString);
|
||||
});
|
||||
|
||||
if (gotInitialSetup) {
|
||||
await Promise.all(state.candidates.map((c) => state!.connection.addIceCandidate({
|
||||
candidate: c,
|
||||
sdpMLineIndex: 0,
|
||||
})));
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'InitialSetup': {
|
||||
const {
|
||||
connection, isOutgoing,
|
||||
} = state;
|
||||
if (!connection) return;
|
||||
|
||||
const newConference = {
|
||||
transport: {
|
||||
candidates: [],
|
||||
ufrag: message.ufrag,
|
||||
pwd: message.pwd,
|
||||
fingerprints: message.fingerprints,
|
||||
'rtcp-mux': false,
|
||||
xmlns: '',
|
||||
},
|
||||
sessionId: Date.now(),
|
||||
ssrcs: [
|
||||
message.audio && {
|
||||
isVideo: false,
|
||||
isMain: false,
|
||||
userId: '123',
|
||||
endpoint: '0',
|
||||
sourceGroups: [{
|
||||
semantics: 'FID',
|
||||
sources: [message.audio.ssrc],
|
||||
}],
|
||||
},
|
||||
message.video && {
|
||||
isVideo: true,
|
||||
isPresentation: false,
|
||||
isMain: false,
|
||||
userId: '123',
|
||||
endpoint: '1',
|
||||
sourceGroups: message.video.ssrcGroups.map((l) => ({
|
||||
semantics: l.semantics,
|
||||
sources: l.ssrcs,
|
||||
})),
|
||||
},
|
||||
message.screencast && {
|
||||
isVideo: true,
|
||||
isPresentation: true,
|
||||
isMain: false,
|
||||
userId: '123',
|
||||
endpoint: '2',
|
||||
sourceGroups: message.screencast.ssrcGroups.map((l) => ({
|
||||
semantics: l.semantics,
|
||||
sources: l.ssrcs,
|
||||
})),
|
||||
},
|
||||
],
|
||||
audioPayloadTypes: message.audio!.payloadTypes?.map(p2pPayloadTypeToConference) || [],
|
||||
audioExtensions: message.audio!.rtpExtensions,
|
||||
videoPayloadTypes: filterVP8(message.video!).payloadTypes?.map(p2pPayloadTypeToConference) || [],
|
||||
videoExtensions: message.video!.rtpExtensions,
|
||||
} as Conference;
|
||||
|
||||
await connection.setRemoteDescription({
|
||||
sdp: buildSdp(newConference, isOutgoing, undefined, true),
|
||||
type: isOutgoing ? 'answer' : 'offer',
|
||||
});
|
||||
|
||||
state.conference = newConference;
|
||||
|
||||
if (!isOutgoing) {
|
||||
const answer = await connection.createAnswer();
|
||||
if (!answer) return;
|
||||
|
||||
await connection.setLocalDescription(answer);
|
||||
|
||||
sendInitialSetup(parseSdp(answer, true) as P2pParsedSdp);
|
||||
}
|
||||
|
||||
state.gotInitialSetup = true;
|
||||
await Promise.all(state.candidates.map((c) => connection.addIceCandidate({
|
||||
candidate: c,
|
||||
sdpMLineIndex: 0,
|
||||
})));
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
47
src/lib/secret-sauce/p2pMessage.d.ts
vendored
47
src/lib/secret-sauce/p2pMessage.d.ts
vendored
@ -1,47 +0,0 @@
|
||||
import type { Fingerprint, RTCPFeedbackParam, RTPExtension } from './types';
|
||||
export declare type VideoState = 'inactive' | 'active' | 'suspended';
|
||||
export declare type VideoRotation = 0 | 90 | 180 | 270;
|
||||
export declare type MediaStateMessage = {
|
||||
'@type': 'MediaState';
|
||||
isMuted: boolean;
|
||||
videoState: VideoState;
|
||||
videoRotation: VideoRotation;
|
||||
screencastState: VideoState;
|
||||
isBatteryLow: boolean;
|
||||
};
|
||||
declare type CandidatesMessage = {
|
||||
'@type': 'Candidates';
|
||||
candidates: P2pCandidate[];
|
||||
};
|
||||
export declare type InitialSetupMessage = {
|
||||
'@type': 'InitialSetup';
|
||||
ufrag: string;
|
||||
pwd: string;
|
||||
fingerprints: Fingerprint[];
|
||||
audio?: MediaContent;
|
||||
video?: MediaContent;
|
||||
screencast?: MediaContent;
|
||||
};
|
||||
export declare type MediaContent = {
|
||||
ssrc: string;
|
||||
ssrcGroups: P2pSsrcGroup[];
|
||||
payloadTypes: P2PPayloadType[];
|
||||
rtpExtensions: RTPExtension[];
|
||||
};
|
||||
export interface P2PPayloadType {
|
||||
id: number;
|
||||
name: string;
|
||||
clockrate: number;
|
||||
channels: number;
|
||||
parameters?: Record<string, string>;
|
||||
feedbackTypes?: RTCPFeedbackParam[];
|
||||
}
|
||||
declare type P2pSsrcGroup = {
|
||||
semantics: string;
|
||||
ssrcs: number[];
|
||||
};
|
||||
declare type P2pCandidate = {
|
||||
sdpString: string;
|
||||
};
|
||||
export declare type P2pMessage = CandidatesMessage | InitialSetupMessage | MediaStateMessage;
|
||||
export {};
|
||||
58
src/lib/secret-sauce/p2pMessage.ts
Normal file
58
src/lib/secret-sauce/p2pMessage.ts
Normal file
@ -0,0 +1,58 @@
|
||||
import type {
|
||||
Fingerprint, RTCPFeedbackParam, RTPExtension,
|
||||
} from './types';
|
||||
|
||||
export type VideoState = 'inactive' | 'active' | 'suspended';
|
||||
|
||||
export type VideoRotation = 0 | 90 | 180 | 270;
|
||||
|
||||
export type MediaStateMessage = {
|
||||
'@type': 'MediaState';
|
||||
isMuted: boolean;
|
||||
videoState: VideoState;
|
||||
videoRotation: VideoRotation;
|
||||
screencastState: VideoState;
|
||||
isBatteryLow: boolean;
|
||||
};
|
||||
|
||||
type CandidatesMessage = {
|
||||
'@type': 'Candidates';
|
||||
candidates: P2pCandidate[];
|
||||
};
|
||||
|
||||
export type InitialSetupMessage = {
|
||||
'@type': 'InitialSetup';
|
||||
ufrag: string;
|
||||
pwd: string;
|
||||
fingerprints: Fingerprint[];
|
||||
audio?: MediaContent;
|
||||
video?: MediaContent;
|
||||
screencast?: MediaContent;
|
||||
};
|
||||
|
||||
export type MediaContent = {
|
||||
ssrc: string;
|
||||
ssrcGroups: P2pSsrcGroup[];
|
||||
payloadTypes: P2PPayloadType[];
|
||||
rtpExtensions: RTPExtension[];
|
||||
};
|
||||
|
||||
export interface P2PPayloadType {
|
||||
id: number;
|
||||
name: string;
|
||||
clockrate: number;
|
||||
channels: number;
|
||||
parameters?: Record<string, string>;
|
||||
feedbackTypes?: RTCPFeedbackParam[];
|
||||
}
|
||||
|
||||
type P2pSsrcGroup = {
|
||||
semantics: string;
|
||||
ssrcs: number[];
|
||||
};
|
||||
|
||||
type P2pCandidate = {
|
||||
sdpString: string;
|
||||
};
|
||||
|
||||
export type P2pMessage = CandidatesMessage | InitialSetupMessage | MediaStateMessage;
|
||||
3
src/lib/secret-sauce/parseSdp.d.ts
vendored
3
src/lib/secret-sauce/parseSdp.d.ts
vendored
@ -1,3 +0,0 @@
|
||||
import type { JoinGroupCallPayload } from './types';
|
||||
declare const _default: (sessionDescription: RTCSessionDescriptionInit, isP2p?: boolean) => JoinGroupCallPayload;
|
||||
export default _default;
|
||||
140
src/lib/secret-sauce/parseSdp.ts
Normal file
140
src/lib/secret-sauce/parseSdp.ts
Normal file
@ -0,0 +1,140 @@
|
||||
import { toTelegramSource } from './utils';
|
||||
import type { JoinGroupCallPayload, SsrcGroup } from './types';
|
||||
|
||||
export default (sessionDescription: RTCSessionDescriptionInit, isP2p = false): JoinGroupCallPayload => {
|
||||
if (!sessionDescription || !sessionDescription.sdp) {
|
||||
throw Error('Failed parsing SDP: session description is null');
|
||||
}
|
||||
|
||||
const sections = sessionDescription
|
||||
.sdp
|
||||
.split('\r\nm=')
|
||||
.map((s, i) => (i === 0 ? s : `m=${s}`))
|
||||
.reduce((acc: Record<string, string[]>, el) => {
|
||||
const name = el.match(/^m=(.+?)\s/)?.[1] || 'header';
|
||||
acc[acc.hasOwnProperty(name) && name === 'video' ? 'screencast' : name] = el.split('\r\n').filter(Boolean);
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const lookup = (prefix: string, sectionName?: string) => {
|
||||
if (!sectionName) {
|
||||
return Object.values(sections).map((section) => {
|
||||
return section.find((line) => line.startsWith(prefix))?.substr(prefix.length);
|
||||
}).filter(Boolean)[0];
|
||||
} else {
|
||||
return sections[sectionName]?.find((line) => line.startsWith(prefix))?.substr(prefix.length);
|
||||
}
|
||||
};
|
||||
|
||||
const parseExtmaps = (sectionName: string) => {
|
||||
return sections[sectionName].filter((l) => l.startsWith('a=extmap')).map((l) => {
|
||||
const [, id, uri] = l.match(/extmap:(\d+)(?:\/.+)?\s(.+)/)!;
|
||||
return { id: Number(id), uri };
|
||||
});
|
||||
};
|
||||
|
||||
const parsePayloadTypes = (sectionName: string) => {
|
||||
const payloads = sections[sectionName].filter((l) => l.startsWith('a=rtpmap')).map((l) => {
|
||||
const [, id, data] = l.match(/:(\d+)\s(.+)/) || [];
|
||||
const [name, clockrate, channels] = data.split('/');
|
||||
return {
|
||||
id: Number(id), name, clockrate: Number(clockrate), ...(channels && { channels: Number(channels) }),
|
||||
};
|
||||
});
|
||||
|
||||
const fbParams = sections[sectionName].filter((l) => l.startsWith('a=rtcp-fb')).map((l) => {
|
||||
const [, id, data] = l.match(/:(\d+)\s(.+)/) || [];
|
||||
const [type, subtype] = data.split(' ');
|
||||
return { id: Number(id), type, subtype: subtype || '' };
|
||||
});
|
||||
|
||||
const parameters = sections[sectionName].filter((l) => l.startsWith('a=fmtp')).map((l) => {
|
||||
const [, id, data] = l.match(/:(\d+)\s(.+)/) || [];
|
||||
const d = data.split(';').reduce((acc: Record<string, string>, q) => {
|
||||
const [name, value] = q.split('=');
|
||||
acc[name] = value;
|
||||
return acc;
|
||||
}, {});
|
||||
if (Object.values(d).some((z) => !z)) return undefined;
|
||||
return { id: Number(id), data: d };
|
||||
}).filter(Boolean);
|
||||
|
||||
return payloads.map((payload) => {
|
||||
const p = parameters.filter((l) => l!.id === payload.id).map((q) => q!.data).reduce((acc, el) => {
|
||||
return Object.assign(acc, el);
|
||||
}, {});
|
||||
const f = fbParams.filter((l) => l.id === payload.id).map((l) => {
|
||||
return {
|
||||
type: l.type,
|
||||
subtype: l.subtype,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
...payload,
|
||||
...(Object.keys(p).length > 0 && { parameters: p }),
|
||||
...(f.length > 0 && { feedbackTypes: f }),
|
||||
};
|
||||
});
|
||||
};
|
||||
|
||||
const rawSource = lookup('a=ssrc:', 'audio');
|
||||
const sourceAudio = rawSource && Number(rawSource.split(' ')[0]);
|
||||
|
||||
// TODO multiple source groups
|
||||
const rawSourceVideo = lookup('a=ssrc-group:', 'video')?.split(' ') || undefined;
|
||||
const rawSourceScreencast = lookup('a=ssrc-group:', 'screencast')?.split(' ') || undefined;
|
||||
|
||||
if (!rawSourceVideo) {
|
||||
throw Error('Failed parsing SDP: no video ssrc');
|
||||
}
|
||||
|
||||
const [hash, fingerprint] = lookup('a=fingerprint:')?.split(' ') || [];
|
||||
|
||||
const setup = lookup('a=setup:');
|
||||
if (!hash || !fingerprint) {
|
||||
throw Error('Failed parsing SDP: no fingerprint');
|
||||
}
|
||||
|
||||
console.log(sections);
|
||||
|
||||
const ufrag = lookup('a=ice-ufrag:');
|
||||
const pwd = lookup('a=ice-pwd:');
|
||||
|
||||
if (!ufrag || !pwd) {
|
||||
throw Error('Failed parsing SDP: no ICE ufrag or pwd');
|
||||
}
|
||||
|
||||
return {
|
||||
fingerprints: [
|
||||
{
|
||||
fingerprint,
|
||||
hash,
|
||||
setup: isP2p ? setup! : 'active',
|
||||
},
|
||||
],
|
||||
pwd,
|
||||
ufrag,
|
||||
...(sourceAudio && { ssrc: toTelegramSource(sourceAudio) }),
|
||||
...(rawSourceVideo && {
|
||||
'ssrc-groups': [
|
||||
{
|
||||
semantics: rawSourceVideo[0],
|
||||
sources: rawSourceVideo.slice(1, rawSourceVideo.length).map(Number).map(toTelegramSource),
|
||||
},
|
||||
(isP2p && rawSourceScreencast && {
|
||||
semantics: rawSourceScreencast[0],
|
||||
sources: rawSourceScreencast.slice(1, rawSourceScreencast.length).map(Number).map(toTelegramSource),
|
||||
}),
|
||||
].filter(Boolean) as SsrcGroup[],
|
||||
}),
|
||||
...(isP2p && {
|
||||
audioExtmap: parseExtmaps('audio'),
|
||||
videoExtmap: parseExtmaps('video'),
|
||||
screencastExtmap: parseExtmaps('screencast'),
|
||||
audioPayloadTypes: parsePayloadTypes('audio'),
|
||||
videoPayloadTypes: parsePayloadTypes('video'),
|
||||
screencastPayloadTypes: parsePayloadTypes('screencast'),
|
||||
}),
|
||||
};
|
||||
};
|
||||
19
src/lib/secret-sauce/secretsauce.d.ts
vendored
19
src/lib/secret-sauce/secretsauce.d.ts
vendored
@ -1,19 +0,0 @@
|
||||
import type { GroupCallConnectionData, GroupCallParticipant, JoinGroupCallPayload } from './types';
|
||||
export declare type StreamType = 'audio' | 'video' | 'presentation';
|
||||
export declare function getDevices(streamType: StreamType, isInput?: boolean): Promise<MediaDeviceInfo[]>;
|
||||
export declare function toggleSpeaker(): void;
|
||||
export declare function toggleNoiseSuppression(): void;
|
||||
export declare function getUserStreams(userId: string): {
|
||||
audio?: MediaStream | undefined;
|
||||
video?: MediaStream | undefined;
|
||||
presentation?: MediaStream | undefined;
|
||||
} | undefined;
|
||||
export declare function setVolume(userId: string, volume: number): void;
|
||||
export declare function isStreamEnabled(streamType: StreamType, userId?: string): boolean;
|
||||
export declare function switchCameraInput(): Promise<void>;
|
||||
export declare function toggleStream(streamType: StreamType, value?: boolean | undefined): Promise<void>;
|
||||
export declare function leaveGroupCall(): void;
|
||||
export declare function handleUpdateGroupCallParticipants(updatedParticipants: GroupCallParticipant[]): Promise<void>;
|
||||
export declare function handleUpdateGroupCallConnection(data: GroupCallConnectionData, isPresentation: boolean): Promise<void>;
|
||||
export declare function startSharingScreen(): Promise<JoinGroupCallPayload | undefined>;
|
||||
export declare function joinGroupCall(myId: string, audioContext: AudioContext, audioElement: HTMLAudioElement, onUpdate: (...args: any[]) => void): Promise<JoinGroupCallPayload>;
|
||||
851
src/lib/secret-sauce/secretsauce.ts
Normal file
851
src/lib/secret-sauce/secretsauce.ts
Normal file
@ -0,0 +1,851 @@
|
||||
import parseSdp from './parseSdp';
|
||||
import { ColibriClass } from './colibriClass';
|
||||
import type {
|
||||
GroupCallConnectionData, GroupCallConnectionState, GroupCallParticipant, JoinGroupCallPayload,
|
||||
} from './types';
|
||||
import buildSdp, { Conference, Ssrc } from './buildSdp';
|
||||
import { black, silence } from './blacksilence';
|
||||
import {
|
||||
getAmplitude,
|
||||
IS_ECHO_CANCELLATION_SUPPORTED,
|
||||
IS_NOISE_SUPPRESSION_SUPPORTED,
|
||||
THRESHOLD,
|
||||
} from './utils';
|
||||
|
||||
export type StreamType = 'audio' | 'video' | 'presentation';
|
||||
|
||||
type GroupCallState = {
|
||||
connection?: RTCPeerConnection;
|
||||
screenshareConnection?: RTCPeerConnection;
|
||||
dataChannel?: RTCDataChannel;
|
||||
screenshareDataChannel?: RTCDataChannel;
|
||||
participants?: GroupCallParticipant[];
|
||||
conference?: Partial<Conference>;
|
||||
screenshareConference?: Partial<Conference>;
|
||||
streams?: Record<string, {
|
||||
audio?: MediaStream;
|
||||
video?: MediaStream;
|
||||
presentation?: MediaStream;
|
||||
}>;
|
||||
participantFunctions?: Record<string, {
|
||||
setVolume?: (volume: number) => void;
|
||||
toggleMute?: (muted: boolean) => void;
|
||||
getCurrentAmplitude?: () => number;
|
||||
}>;
|
||||
onUpdate?: (...args: any[]) => void;
|
||||
myId?: string;
|
||||
black?: MediaStream;
|
||||
silence?: MediaStream;
|
||||
updatingParticipantsQueue?: any[];
|
||||
facingMode?: VideoFacingModeEnum;
|
||||
isSpeakerDisabled?: boolean;
|
||||
analyserInterval?: number;
|
||||
speaking?: Record<string, number>;
|
||||
audioElement?: HTMLAudioElement;
|
||||
destination?: MediaStreamAudioDestinationNode;
|
||||
audioContext?: AudioContext;
|
||||
mediaStream?: MediaStream;
|
||||
};
|
||||
|
||||
let state: GroupCallState | undefined;
|
||||
|
||||
export async function getDevices(streamType: StreamType, isInput = true) {
|
||||
return (await navigator.mediaDevices.enumerateDevices())
|
||||
.filter((l) => l.kind === `${streamType}${isInput ? 'input' : 'output'}`);
|
||||
}
|
||||
|
||||
export function toggleSpeaker() {
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.isSpeakerDisabled = !state.isSpeakerDisabled;
|
||||
state?.onUpdate?.({
|
||||
'@type': 'updateGroupCallConnectionState',
|
||||
connectionState: 'connected',
|
||||
isSpeakerDisabled: state.isSpeakerDisabled,
|
||||
});
|
||||
if (state.participantFunctions) {
|
||||
Object.values(state.participantFunctions).forEach((l) => {
|
||||
l.toggleMute?.(!!state?.isSpeakerDisabled);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function leavePresentation(isFromToggle?: boolean) {
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
state.screenshareDataChannel?.close();
|
||||
state.screenshareConnection?.close();
|
||||
|
||||
if (!isFromToggle) {
|
||||
state.onUpdate?.({
|
||||
'@type': 'updateGroupCallLeavePresentation',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function toggleNoiseSuppression() {
|
||||
if (!state || !state.myId || !state.streams) {
|
||||
return;
|
||||
}
|
||||
|
||||
const audioStream = state.streams[state.myId].audio;
|
||||
if (!audioStream) {
|
||||
return;
|
||||
}
|
||||
|
||||
const track = audioStream.getTracks()[0];
|
||||
|
||||
if (!track) {
|
||||
return;
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
const { echoCancellation, noiseSuppression } = track.getConstraints();
|
||||
|
||||
track.applyConstraints({
|
||||
echoCancellation: !echoCancellation,
|
||||
// @ts-ignore
|
||||
noiseSuppression: !noiseSuppression,
|
||||
});
|
||||
}
|
||||
|
||||
export function getUserStreams(userId: string) {
|
||||
return state?.streams?.[userId];
|
||||
}
|
||||
|
||||
export function setVolume(userId: string, volume: number) {
|
||||
const participantFunctions = state?.participantFunctions?.[userId];
|
||||
if (!participantFunctions) return;
|
||||
participantFunctions.setVolume?.(volume);
|
||||
}
|
||||
|
||||
export function isStreamEnabled(streamType: StreamType, userId?: string) {
|
||||
const id = userId || state?.myId;
|
||||
const stream = id && getUserStreams(id)?.[streamType];
|
||||
if (!stream) return false;
|
||||
|
||||
return stream.getTracks()[0]?.enabled;
|
||||
}
|
||||
|
||||
function updateGroupCallStreams(userId: string) {
|
||||
state?.onUpdate?.({
|
||||
'@type': 'updateGroupCallStreams',
|
||||
userId,
|
||||
hasAudioStream: isStreamEnabled('audio', userId),
|
||||
hasVideoStream: isStreamEnabled('video', userId),
|
||||
hasPresentationStream: isStreamEnabled('presentation', userId),
|
||||
amplitude: state.speaking?.[userId],
|
||||
});
|
||||
}
|
||||
|
||||
function getUserStream(streamType: StreamType, facing: VideoFacingModeEnum = 'user') {
|
||||
if (streamType === 'presentation') {
|
||||
return (navigator.mediaDevices as any).getDisplayMedia({
|
||||
audio: false,
|
||||
video: true,
|
||||
});
|
||||
}
|
||||
|
||||
return navigator.mediaDevices.getUserMedia({
|
||||
audio: streamType === 'audio' ? {
|
||||
// @ts-ignore
|
||||
...(IS_ECHO_CANCELLATION_SUPPORTED && { echoCancellation: true }),
|
||||
...(IS_NOISE_SUPPRESSION_SUPPORTED && { noiseSuppression: true }),
|
||||
} : false,
|
||||
video: streamType === 'video' ? {
|
||||
facingMode: facing,
|
||||
} : false,
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
export async function switchCameraInput() {
|
||||
if (!state?.myId || !state.connection || !state.streams || !state.facingMode) {
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = getUserStreams(state.myId)?.video;
|
||||
|
||||
if (!stream) return;
|
||||
|
||||
const track = stream.getTracks()[0];
|
||||
|
||||
if (!track) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sender = state.connection.getSenders().find((l) => track.id === l.track?.id);
|
||||
|
||||
if (!sender) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.facingMode = state.facingMode === 'environment' ? 'user' : 'environment';
|
||||
try {
|
||||
const newStream = await getUserStream('video', state.facingMode);
|
||||
|
||||
await sender.replaceTrack(newStream.getTracks()[0]);
|
||||
state.streams[state.myId].video = newStream;
|
||||
} catch (e) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
export async function toggleStream(streamType: StreamType, value: boolean | undefined = undefined) {
|
||||
if (!state || !state.myId || !state.connection || !state.streams) {
|
||||
return;
|
||||
}
|
||||
|
||||
const stream = getUserStreams(state.myId)?.[streamType];
|
||||
if (!stream) return;
|
||||
|
||||
const track = stream.getTracks()[0];
|
||||
|
||||
if (!track) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sender = [
|
||||
...state.connection.getSenders(),
|
||||
...(state.screenshareConnection?.getSenders() || []),
|
||||
].find((l) => track.id === l.track?.id);
|
||||
|
||||
if (!sender) {
|
||||
return;
|
||||
}
|
||||
|
||||
value = value === undefined ? !track.enabled : value;
|
||||
|
||||
try {
|
||||
if (value && !track.enabled) {
|
||||
const newStream = await getUserStream(streamType);
|
||||
await sender.replaceTrack(newStream.getTracks()[0]);
|
||||
state.streams[state.myId][streamType] = newStream;
|
||||
if (streamType === 'video') {
|
||||
state.facingMode = 'user';
|
||||
} else if (streamType === 'audio') {
|
||||
const { audioContext } = state;
|
||||
if (!audioContext) return;
|
||||
const source = audioContext.createMediaStreamSource(newStream);
|
||||
|
||||
const analyser = audioContext.createAnalyser();
|
||||
analyser.minDecibels = -100;
|
||||
analyser.maxDecibels = -30;
|
||||
analyser.smoothingTimeConstant = 0.05;
|
||||
analyser.fftSize = 1024;
|
||||
|
||||
source.connect(analyser);
|
||||
|
||||
state = {
|
||||
...state,
|
||||
participantFunctions: {
|
||||
...state.participantFunctions,
|
||||
[state.myId]: {
|
||||
...state.participantFunctions?.[state.myId],
|
||||
getCurrentAmplitude: () => {
|
||||
const array = new Uint8Array(analyser.frequencyBinCount);
|
||||
analyser.getByteFrequencyData(array);
|
||||
return getAmplitude(array, 1.5);
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
} else if (!value && track.enabled) {
|
||||
track.stop();
|
||||
const newStream = streamType === 'audio' ? state.silence : state.black;
|
||||
if (!newStream) return;
|
||||
|
||||
await sender.replaceTrack(newStream.getTracks()[0]);
|
||||
state.streams[state.myId][streamType] = newStream;
|
||||
if (streamType === 'video') {
|
||||
state.facingMode = undefined;
|
||||
}
|
||||
}
|
||||
updateGroupCallStreams(state.myId!);
|
||||
if (streamType === 'presentation' && !value) leavePresentation(true);
|
||||
} catch (e) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
function updateConnectionState(connectionState: GroupCallConnectionState) {
|
||||
state?.onUpdate?.({
|
||||
'@type': 'updateGroupCallConnectionState',
|
||||
connectionState,
|
||||
});
|
||||
}
|
||||
|
||||
export function leaveGroupCall() {
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.myId && state.streams?.[state.myId]) {
|
||||
Object.values(state.streams[state.myId] || {}).forEach((stream) => {
|
||||
stream?.getTracks().forEach((track) => {
|
||||
track.stop();
|
||||
});
|
||||
});
|
||||
}
|
||||
leavePresentation(true);
|
||||
state.dataChannel?.close();
|
||||
state.connection?.close();
|
||||
updateConnectionState('disconnected');
|
||||
|
||||
if (state.analyserInterval) {
|
||||
clearInterval(state.analyserInterval);
|
||||
}
|
||||
|
||||
state = undefined;
|
||||
}
|
||||
|
||||
function analyzeAmplitudes() {
|
||||
if (!state || !state.participantFunctions) return;
|
||||
|
||||
Object.keys(state.participantFunctions).forEach((id) => {
|
||||
const { getCurrentAmplitude } = state!.participantFunctions![Number(id)];
|
||||
|
||||
if (getCurrentAmplitude) {
|
||||
const amplitude = getCurrentAmplitude();
|
||||
const prevAmplitude = state!.speaking![id] || 0;
|
||||
state!.speaking![id] = amplitude;
|
||||
if ((amplitude > THRESHOLD && prevAmplitude <= THRESHOLD)
|
||||
|| (amplitude <= THRESHOLD && prevAmplitude > THRESHOLD)) {
|
||||
updateGroupCallStreams(id);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function createDataChannel(connection: RTCPeerConnection) {
|
||||
const dataChannel = connection.createDataChannel('data', {
|
||||
id: 0,
|
||||
});
|
||||
|
||||
dataChannel.onopen = () => {
|
||||
// console.log('Data channel open!');
|
||||
};
|
||||
|
||||
dataChannel.onmessage = (e) => {
|
||||
// console.log('onmessage');
|
||||
const data = JSON.parse(e.data) as ColibriClass;
|
||||
// console.log(data);
|
||||
switch (data.colibriClass) {
|
||||
case 'DominantSpeakerEndpointChangeEvent':
|
||||
break;
|
||||
case 'SenderVideoConstraints':
|
||||
|
||||
break;
|
||||
case 'EndpointConnectivityStatusChangeEvent':
|
||||
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
dataChannel.onerror = (e) => {
|
||||
console.log('%conerror', 'background: green; font-size: 5em');
|
||||
console.error(e);
|
||||
};
|
||||
|
||||
return dataChannel;
|
||||
}
|
||||
|
||||
export async function handleUpdateGroupCallParticipants(updatedParticipants: GroupCallParticipant[]) {
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
|
||||
const {
|
||||
participants, conference, connection, myId,
|
||||
} = state;
|
||||
|
||||
if (!participants || !conference || !connection || !conference.ssrcs || !conference.transport || !myId) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Joined from another client
|
||||
if (updatedParticipants.find((participant) => {
|
||||
return participant.isSelf
|
||||
&& participant.source
|
||||
!== state?.conference?.ssrcs?.find((l) => l.isMain && !l.isVideo)?.sourceGroups[0].sources[0];
|
||||
})) {
|
||||
leaveGroupCall();
|
||||
return;
|
||||
}
|
||||
|
||||
const newEndpoints: string[] = [];
|
||||
updatedParticipants.forEach((participant) => {
|
||||
console.log('handleUpdateGroupCallParticipants', participant);
|
||||
if (participant.isSelf) {
|
||||
if (participant.isMuted && !participant.canSelfUnmute) {
|
||||
// Muted by admin
|
||||
toggleStream('audio', false);
|
||||
toggleStream('video', false);
|
||||
toggleStream('presentation', false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const { isLeft } = participant;
|
||||
const isAudioLeft = participant.isMuted || participant.isMutedByMe;
|
||||
const isVideoLeft = !participant.isVideoJoined || !participant.video || isLeft;
|
||||
const isPresentationLeft = !participant.presentation || isLeft;
|
||||
|
||||
let hasVideo = false;
|
||||
let hasAudio = false;
|
||||
let hasPresentation = false;
|
||||
|
||||
conference.ssrcs!.filter((l) => l.userId === participant.id).forEach((ssrc) => {
|
||||
if (!ssrc.isVideo) {
|
||||
if (ssrc.sourceGroups[0].sources[0] === participant.source) {
|
||||
hasAudio = true;
|
||||
}
|
||||
// console.log('has audio, removed=', isAudioLeft);
|
||||
ssrc.isRemoved = isAudioLeft;
|
||||
}
|
||||
|
||||
if (ssrc.isVideo) {
|
||||
if (!ssrc.isPresentation) {
|
||||
if (!!participant.video && ssrc.endpoint === participant.video.endpoint) {
|
||||
hasVideo = true;
|
||||
}
|
||||
// console.log('has video = ', hasVideo, ' removed=', isVideoLeft);
|
||||
ssrc.isRemoved = isVideoLeft;
|
||||
}
|
||||
|
||||
if (ssrc.isPresentation) {
|
||||
if (!!participant.presentation && ssrc.endpoint === participant.presentation.endpoint) {
|
||||
hasPresentation = true;
|
||||
}
|
||||
// console.log('has presentation, removed=', isPresentationLeft);
|
||||
ssrc.isRemoved = isPresentationLeft;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (!isAudioLeft && !hasAudio) {
|
||||
// console.log('add audio');
|
||||
conference.ssrcs!.push({
|
||||
userId: participant.id,
|
||||
isMain: false,
|
||||
endpoint: `audio${participant.source}`,
|
||||
isVideo: false,
|
||||
sourceGroups: [{
|
||||
semantics: 'FID',
|
||||
sources: [participant.source],
|
||||
}],
|
||||
});
|
||||
}
|
||||
|
||||
if (!isVideoLeft && !hasVideo && participant.video) {
|
||||
// console.log('add video', participant.video);
|
||||
newEndpoints.push(participant.video.endpoint);
|
||||
conference.ssrcs!.push({
|
||||
userId: participant.id,
|
||||
isMain: false,
|
||||
endpoint: participant.video.endpoint,
|
||||
isVideo: true,
|
||||
sourceGroups: participant.video.sourceGroups,
|
||||
});
|
||||
}
|
||||
|
||||
if (!isPresentationLeft && !hasPresentation && participant.presentation) {
|
||||
// console.log('add presentation');
|
||||
conference.ssrcs!.push({
|
||||
isPresentation: true,
|
||||
userId: participant.id,
|
||||
isMain: false,
|
||||
endpoint: participant.presentation.endpoint,
|
||||
isVideo: true,
|
||||
sourceGroups: participant.presentation.sourceGroups,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (state.updatingParticipantsQueue) {
|
||||
state.updatingParticipantsQueue.push(conference);
|
||||
return;
|
||||
} else {
|
||||
state.updatingParticipantsQueue = [];
|
||||
}
|
||||
|
||||
const sdp = buildSdp(conference as Conference);
|
||||
console.log('build sdp!', sdp);
|
||||
await connection.setRemoteDescription({
|
||||
type: 'offer',
|
||||
sdp,
|
||||
});
|
||||
|
||||
try {
|
||||
const answer = await connection.createAnswer();
|
||||
await connection.setLocalDescription(answer);
|
||||
|
||||
updateGroupCallStreams(myId);
|
||||
if (state.updatingParticipantsQueue.length > 0) {
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
for (const newConference of state.updatingParticipantsQueue) {
|
||||
await connection.setRemoteDescription({
|
||||
type: 'offer',
|
||||
sdp: buildSdp(newConference as Conference),
|
||||
});
|
||||
const answerNew = await connection.createAnswer();
|
||||
await connection.setLocalDescription(answerNew);
|
||||
updateGroupCallStreams(myId);
|
||||
|
||||
// if (newEndpoints.length > 0) {
|
||||
// sendDataChannelMessage({
|
||||
// colibriClass: 'ReceiverVideoConstraints',
|
||||
// defaultConstraints: {
|
||||
// maxHeight: 0,
|
||||
// },
|
||||
// constraints: {
|
||||
// ...(newEndpoints.reduce((acc: Record<string, any>, el) => {
|
||||
// acc[el] = {
|
||||
// minHeight: 0,
|
||||
// maxHeight: 1080,
|
||||
// };
|
||||
// return acc;
|
||||
// }, {})),
|
||||
// },
|
||||
// onStageEndpoints: [],
|
||||
// });
|
||||
// }
|
||||
}
|
||||
}
|
||||
state.updatingParticipantsQueue = undefined;
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
//
|
||||
// function sendDataChannelMessage(message: ColibriClass) {
|
||||
// if (!state || !state.dataChannel || state.dataChannel.readyState !== 'open') {
|
||||
// return;
|
||||
// }
|
||||
//
|
||||
// // console.log('SEND!', message);
|
||||
// state.dataChannel.send(JSON.stringify(message));
|
||||
// }
|
||||
|
||||
export async function handleUpdateGroupCallConnection(data: GroupCallConnectionData, isPresentation: boolean) {
|
||||
if (!state) {
|
||||
return;
|
||||
}
|
||||
|
||||
const conference = isPresentation ? state.screenshareConference : state.conference;
|
||||
const connection = isPresentation ? state.screenshareConnection : state.connection;
|
||||
|
||||
if (!conference || !connection || !conference.ssrcs) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sessionId = Date.now();
|
||||
const newConference = {
|
||||
...conference,
|
||||
transport: data.transport,
|
||||
sessionId,
|
||||
audioExtensions: data.audio?.['rtp-hdrexts'],
|
||||
audioPayloadTypes: data.audio?.['payload-types'],
|
||||
videoExtensions: data.video?.['rtp-hdrexts'],
|
||||
videoPayloadTypes: data.video?.['payload-types'],
|
||||
} as Conference;
|
||||
|
||||
state = {
|
||||
...state,
|
||||
...(!isPresentation ? { conference: newConference } : { screenshareConference: newConference }),
|
||||
};
|
||||
|
||||
console.warn('update remote description', newConference, buildSdp(newConference, true, isPresentation));
|
||||
|
||||
try {
|
||||
await connection.setRemoteDescription({
|
||||
type: 'answer',
|
||||
sdp: buildSdp(newConference, true, isPresentation),
|
||||
});
|
||||
|
||||
// state.resolveTest();
|
||||
// state.test = true;
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
function handleTrack(e: RTCTrackEvent) {
|
||||
if (!state || !state.audioElement || !state.audioContext || !state.mediaStream) {
|
||||
return;
|
||||
}
|
||||
const ssrc = state.conference?.ssrcs?.find((l) => l.endpoint === e.track.id);
|
||||
if (!ssrc || !ssrc.userId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const { userId, isPresentation } = ssrc;
|
||||
const participant = state.participants?.find((p) => p.id === userId);
|
||||
|
||||
const streamType = (e.track.kind === 'video' ? (isPresentation ? 'presentation' : 'video') : 'audio') as StreamType;
|
||||
|
||||
e.track.onended = () => {
|
||||
delete state?.streams?.[userId][streamType];
|
||||
updateGroupCallStreams(userId);
|
||||
};
|
||||
|
||||
const stream = e.streams[0];
|
||||
|
||||
if (e.track.kind === 'audio') {
|
||||
const { mediaStream } = state;
|
||||
const audioContext = new (window.AudioContext)();
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
|
||||
const gainNode = audioContext.createGain();
|
||||
gainNode.gain.value = (participant?.volume || 10000) / 10000;
|
||||
|
||||
const muteNode = audioContext.createGain();
|
||||
gainNode.gain.value = 1;
|
||||
|
||||
const analyser = audioContext.createAnalyser();
|
||||
analyser.minDecibels = -100;
|
||||
analyser.maxDecibels = -30;
|
||||
analyser.smoothingTimeConstant = 0.05;
|
||||
analyser.fftSize = 1024;
|
||||
|
||||
source.connect(analyser).connect(muteNode).connect(gainNode).connect(audioContext.destination);
|
||||
|
||||
mediaStream!.addTrack(source.mediaStream.getAudioTracks()[0]);
|
||||
|
||||
// https://stackoverflow.com/questions/41784137/webrtc-doesnt-work-with-audiocontext#comment117600018_41784241
|
||||
const test = new Audio();
|
||||
test.srcObject = stream;
|
||||
// test.srcObject = source.mediaStream;
|
||||
test.muted = true;
|
||||
test.remove();
|
||||
|
||||
state = {
|
||||
...state,
|
||||
participantFunctions: {
|
||||
...state.participantFunctions,
|
||||
[userId]: {
|
||||
...state.participantFunctions?.[userId],
|
||||
setVolume: (volume: number) => {
|
||||
gainNode.gain.value = volume > 1 ? volume * 2 : volume;
|
||||
},
|
||||
toggleMute: (muted?: boolean) => {
|
||||
muteNode.gain.value = muted ? 0 : 1;
|
||||
},
|
||||
getCurrentAmplitude: () => {
|
||||
const array = new Uint8Array(analyser.frequencyBinCount);
|
||||
analyser.getByteFrequencyData(array);
|
||||
return getAmplitude(array, 1.5);
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
state = {
|
||||
...state,
|
||||
streams: {
|
||||
...state.streams,
|
||||
[userId]: {
|
||||
...state.streams?.[userId],
|
||||
[streamType]: stream,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
updateGroupCallStreams(userId);
|
||||
}
|
||||
|
||||
function initializeConnection(
|
||||
streams: MediaStream[],
|
||||
resolve: (payload: JoinGroupCallPayload) => void,
|
||||
isPresentation = false,
|
||||
) {
|
||||
const connection = new RTCPeerConnection();
|
||||
|
||||
const dataChannel = isPresentation ? undefined : createDataChannel(connection);
|
||||
|
||||
streams.forEach((stream) => stream.getTracks().forEach((track) => {
|
||||
connection.addTrack(track, stream);
|
||||
}));
|
||||
|
||||
if (!isPresentation) {
|
||||
connection.oniceconnectionstatechange = () => {
|
||||
const connectionState = connection.iceConnectionState;
|
||||
console.log('ice', connectionState);
|
||||
if (connectionState === 'connected' || connectionState === 'completed') {
|
||||
updateConnectionState('connected');
|
||||
} else if (connectionState === 'checking' || connectionState === 'new') {
|
||||
updateConnectionState('connecting');
|
||||
} else if (connection.iceConnectionState === 'disconnected') {
|
||||
updateConnectionState('reconnecting');
|
||||
}
|
||||
};
|
||||
}
|
||||
connection.ontrack = handleTrack;
|
||||
connection.onnegotiationneeded = async () => {
|
||||
if (!state) return;
|
||||
|
||||
const { myId } = state;
|
||||
|
||||
if (!myId) {
|
||||
return;
|
||||
}
|
||||
const offer = await connection.createOffer({
|
||||
offerToReceiveVideo: true,
|
||||
offerToReceiveAudio: !isPresentation,
|
||||
});
|
||||
|
||||
console.log('created offer!', offer);
|
||||
|
||||
await connection.setLocalDescription(offer);
|
||||
|
||||
if (!offer.sdp) {
|
||||
return;
|
||||
}
|
||||
|
||||
const sdp = parseSdp(offer);
|
||||
console.log('parsed sdp', sdp);
|
||||
const audioSsrc: Ssrc | undefined = !isPresentation ? {
|
||||
userId: '',
|
||||
sourceGroups: [
|
||||
{
|
||||
semantics: 'FID',
|
||||
sources: [sdp.ssrc || 0],
|
||||
},
|
||||
],
|
||||
isRemoved: isPresentation,
|
||||
isMain: true,
|
||||
isVideo: false,
|
||||
isPresentation,
|
||||
endpoint: isPresentation ? '1' : '0',
|
||||
} : undefined;
|
||||
|
||||
const videoSsrc: Ssrc | undefined = sdp['ssrc-groups'] && {
|
||||
isPresentation,
|
||||
userId: '',
|
||||
sourceGroups: sdp['ssrc-groups'],
|
||||
isMain: true,
|
||||
isVideo: true,
|
||||
endpoint: isPresentation ? '0' : '1',
|
||||
};
|
||||
|
||||
const conference = isPresentation ? state.screenshareConference : state.conference;
|
||||
|
||||
const ssrcs: Ssrc[] = [];
|
||||
if (isPresentation) {
|
||||
if (videoSsrc) ssrcs.push(videoSsrc);
|
||||
if (audioSsrc) ssrcs.push(audioSsrc);
|
||||
} else {
|
||||
if (audioSsrc) ssrcs.push(audioSsrc);
|
||||
if (videoSsrc) ssrcs.push(videoSsrc);
|
||||
}
|
||||
|
||||
const audioStream = streams.find((l) => l.getTracks()[0].kind === 'audio');
|
||||
const videoStream = streams.find((l) => l.getTracks()[0].kind === 'video');
|
||||
|
||||
state = {
|
||||
...state,
|
||||
...(!isPresentation ? {
|
||||
conference: {
|
||||
...conference,
|
||||
ssrcs,
|
||||
},
|
||||
} : {
|
||||
screenshareConference: {
|
||||
...conference,
|
||||
ssrcs,
|
||||
},
|
||||
}),
|
||||
streams: {
|
||||
...state.streams,
|
||||
[myId]: {
|
||||
...state.streams?.[myId],
|
||||
...(audioStream && { audio: audioStream }),
|
||||
...(!isPresentation && videoStream ? { video: videoStream } : { presentation: videoStream }),
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
updateGroupCallStreams(myId);
|
||||
|
||||
resolve(sdp);
|
||||
};
|
||||
|
||||
return { connection, dataChannel };
|
||||
}
|
||||
|
||||
export async function startSharingScreen(): Promise<JoinGroupCallPayload | undefined> {
|
||||
if (!state) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
const stream: MediaStream | undefined = await getUserStream('presentation');
|
||||
|
||||
if (!stream) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
stream.getTracks()[0].onended = () => {
|
||||
if (state && state.myId) {
|
||||
delete state.streams?.[state.myId].presentation;
|
||||
updateGroupCallStreams(state.myId);
|
||||
leavePresentation();
|
||||
}
|
||||
};
|
||||
|
||||
return await new Promise((resolve) => {
|
||||
const { connection, dataChannel } = initializeConnection([stream], resolve, true);
|
||||
state = {
|
||||
...state,
|
||||
screenshareConnection: connection,
|
||||
screenshareDataChannel: dataChannel,
|
||||
};
|
||||
});
|
||||
} catch (e) {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function joinGroupCall(
|
||||
myId: string,
|
||||
audioContext: AudioContext,
|
||||
audioElement: HTMLAudioElement,
|
||||
onUpdate: (...args: any[]) => void,
|
||||
): Promise<JoinGroupCallPayload> {
|
||||
if (state) {
|
||||
throw Error('Already in call');
|
||||
}
|
||||
|
||||
updateConnectionState('connecting');
|
||||
|
||||
const mediaStream = new MediaStream();
|
||||
audioElement.srcObject = mediaStream;
|
||||
audioElement.play().catch((l) => console.warn(l));
|
||||
|
||||
state = {
|
||||
onUpdate,
|
||||
participants: [],
|
||||
myId,
|
||||
speaking: {},
|
||||
silence: silence(audioContext),
|
||||
black: black({ width: 640, height: 480 }),
|
||||
// @ts-ignore
|
||||
analyserInterval: setInterval(analyzeAmplitudes, 1000),
|
||||
audioElement,
|
||||
// destination,
|
||||
audioContext,
|
||||
mediaStream,
|
||||
};
|
||||
|
||||
return new Promise((resolve) => {
|
||||
state = {
|
||||
...state,
|
||||
...initializeConnection([state!.silence!, state!.black!], resolve),
|
||||
};
|
||||
});
|
||||
}
|
||||
126
src/lib/secret-sauce/types.d.ts
vendored
126
src/lib/secret-sauce/types.d.ts
vendored
@ -1,126 +0,0 @@
|
||||
import { P2PPayloadType } from './p2pMessage';
|
||||
export interface GroupCallParticipant {
|
||||
isSelf?: boolean;
|
||||
isMuted?: boolean;
|
||||
isLeft?: boolean;
|
||||
isUser?: boolean;
|
||||
canSelfUnmute?: boolean;
|
||||
hasJustJoined?: boolean;
|
||||
isVideoJoined?: boolean;
|
||||
isMutedByMe?: boolean;
|
||||
isVolumeByAdmin?: boolean;
|
||||
isMin?: boolean;
|
||||
isVersioned?: boolean;
|
||||
source: number;
|
||||
date: Date;
|
||||
id: string;
|
||||
volume?: number;
|
||||
about?: string;
|
||||
video?: GroupCallParticipantVideo;
|
||||
presentation?: GroupCallParticipantVideo;
|
||||
raiseHandRating?: string;
|
||||
hasAudioStream?: boolean;
|
||||
hasVideoStream?: boolean;
|
||||
hasPresentationStream?: boolean;
|
||||
amplitude?: number;
|
||||
}
|
||||
export declare type GroupCallConnectionState = 'connecting' | 'connected' | 'disconnected' | 'reconnecting' | 'discarded';
|
||||
export interface GroupCallParticipantVideo {
|
||||
endpoint: string;
|
||||
isPaused?: boolean;
|
||||
sourceGroups: SsrcGroup[];
|
||||
audioSource?: number;
|
||||
}
|
||||
export declare type Fingerprint = {
|
||||
hash: string;
|
||||
setup: string;
|
||||
fingerprint: string;
|
||||
};
|
||||
export declare type SsrcGroup = {
|
||||
semantics: string;
|
||||
sources: number[];
|
||||
};
|
||||
export declare type Candidate = {
|
||||
generation: string;
|
||||
component: string;
|
||||
protocol: string;
|
||||
port: string;
|
||||
ip: string;
|
||||
foundation: string;
|
||||
id: string;
|
||||
priority: string;
|
||||
type: string;
|
||||
network: string;
|
||||
'rel-addr': string;
|
||||
'rel-port': string;
|
||||
sdpString?: string;
|
||||
};
|
||||
export declare type JoinGroupCallPayload = {
|
||||
ufrag: string;
|
||||
pwd: string;
|
||||
fingerprints: Fingerprint[];
|
||||
ssrc?: number;
|
||||
'ssrc-groups'?: SsrcGroup[];
|
||||
};
|
||||
export declare type P2pParsedSdp = JoinGroupCallPayload & {
|
||||
audioExtmap: RTPExtension[];
|
||||
videoExtmap: RTPExtension[];
|
||||
screencastExtmap: RTPExtension[];
|
||||
audioPayloadTypes: P2PPayloadType[];
|
||||
videoPayloadTypes: P2PPayloadType[];
|
||||
screencastPayloadTypes: P2PPayloadType[];
|
||||
};
|
||||
export interface RTPExtension {
|
||||
id: number;
|
||||
uri: string;
|
||||
}
|
||||
export interface RTCPFeedbackParam {
|
||||
type: string;
|
||||
subtype?: string;
|
||||
}
|
||||
export interface PayloadType {
|
||||
id: number;
|
||||
name: string;
|
||||
clockrate: number;
|
||||
channels: number;
|
||||
parameters?: Record<string, string | number>;
|
||||
'rtcp-fbs'?: RTCPFeedbackParam[];
|
||||
}
|
||||
export interface GroupCallTransport {
|
||||
candidates: Candidate[];
|
||||
pwd: string;
|
||||
ufrag: string;
|
||||
fingerprints: Fingerprint[];
|
||||
'rtcp-mux': boolean;
|
||||
xmlns: string;
|
||||
}
|
||||
export interface GroupCallConnectionData {
|
||||
transport: GroupCallTransport;
|
||||
audio: {
|
||||
'payload-types': PayloadType[];
|
||||
'rtp-hdrexts': RTPExtension[];
|
||||
};
|
||||
video: {
|
||||
endpoint: string;
|
||||
'payload-types': PayloadType[];
|
||||
'rtp-hdrexts': RTPExtension[];
|
||||
server_sources: number[];
|
||||
};
|
||||
stream?: boolean;
|
||||
}
|
||||
export interface ApiPhoneCallConnection {
|
||||
username: string;
|
||||
password: string;
|
||||
isTurn?: boolean;
|
||||
isStun?: boolean;
|
||||
ip: string;
|
||||
ipv6: string;
|
||||
port: number;
|
||||
}
|
||||
export interface ApiCallProtocol {
|
||||
libraryVersions: string[];
|
||||
minLayer: number;
|
||||
maxLayer: number;
|
||||
isUdpP2p?: boolean;
|
||||
isUdpReflector?: boolean;
|
||||
}
|
||||
143
src/lib/secret-sauce/types.ts
Normal file
143
src/lib/secret-sauce/types.ts
Normal file
@ -0,0 +1,143 @@
|
||||
import { P2PPayloadType } from './p2pMessage';
|
||||
|
||||
export interface GroupCallParticipant {
|
||||
isSelf?: boolean;
|
||||
isMuted?: boolean;
|
||||
isLeft?: boolean;
|
||||
isUser?: boolean;
|
||||
canSelfUnmute?: boolean;
|
||||
hasJustJoined?: boolean;
|
||||
isVideoJoined?: boolean;
|
||||
isMutedByMe?: boolean;
|
||||
isVolumeByAdmin?: boolean;
|
||||
isMin?: boolean;
|
||||
isVersioned?: boolean;
|
||||
source: number;
|
||||
date: Date;
|
||||
id: string;
|
||||
volume?: number;
|
||||
about?: string;
|
||||
video?: GroupCallParticipantVideo;
|
||||
presentation?: GroupCallParticipantVideo;
|
||||
raiseHandRating?: string;
|
||||
|
||||
hasAudioStream?: boolean;
|
||||
hasVideoStream?: boolean;
|
||||
hasPresentationStream?: boolean;
|
||||
amplitude?: number;
|
||||
}
|
||||
|
||||
export type GroupCallConnectionState = 'connecting' | 'connected' | 'disconnected' | 'reconnecting' | 'discarded';
|
||||
|
||||
export interface GroupCallParticipantVideo {
|
||||
endpoint: string;
|
||||
isPaused?: boolean;
|
||||
sourceGroups: SsrcGroup[];
|
||||
audioSource?: number;
|
||||
}
|
||||
|
||||
export type Fingerprint = {
|
||||
hash: string;
|
||||
setup: string;
|
||||
fingerprint: string;
|
||||
};
|
||||
|
||||
export type SsrcGroup = {
|
||||
semantics: string;
|
||||
sources: number[];
|
||||
};
|
||||
|
||||
export type Candidate = {
|
||||
generation: string;
|
||||
component: string;
|
||||
protocol: string;
|
||||
port: string;
|
||||
ip: string;
|
||||
foundation: string;
|
||||
id: string;
|
||||
priority: string;
|
||||
type: string;
|
||||
network: string;
|
||||
'rel-addr': string;
|
||||
'rel-port': string;
|
||||
|
||||
sdpString?: string; // Used for P2P
|
||||
};
|
||||
|
||||
export type JoinGroupCallPayload = {
|
||||
ufrag: string;
|
||||
pwd: string;
|
||||
fingerprints: Fingerprint[];
|
||||
ssrc?: number;
|
||||
'ssrc-groups'?: SsrcGroup[];
|
||||
};
|
||||
|
||||
export type P2pParsedSdp = JoinGroupCallPayload & {
|
||||
audioExtmap: RTPExtension[];
|
||||
videoExtmap: RTPExtension[];
|
||||
screencastExtmap: RTPExtension[];
|
||||
audioPayloadTypes: P2PPayloadType[];
|
||||
videoPayloadTypes: P2PPayloadType[];
|
||||
screencastPayloadTypes: P2PPayloadType[];
|
||||
};
|
||||
|
||||
export interface RTPExtension {
|
||||
id: number;
|
||||
uri: string;
|
||||
}
|
||||
|
||||
export interface RTCPFeedbackParam {
|
||||
type: string;
|
||||
subtype?: string;
|
||||
}
|
||||
|
||||
export interface PayloadType {
|
||||
id: number;
|
||||
name: string;
|
||||
clockrate: number;
|
||||
channels: number;
|
||||
parameters?: Record<string, string | number>;
|
||||
'rtcp-fbs'?: RTCPFeedbackParam[];
|
||||
}
|
||||
|
||||
export interface GroupCallTransport {
|
||||
candidates: Candidate[];
|
||||
pwd: string;
|
||||
ufrag: string;
|
||||
fingerprints: Fingerprint[];
|
||||
'rtcp-mux': boolean;
|
||||
xmlns: string;
|
||||
}
|
||||
|
||||
export interface GroupCallConnectionData {
|
||||
transport: GroupCallTransport;
|
||||
audio: {
|
||||
'payload-types': PayloadType[];
|
||||
'rtp-hdrexts': RTPExtension[];
|
||||
};
|
||||
video: {
|
||||
endpoint: string;
|
||||
'payload-types': PayloadType[];
|
||||
'rtp-hdrexts': RTPExtension[];
|
||||
server_sources: number[];
|
||||
};
|
||||
stream?: boolean;
|
||||
}
|
||||
|
||||
export interface ApiPhoneCallConnection {
|
||||
username: string;
|
||||
password: string;
|
||||
isTurn?: boolean;
|
||||
isStun?: boolean;
|
||||
ip: string;
|
||||
ipv6: string;
|
||||
port: number;
|
||||
}
|
||||
|
||||
export interface ApiCallProtocol {
|
||||
libraryVersions: string[];
|
||||
minLayer: number;
|
||||
maxLayer: number;
|
||||
isUdpP2p?: boolean;
|
||||
isUdpReflector?: boolean;
|
||||
}
|
||||
10
src/lib/secret-sauce/utils.d.ts
vendored
10
src/lib/secret-sauce/utils.d.ts
vendored
@ -1,10 +0,0 @@
|
||||
import { P2PPayloadType } from './p2pMessage';
|
||||
import type { PayloadType } from './types';
|
||||
export declare function toTelegramSource(source: number): number;
|
||||
export declare function fromTelegramSource(source: number): number;
|
||||
export declare function getAmplitude(array: Uint8Array, scale?: number): number;
|
||||
export declare function p2pPayloadTypeToConference(p: P2PPayloadType): PayloadType;
|
||||
export declare const THRESHOLD = 0.1;
|
||||
export declare const IS_SCREENSHARE_SUPPORTED: boolean;
|
||||
export declare const IS_ECHO_CANCELLATION_SUPPORTED: boolean | undefined;
|
||||
export declare const IS_NOISE_SUPPRESSION_SUPPORTED: any;
|
||||
47
src/lib/secret-sauce/utils.ts
Normal file
47
src/lib/secret-sauce/utils.ts
Normal file
@ -0,0 +1,47 @@
|
||||
import type { P2PPayloadType } from './p2pMessage';
|
||||
import type { PayloadType } from './types';
|
||||
|
||||
/// NOTE: telegram returns sign source, while webrtc uses unsign source internally
|
||||
/// unsign => sign
|
||||
export function toTelegramSource(source: number) {
|
||||
// eslint-disable-next-line no-bitwise
|
||||
return source << 0;
|
||||
}
|
||||
|
||||
/// NOTE: telegram returns sign source, while webrtc uses unsign source internally
|
||||
/// sign => unsign
|
||||
export function fromTelegramSource(source: number) {
|
||||
// eslint-disable-next-line no-bitwise
|
||||
return source >>> 0;
|
||||
}
|
||||
|
||||
export function getAmplitude(array: Uint8Array, scale = 3) {
|
||||
if (!array) return 0;
|
||||
|
||||
const { length } = array;
|
||||
let total = 0;
|
||||
for (let i = 0; i < length; i++) {
|
||||
total += array[i] * array[i];
|
||||
}
|
||||
const rms = Math.sqrt(total / length) / 255;
|
||||
|
||||
return Math.min(1, rms * scale);
|
||||
}
|
||||
|
||||
export function p2pPayloadTypeToConference(p: P2PPayloadType): PayloadType {
|
||||
return {
|
||||
id: p.id,
|
||||
name: p.name,
|
||||
'rtcp-fbs': p.feedbackTypes,
|
||||
clockrate: p.clockrate,
|
||||
parameters: p.parameters,
|
||||
channels: p.channels,
|
||||
};
|
||||
}
|
||||
|
||||
export const THRESHOLD = 0.1;
|
||||
|
||||
export const IS_SCREENSHARE_SUPPORTED = 'getDisplayMedia' in (navigator?.mediaDevices || {});
|
||||
export const IS_ECHO_CANCELLATION_SUPPORTED = navigator?.mediaDevices?.getSupportedConstraints().echoCancellation;
|
||||
// @ts-ignore
|
||||
export const IS_NOISE_SUPPRESSION_SUPPORTED = navigator?.mediaDevices?.getSupportedConstraints().noiseSuppression;
|
||||
Loading…
x
Reference in New Issue
Block a user