client volumes

Reviewed-on: #9

client/app/assets/styles/main.scss

@@ -31,17 +31,17 @@ body {
 
 .p-select-overlay {
   /* Force dropdown width to match computed min-width from PrimeVue internals. */
-  width: 0;
+  width: 0 !important;
 }
 
 .p-select-label {
-  width: 0;
-  overflow: hidden;
-  text-overflow: ellipsis;
+  width: 0 !important;
+  overflow: hidden !important;
+  text-overflow: ellipsis !important;
 }
 
 .p-select-option-label {
-  min-width: 0;
-  overflow: hidden;
-  text-overflow: ellipsis;
+  min-width: 0 !important;
+  overflow: hidden !important;
+  text-overflow: ellipsis !important;
 }

client/app/components.d.ts

@@ -16,7 +16,6 @@ declare module 'vue' {
     PrimeDivider: typeof import('primevue/divider')['default']
     PrimeFloatLabel: typeof import('primevue/floatlabel')['default']
     PrimeInputText: typeof import('primevue/inputtext')['default']
-    PrimeMenu: typeof import('primevue/menu')['default']
     PrimePassword: typeof import('primevue/password')['default']
     PrimeProgressBar: typeof import('primevue/progressbar')['default']
     PrimeScrollPanel: typeof import('primevue/scrollpanel')['default']

client/app/components/ClientRow.vue

@@ -7,7 +7,12 @@
     }"
   >
     <div class="p-3 flex items-center gap-3" @click="toggleExpand">
-      <PrimeAvatar size="small">
+      <PrimeAvatar
+        size="small"
+        :class="{
+          'outline-1 outline-primary outline-offset-2': speaking,
+        }"
+      >
         <template #icon>
           <User :size="20" />
         </template>
@@ -27,7 +32,7 @@
         <PrimeBadge v-if="isMe" severity="secondary" value="You" size="small" class="shrink-0" />
       </div>
 
-      <Component :is="expanded ? ChevronUp : ChevronDown" v-if="!isMe" :size="20" />
+      <Component :is="expanded ? ChevronUp : ChevronDown" v-if="!isMe" :size="20" class="text-muted-color" />
     </div>
 
     <CollapseTransition v-if="!isMe">
@@ -61,7 +66,7 @@ const props = defineProps<{
 }>()
 
 const { outputMuted } = useApp()
-const { consumers: allConsumers, micProducer } = useMediasoup()
+const { consumers: allConsumers, micProducer, speakingClients } = useMediasoup()
 const { me } = useClients()
 const { show } = useFullscreenVideo()
 
@@ -94,6 +99,10 @@ const audioTrack = computed(() => {
   return audioConsumer.value?.track
 })
 
+const speaking = computed(() => {
+  return speakingClients.value.find(speaker => speaker.clientId === props.client.socketId)
+})
+
 const audioConsumerPaused = ref(false)
 
 const inputMuted = computed(() => {

client/app/composables/use-app.ts

@@ -12,7 +12,17 @@ export const useApp = createGlobalState(() => {
   const ready = ref(false)
   const isTauri = computed(() => '__TAURI_INTERNALS__' in window)
   const commitSha = __COMMIT_SHA__
-  const version = computedAsync(() => isTauri.value ? getVersion() : 'web', '-')
+  const version = computedAsync(() => {
+    if (import.meta.dev) {
+      return 'dev'
+    }
+    else if (isTauri.value) {
+      return getVersion()
+    }
+    else {
+      return 'web'
+    }
+  }, '-')
 
   const inputMuted = computed(() => {
     return !!mediasoup.micProducer.value?.paused

client/app/composables/use-fullscreen-video.ts

@@ -29,11 +29,25 @@ export const useFullscreenVideo = createGlobalState(() => {
       videoEl.value = el
     }
 
+    stream.getTracks().forEach(t =>
+      t.addEventListener('ended', hide),
+    )
+    videoEl.value.addEventListener('ended', hide)
+
     await videoEl.value.requestFullscreen()
   }
 
   function hide() {
+    if (!videoEl.value)
+      return
+
+    (videoEl.value.srcObject as MediaStream).getTracks().forEach(t =>
+      t.removeEventListener('ended', hide),
+    )
+    videoEl.value.removeEventListener('ended', hide)
+
     videoEl.value?.remove()
+    videoEl.value = undefined
   }
 
   useEventListener(document, 'fullscreenchange', () => {

client/app/composables/use-mediasoup.ts

@@ -1,6 +1,8 @@
+import type { SpeakingClient } from '#shared/types'
 import type { MediaKind, ProducerOptions } from 'mediasoup-client/types'
 import { createSharedComposable } from '@vueuse/core'
 import * as mediasoupClient from 'mediasoup-client'
+import { shallowRef } from 'vue'
 import { useDevices } from '~/composables/use-devices'
 import { usePreferences } from '~/composables/use-preferences'
 import { useSignaling } from '~/composables/use-signaling'
@@ -40,6 +42,8 @@ export const useMediasoup = createSharedComposable(() => {
   const consumers = shallowRef<Map<string, mediasoupClient.types.Consumer>>(new Map())
   const producers = shallowRef<Map<string, mediasoupClient.types.Producer>>(new Map())
 
+  const speakingClients = shallowRef<SpeakingClient[]>([])
+
   watch(signaling.socket, (socket) => {
     if (!socket)
       return
@@ -227,6 +231,10 @@ export const useMediasoup = createSharedComposable(() => {
 
       triggerRef(consumers)
     })
+
+    socket.on('speakingPeers', (value: SpeakingClient[]) => {
+      speakingClients.value = value
+    })
   }, { immediate: true, flush: 'sync' })
 
   async function enableProducer(type: ProducerType, options: ProducerOptions) {
@@ -317,7 +325,7 @@ export const useMediasoup = createSharedComposable(() => {
     if (!device.value)
       return
 
-    const stream = await getShareStream()
+    const stream = await getShareStream(preferences.shareFps.value)
 
     const track = stream.getVideoTracks()[0]
 
@@ -424,6 +432,7 @@ export const useMediasoup = createSharedComposable(() => {
     init,
     consumers,
     producers,
+    speakingClients,
     sendTransport,
     recvTransport,
     rtpCapabilities,

client/app/composables/use-preferences.ts

@@ -19,6 +19,8 @@ export const usePreferences = createGlobalState(() => {
   const noiseSuppression = useLocalStorage('NOISE_SUPPRESSION', true)
   const echoCancellation = useLocalStorage('ECHO_CANCELLATION', true)
 
+  const shareFps = useLocalStorage('SHARE_FPS', 30)
+
   const toggleInputHotkey = ref<SyncedPreferences['toggleInputHotkey']>('')
   const toggleOutputHotkey = ref<SyncedPreferences['toggleOutputHotkey']>('')
 
@@ -57,6 +59,7 @@ export const usePreferences = createGlobalState(() => {
     autoGainControl,
     noiseSuppression,
     echoCancellation,
+    shareFps,
     toggleInputHotkey,
     toggleOutputHotkey,
     inputDeviceExist,

client/app/layouts/default.vue

@@ -4,24 +4,24 @@
       class="flex items-center justify-between gap-2 rounded-xl p-3 bg-surface-950"
     >
       <div class="inline-flex items-center gap-3">
-        <PrimeBadge v-if="isTauri" class="opacity-50" severity="secondary" :value="version" size="small" />
+        <PrimeBadge class="opacity-50" severity="secondary" :value="version" size="small" />
         <PrimeBadge :severity="connected ? 'success' : 'danger' " />
       </div>
 
       <PrimeButtonGroup class="ml-auto">
-        <PrimeButton outlined @click="toggleInput">
+        <PrimeButton :severity="inputMuted ? 'info' : undefined" outlined @click="toggleInput">
           <template #icon>
             <Component :is="inputMuted ? MicOff : Mic" />
           </template>
         </PrimeButton>
-        <PrimeButton outlined @click="toggleOutput">
+        <PrimeButton :severity="outputMuted ? 'info' : undefined" outlined @click="toggleOutput">
           <template #icon>
             <Component :is="outputMuted ? VolumeOff : Volume2" />
           </template>
         </PrimeButton>
       </PrimeButtonGroup>
 
-      <PrimeButton outlined @click="toggleShare">
+      <PrimeButton :severity="sharingEnabled ? 'success' : undefined" outlined @click="toggleShare">
         <template #icon>
           <Component :is="sharingEnabled ? ScreenShareOff : ScreenShare" />
         </template>
@@ -72,7 +72,6 @@ import {
 } from 'lucide-vue-next'
 
 const {
-  isTauri,
   version,
   clients,
   inputMuted,

client/app/pages/preferences.vue

@@ -47,6 +47,18 @@
     <!--      <label for="outputDevice">Output device</label> -->
     <!--    </PrimeFloatLabel> -->
 
+    <PrimeDivider align="left">
+      Video
+    </PrimeDivider>
+
+    <div>
+      <div class="text-sm mb-2">
+        Share FPS
+      </div>
+
+      <PrimeSelectButton v-model="shareFps" :options="[5, 30, 60]" fluid size="small" />
+    </div>
+
     <template v-if="isTauri">
       <PrimeDivider align="left">
         Hotkeys
@@ -117,6 +129,7 @@ const {
   toggleOutputHotkey,
   inputDeviceExist,
   outputDeviceExist,
+  shareFps,
 } = usePreferences()
 
 const setupToggleInputHotkey = (event: KeyboardEvent) => setupHotkey(event, toggleInputHotkey)

client/shared/types.ts

@@ -8,3 +8,8 @@ export interface ChadClient {
 }
 
 export type UpdatedClient = Omit<ChadClient, 'socketId' | 'userId' | 'isMe'>
+
+export interface SpeakingClient {
+  clientId: ChadClient['socketId']
+  volume: number
+}

client/src-tauri/tauri.conf.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
   "productName": "chad",
-  "version": "0.2.17",
+  "version": "0.2.21",
   "identifier": "xyz.koptilnya.chad",
   "build": {
     "frontendDist": "../.output/public",

server/plugins/socket.ts

@@ -22,8 +22,8 @@ export default fp<Partial<ServerOptions>>(
       await fastify.io.close()
     })
 
-    fastify.ready(() => {
-      registerWebrtcSocket(fastify.io, fastify.mediasoupRouter)
+    fastify.ready(async () => {
+      await registerWebrtcSocket(fastify.io, fastify.mediasoupRouter)
     })
   },
   { name: 'socket-io', dependencies: ['mediasoup-worker', 'mediasoup-router'] },

server/socket/webrtc.ts

@@ -1,6 +1,7 @@
 import type { types } from 'mediasoup'
 import type { Server as SocketServer } from 'socket.io'
 import type {
+  ChadClient,
   Namespace,
   SomeSocket,
 } from '../types/webrtc.ts'
@@ -8,9 +9,39 @@ import { consola } from 'consola'
 import prisma from '../prisma/client.ts'
 import { socketToClient } from '../utils/socket-to-client.ts'
 
-export default function (io: SocketServer, router: types.Router) {
+export default async function (io: SocketServer, router: types.Router) {
   const namespace: Namespace = io.of('/webrtc')
 
+  const audioLevelObserver = await router.createAudioLevelObserver({
+    maxEntries: 10,
+    threshold: -80,
+    interval: 800,
+  })
+
+  const activeSpeakerObserver = await router.createActiveSpeakerObserver()
+
+  audioLevelObserver.on('volumes', async (volumes: types.AudioLevelObserverVolume[]) => {
+    namespace.emit('speakingPeers', volumes.map(({ producer, volume }) => {
+      const { socketId } = producer.appData as { socketId: ChadClient['socketId'] }
+
+      return {
+        clientId: socketId,
+        volume,
+      }
+    }))
+  })
+
+  audioLevelObserver.on('silence', () => {
+    namespace.emit('speakingPeers', [])
+    namespace.emit('activeSpeaker', undefined)
+  })
+
+  activeSpeakerObserver.on('dominantspeaker', ({ producer }) => {
+    const { socketId } = producer.appData as { socketId: ChadClient['socketId'] }
+
+    namespace.emit('activeSpeaker', socketId)
+  })
+
   namespace.on('connection', async (socket) => {
     consola.info('[WebRtc]', 'Client connected', socket.id)
 
@@ -182,8 +213,8 @@ export default function (io: SocketServer, router: types.Router) {
           )
         }
 
-        // TODO: Add into the AudioLevelObserver and ActiveSpeakerObserver.
-        // https://github.com/versatica/mediasoup-demo/blob/v3/server/lib/Room.js#L1276
+        await audioLevelObserver.addProducer({ producerId: producer.id })
+        await activeSpeakerObserver.addProducer({ producerId: producer.id })
       }
       catch (error) {
         if (error instanceof Error) {

server/types/webrtc.ts

@@ -118,6 +118,8 @@ export interface ServerToClientEvents {
   consumerResumed: (arg: { consumerId: string }) => void
   consumerScore: (arg: { consumerId: string, score: types.ConsumerScore }) => void
   clientChanged: (clientId: ChadClient['socketId'], client: ChadClient) => void
+  speakingPeers: (arg: { clientId: ChadClient['socketId'], volume: types.AudioLevelObserverVolume['volume'] }[]) => void
+  activeSpeaker: (clientId?: ChadClient['socketId']) => void
 }
 
 export interface InterServerEvent {}