показывать себя в превью

Reviewed-on: #9

client/app/assets/styles/main.scss

@@ -31,17 +31,17 @@ body {
 
 .p-select-overlay {
   /* Force dropdown width to match computed min-width from PrimeVue internals. */
-  width: 0;
+  width: 0 !important;
 }
 
 .p-select-label {
-  width: 0;
+  width: 0 !important;
-  overflow: hidden;
+  overflow: hidden !important;
-  text-overflow: ellipsis;
+  text-overflow: ellipsis !important;
 }
 
 .p-select-option-label {
-  min-width: 0;
+  min-width: 0 !important;
-  overflow: hidden;
+  overflow: hidden !important;
-  text-overflow: ellipsis;
+  text-overflow: ellipsis !important;
 }

client/app/components.d.ts

@@ -13,19 +13,18 @@ declare module 'vue' {
     PrimeButton: typeof import('primevue/button')['default']
     PrimeButtonGroup: typeof import('primevue/buttongroup')['default']
     PrimeCard: typeof import('primevue/card')['default']
+    PrimeDivider: typeof import('primevue/divider')['default']
     PrimeFloatLabel: typeof import('primevue/floatlabel')['default']
     PrimeInputText: typeof import('primevue/inputtext')['default']
     PrimePassword: typeof import('primevue/password')['default']
+    PrimeProgressBar: typeof import('primevue/progressbar')['default']
     PrimeScrollPanel: typeof import('primevue/scrollpanel')['default']
+    PrimeSelect: typeof import('primevue/select')['default']
     PrimeSelectButton: typeof import('primevue/selectbutton')['default']
     PrimeSlider: typeof import('primevue/slider')['default']
-    PrimeTab: typeof import('primevue/tab')['default']
+    PrimeTag: typeof import('primevue/tag')['default']
-    PrimeTabList: typeof import('primevue/tablist')['default']
-    PrimeTabPanel: typeof import('primevue/tabpanel')['default']
-    PrimeTabPanels: typeof import('primevue/tabpanels')['default']
-    PrimeTabs: typeof import('primevue/tabs')['default']
-    PrimeTextarea: typeof import('primevue/textarea')['default']
     PrimeToast: typeof import('primevue/toast')['default']
+    PrimeToggleSwitch: typeof import('primevue/toggleswitch')['default']
     RouterLink: typeof import('vue-router')['RouterLink']
     RouterView: typeof import('vue-router')['RouterView']
   }

client/app/components/ClientRow.vue

@@ -6,19 +6,29 @@
       'bg-surface-800': expanded,
     }"
   >
-    <div class="p-3 flex items-center gap-3" @click="toggleExpand">
+    <div class="p-3" @click="toggleExpand">
-      <PrimeAvatar size="small">
+      <div class="flex items-center gap-3">
-        <template #icon>
+        <PrimeAvatar
-          <User :size="20" />
+          size="small"
-        </template>
+          class="shrink-0"
-      </PrimeAvatar>
+          :class="{
+            'outline-1 outline-primary outline-offset-2': speaking,
+          }"
+        >
+          <template #icon>
+            <User :size="20" />
+          </template>
+        </PrimeAvatar>
 
-      <div class="flex-1 overflow-hidden text-sm leading-5 font-medium text-color whitespace-nowrap overflow-ellipsis">
+        <p class="flex-1 text-sm leading-5 font-medium text-color truncate w-0">
-        {{ client.displayName || client.username }}
+          {{ client.displayName || client.username }}
+        </p>
+
+        <Component :is="expanded ? ChevronUp : ChevronDown" v-if="!isMe" :size="20" class="text-muted-color" />
       </div>
 
-      <div class="flex align-center gap-1">
+      <div v-if="hasBadges" class="flex justify-end align-center gap-1 mt-2">
-        <PrimeBadge v-if="!!shareConsumer" v-tooltip.top="'Watch'" severity="success" value="Streaming" size="small" @click.stop="watchStream" />
+        <PrimeBadge v-if="streaming" v-tooltip.top="'Watch'" severity="success" value="Streaming" size="small" @click.stop="watchStream" />
 
         <PrimeBadge v-if="premuted" severity="danger" value="Muted" size="small" />
         <PrimeBadge v-else-if="client.outputMuted" severity="info" value="No sound" size="small" />
@@ -26,8 +36,6 @@
 
         <PrimeBadge v-if="isMe" severity="secondary" value="You" size="small" class="shrink-0" />
       </div>
-
-      <Component :is="expanded ? ChevronUp : ChevronDown" v-if="!isMe" :size="20" />
     </div>
 
     <CollapseTransition v-if="!isMe">
@@ -52,7 +60,6 @@
 
 <script setup lang="ts">
 import type { ChadClient } from '#shared/types'
-import { useLocalStorage } from '@vueuse/core'
 import { ChevronDown, ChevronUp, User } from 'lucide-vue-next'
 import CollapseTransition from '~/components/CollapseTransition.vue'
 
@@ -67,34 +74,34 @@ const { show } = useFullscreenVideo()
 
 const expanded = ref(false)
 
-const volume = useLocalStorage<number>(computed(() => `CLIENT_VOLUME_${props.client.userId}`), 100, { writeDefaults: false })
+const {
-const premuted = useLocalStorage<boolean>(computed(() => `CLIENT_PREMUTED_${props.client.userId}`), false, { writeDefaults: false })
+  volume,
+  premuted,
+  speaking,
+  audioConsumers,
+  videoConsumers,
+  shareConsumers,
+  streaming,
+} = useClient(toRef(() => props.client.socketId))
 
 const isMe = computed(() => {
   return me.value && props.client.userId === me.value.userId
 })
 
-const consumers = computed(() => {
-  return allConsumers.value.values().filter(consumer => consumer.appData.socketId === props.client.socketId).toArray()
-})
-
 const audioConsumer = computed(() => {
-  return consumers.value.find(consumer => consumer.track.kind === 'audio')
+  return audioConsumers.value[0]
-})
-
-const videoConsumers = computed(() => {
-  return consumers.value.filter(consumer => consumer.track.kind === 'video')
-})
-
-const shareConsumer = computed(() => {
-  return videoConsumers.value.find(consumer => consumer.appData.source === 'share')
 })
 
 const audioTrack = computed(() => {
-  return audioConsumer.value?.track
+  return audioConsumer.value?.raw.track
 })
 
-const audioConsumerPaused = ref(false)
+const audioConsumerPaused = computed(() => {
+  if (Object.keys(allConsumers.value).length === 0)
+    return false
+
+  return audioConsumer.value?.paused ?? false
+})
 
 const inputMuted = computed(() => {
   if (isMe.value)
@@ -103,8 +110,12 @@ const inputMuted = computed(() => {
   return premuted.value || audioConsumerPaused.value
 })
 
-watch(allConsumers, () => {
+const hasBadges = computed(() => {
-  audioConsumerPaused.value = audioConsumer.value?.paused ?? false
+  return streaming.value
+    || premuted.value
+    || inputMuted.value
+    || props.client.outputMuted
+    || isMe.value
 })
 
 const { setGain } = useAudioContext(audioTrack)
@@ -121,9 +132,11 @@ function toggleExpand() {
 }
 
 function watchStream() {
-  if (!shareConsumer.value)
+  if (!streaming.value)
     return
 
-  show(new MediaStream([shareConsumer.value.track]))
+  const consumer = [...videoConsumers.value, ...shareConsumers.value][0]!
+
+  show(new MediaStream([consumer.raw.track]))
 }
 </script>

client/app/components/Debug/Consumer.vue

@@ -0,0 +1,25 @@
+<template>
+  <div class="text-sm overflow-x-auto">
+    <p class="text-muted-color">
+      {{ consumer.id }}
+    </p>
+
+    <p>paused: {{ consumer.paused }}</p>
+
+    <p v-for="[key, value] in appData" :key="key">
+      {{ key }}: {{ value }}
+    </p>
+  </div>
+</template>
+
+<script setup lang="ts">
+import type { Consumer } from 'mediasoup-client/types'
+
+const props = defineProps<{
+  consumer: Consumer
+}>()
+
+const appData = computed(() => {
+  return Object.entries(props.consumer.appData)
+})
+</script>

client/app/components/FullscreenGallery.vue

@@ -0,0 +1,20 @@
+<template>
+  <Teleport to="body">
+    <div ref="root" class="fullscreen-gallery">
+      {{ videoConsumers.length + shareConsumers.length }}
+    </div>
+  </Teleport>
+</template>
+
+<script setup lang="ts">
+import { useFullscreen } from '@vueuse/core'
+
+const rootRef = useTemplateRef('root')
+
+const { enter } = useFullscreen(rootRef)
+const { videoConsumers, shareConsumers } = useMediasoup()
+
+onMounted(() => {
+  // enter()
+})
+</script>

client/app/components/FullscreenGallery/Card.vue

@@ -0,0 +1,13 @@
+<template>
+  <div class="fullscreen-gallery-card">
+    sasd
+  </div>
+</template>
+
+<script setup lang="ts">
+
+</script>
+
+<style>
+
+</style>

client/app/components/Gallery/Card.vue

@@ -0,0 +1,49 @@
+<template>
+  <div
+    class="relative rounded overflow-hidden flex items-center justify-center"
+    :class="{
+      'group cursor-pointer hover:outline outline-primary': !isMe,
+    }"
+    @click="watch"
+  >
+    <video :srcObject="stream" muted autoplay />
+
+    <PrimeTag
+      severity="secondary"
+      class="absolute bottom-2 text-sm opacity-70"
+      :class="{
+        'group-hover:opacity-100': !isMe,
+      }"
+      rounded
+    >
+      {{ isMe ? 'You' : client.username }}
+    </PrimeTag>
+  </div>
+</template>
+
+<script setup lang="ts">
+import type { ChadClient } from '#shared/types'
+
+const props = defineProps<{
+  client: ChadClient
+  stream: MediaStream
+}>()
+
+const { me } = useClients()
+const fullscreenVideo = useFullscreenVideo()
+
+const isMe = computed(() => {
+  return props.client.socketId === me.value?.socketId
+})
+
+function watch() {
+  if (isMe.value)
+    return
+
+  fullscreenVideo.show(props.stream)
+}
+</script>
+
+<style>
+
+</style>

client/app/components/chat/ChatEditor.vue

@@ -1,28 +0,0 @@
-<template>
-  <div class="chat-editor">
-    <PrimeTextarea v-model="msg" />
-    <PrimeButton :disabled="!msg" @click="handleSend()">
-      Send
-    </PrimeButton>
-  </div>
-</template>
-
-<script lang="ts" setup>
-interface Emits {
-  (e: 'send', msg: string): void
-}
-
-const emit = defineEmits<Emits>()
-
-const msg = ref<string | undefined>()
-
-function handleSend() {
-  emit('send', msg.value!)
-
-  msg.value = ''
-}
-</script>
-
-<style lang="scss">
-
-</style>

client/app/components/chat/ChatMessage.vue

@@ -1,27 +0,0 @@
-<template>
-  <PrimeCard>
-    <template #header>
-      <span class="font-bold">
-        {{ username }}
-      </span>
-    </template>
-    <template #content>
-      {{ message }}
-    </template>
-    <template #footer>
-      {{ createdAt }}
-    </template>
-  </PrimeCard>
-</template>
-
-<script lang="ts" setup>
-interface Props {
-  username: string
-  message: string
-  createdAt: Date
-}
-
-defineProps<Props>()
-</script>
-
-<style lang="scss"></style>

client/app/components/chat/ChatWidget.vue

@@ -1,33 +0,0 @@
-<template>
-  <div class="chat-tabs">
-    <div class="chat-tabs__messages">
-      <ChatMessage v-for="msg in messages" :key="msg.id" :created-at="msg.createdAt" :username="msg.username" :message="msg.message" />
-    </div>
-
-    <PrimeTabs :value="channels[0]">
-      <PrimeTabList>
-        <PrimeTab v-for="channel in channels" :key="channel" :value="channel">
-          Channel: {{ channel }}
-        </PrimeTab>
-      </PrimeTabList>
-      <PrimeTabPanels>
-        <PrimeTabPanel :value="channel">
-          <ChatEditor />
-        </PrimeTabPanel>
-      </PrimeTabPanels>
-    </PrimeTabs>
-  </div>
-</template>
-
-<script lang="ts" setup>
-const {
-  channel,
-
-  messages,
-  channels,
-} = useChat()
-</script>
-
-<style lang="scss" scoped>
-
-</style>

client/app/composables/use-app.ts

@@ -12,7 +12,17 @@ export const useApp = createGlobalState(() => {
   const ready = ref(false)
   const isTauri = computed(() => '__TAURI_INTERNALS__' in window)
   const commitSha = __COMMIT_SHA__
-  const version = computedAsync(() => isTauri.value ? getVersion() : 'web', '-')
+  const version = computedAsync(() => {
+    if (import.meta.dev) {
+      return 'dev'
+    }
+    else if (isTauri.value) {
+      return getVersion()
+    }
+    else {
+      return 'web'
+    }
+  }, '-')
 
   const inputMuted = computed(() => {
     return !!mediasoup.micProducer.value?.paused
@@ -21,28 +31,36 @@ export const useApp = createGlobalState(() => {
 
   const outputMuted = ref(false)
 
+  const videoEnabled = computed(() => {
+    return !!mediasoup.videoProducer.value
+  })
+
   const sharingEnabled = computed(() => {
     return !!mediasoup.shareProducer.value
   })
 
+  const somebodyStreamingVideo = computed(() => {
+    return mediasoup.videoConsumers.value.length > 0 || mediasoup.shareConsumers.value.length > 0
+  })
+
   async function muteInput() {
-    if (inputMuted.value)
+    if (inputMuted.value || !mediasoup.micProducer.value)
       return
 
-    await mediasoup.pauseProducer('microphone')
+    await mediasoup.pauseProducer(mediasoup.micProducer.value)
 
     toast.add({ severity: 'info', summary: 'Microphone muted', closable: false, life: 1000 })
   }
 
   async function unmuteInput() {
-    if (!inputMuted.value)
+    if (!inputMuted.value || !mediasoup.micProducer.value)
       return
 
     if (outputMuted.value) {
       await unmuteOutput()
     }
 
-    await mediasoup.resumeProducer('microphone')
+    await mediasoup.resumeProducer(mediasoup.micProducer.value)
 
     toast.add({ severity: 'info', summary: 'Microphone activated', closable: false, life: 1000 })
   }
@@ -91,12 +109,21 @@ export const useApp = createGlobalState(() => {
       await muteOutput()
   }
 
+  async function toggleVideo() {
+    if (!mediasoup.videoProducer.value) {
+      await mediasoup.enableVideo()
+    }
+    else {
+      await mediasoup.disableProducer(mediasoup.videoProducer.value)
+    }
+  }
+
   async function toggleShare() {
     if (!mediasoup.shareProducer.value) {
       await mediasoup.enableShare()
     }
     else {
-      await mediasoup.disableProducer('share')
+      await mediasoup.disableProducer(mediasoup.shareProducer.value)
     }
   }
 
@@ -111,10 +138,13 @@ export const useApp = createGlobalState(() => {
     muteOutput,
     unmuteOutput,
     toggleOutput,
+    toggleVideo,
     version,
     isTauri,
     commitSha,
     toggleShare,
+    videoEnabled,
     sharingEnabled,
+    somebodyStreamingVideo,
   }
 })

client/app/composables/use-chat.ts

@@ -1,44 +0,0 @@
-import { createGlobalState } from '@vueuse/core'
-
-interface ChatMessage {
-  id: string
-  username: string
-  message: string
-}
-
-interface ChatChannel {
-  id: number
-  name: string
-}
-
-export const useChat = createGlobalState(() => {
-  const messages = ref([
-    {
-      id: '1337',
-      username: 'Yes',
-      message: 'Fisting is 300 bucks',
-      createdAt: Date.now(),
-    },
-  ])
-
-  const channel = ref<number>(0)
-
-  async function sendMsg(channelId: ChatChannel['id'], msg: ChatMessage['message']) {
-    console.log('Trying to send message', channelId, msg)
-  }
-
-  watch(channel, async (id) => {
-    await console.log('Yes', id)
-  }, {
-    immediate: true,
-  })
-
-  return {
-    channel,
-
-    channels,
-    messages,
-
-    sendMsg,
-  }
-})

client/app/composables/use-client.ts

@@ -0,0 +1,52 @@
+import type { ChadClient } from '#shared/types'
+import { useLocalStorage } from '@vueuse/core'
+
+export function useClient(socketId: MaybeRef<ChadClient['socketId']>) {
+  const mediasoup = useMediasoup()
+  const { getClient } = useClients()
+
+  const client = computed(() => getClient(unref(socketId))!)
+
+  const volume = useLocalStorage<number>(computed(() => `CLIENT_VOLUME_${client.value.userId}`), 100, { writeDefaults: false })
+  const premuted = useLocalStorage<boolean>(computed(() => `CLIENT_PREMUTED_${client.value.userId}`), false, { writeDefaults: false })
+
+  const consumers = computed(() => {
+    return Object.values(mediasoup.consumers.value).filter(consumer => consumer.appData.socketId === client.value.socketId)
+  })
+
+  const audioConsumers = computed(() => {
+    return mediasoup.audioConsumers.value.filter(consumer => consumer.appData.socketId === client.value.socketId)
+  })
+
+  const videoConsumers = computed(() => {
+    return mediasoup.videoConsumers.value.filter(consumer => consumer.appData.socketId === client.value.socketId)
+  })
+
+  const shareConsumers = computed(() => {
+    return mediasoup.shareConsumers.value.filter(consumer => consumer.appData.socketId === client.value.socketId)
+  })
+
+  const producers = computed(() => {
+    return Object.values(mediasoup.producers.value).filter(producer => producer.appData.socketId === client.value.socketId)
+  })
+
+  const streaming = computed(() => {
+    return videoConsumers.value.length > 0 || shareConsumers.value.length > 0
+  })
+
+  const speaking = computed(() => {
+    return mediasoup.speakingClients.value.some(speaker => speaker.clientId === client.value.socketId)
+  })
+
+  return {
+    volume,
+    premuted,
+    consumers,
+    producers,
+    audioConsumers,
+    videoConsumers,
+    shareConsumers,
+    streaming,
+    speaking,
+  }
+}

client/app/composables/use-devices.ts

@@ -19,7 +19,16 @@ export const useDevices = createGlobalState(() => {
     })
   }
 
+  ;(async () => {
+    if (permissionGranted.value)
+      return
+
+    await ensurePermissions()
+  })()
+
   return {
+    ensurePermissions,
+    permissionGranted,
     videoInputs: computed<MediaDeviceInfo[]>(() => JSON.parse(JSON.stringify(videoInputs.value))),
     audioInputs: computed<MediaDeviceInfo[]>(() => JSON.parse(JSON.stringify(audioInputs.value))),
     audioOutputs: computed<MediaDeviceInfo[]>(() => JSON.parse(JSON.stringify(audioOutputs.value))),

client/app/composables/use-fullscreen-gallery.ts

@@ -0,0 +1,5 @@
+import { createSharedComposable } from '@vueuse/core'
+
+export const useFullscreenGallery = createSharedComposable(() => {
+  return {}
+})

client/app/composables/use-fullscreen-video.ts

@@ -29,11 +29,25 @@ export const useFullscreenVideo = createGlobalState(() => {
       videoEl.value = el
     }
 
+    stream.getTracks().forEach(t =>
+      t.addEventListener('ended', hide),
+    )
+    videoEl.value.addEventListener('ended', hide)
+
     await videoEl.value.requestFullscreen()
   }
 
   function hide() {
+    if (!videoEl.value)
+      return
+
+    (videoEl.value.srcObject as MediaStream).getTracks().forEach(t =>
+      t.removeEventListener('ended', hide),
+    )
+    videoEl.value.removeEventListener('ended', hide)
+
     videoEl.value?.remove()
+    videoEl.value = undefined
   }
 
   useEventListener(document, 'fullscreenchange', () => {

client/app/composables/use-mediasoup.ts

@@ -1,11 +1,18 @@
+import type { ChadClient, Consumer, Producer } from '#shared/types'
 import type { MediaKind, ProducerOptions } from 'mediasoup-client/types'
 import { createSharedComposable } from '@vueuse/core'
 import * as mediasoupClient from 'mediasoup-client'
+import { shallowRef } from 'vue'
 import { useDevices } from '~/composables/use-devices'
 import { usePreferences } from '~/composables/use-preferences'
 import { useSignaling } from '~/composables/use-signaling'
 
-type ProducerType = 'microphone' | 'camera' | 'share'
+type ProducerType = 'microphone' | 'video' | 'share'
+
+interface SpeakingClient {
+  clientId: ChadClient['socketId']
+  volume: number
+}
 
 const ICE_SERVERS: RTCIceServer[] = [
   { urls: 'stun:stun.l.google.com:19302' },
@@ -33,12 +40,42 @@ export const useMediasoup = createSharedComposable(() => {
   const sendTransport = shallowRef<mediasoupClient.types.Transport>()
   const recvTransport = shallowRef<mediasoupClient.types.Transport>()
 
-  const micProducer = shallowRef<mediasoupClient.types.Producer>()
+  const consumers = ref<Record<Consumer['id'], Consumer>>({})
-  const cameraProducer = shallowRef<mediasoupClient.types.Producer>()
+  const producers = ref<Record<Producer['id'], Producer>>({})
-  const shareProducer = shallowRef<mediasoupClient.types.Producer>()
 
-  const consumers = shallowRef<Map<string, mediasoupClient.types.Consumer>>(new Map())
+  const consumersArray = computed(() => {
-  const producers = shallowRef<Map<string, mediasoupClient.types.Producer>>(new Map())
+    return Object.values(consumers.value)
+  })
+
+  const audioConsumers = computed(() => {
+    return consumersArray.value.filter(consumer => consumer.raw.kind === 'audio')
+  })
+
+  const videoConsumers = computed(() => {
+    return consumersArray.value.filter(consumer => consumer.raw.kind === 'video' && consumer.appData.source !== 'share')
+  })
+
+  const shareConsumers = computed(() => {
+    return consumersArray.value.filter(consumer => consumer.raw.kind === 'video' && consumer.appData.source === 'share')
+  })
+
+  const producersArray = computed(() => {
+    return Object.values(producers.value)
+  })
+
+  const micProducer = computed(() => {
+    return producersArray.value.find(producer => producer.raw.kind === 'audio' && producer.raw.appData.source === 'mic-video')
+  })
+
+  const videoProducer = computed(() => {
+    return producersArray.value.find(producer => producer.raw.kind === 'video' && producer.raw.appData.source !== 'share')
+  })
+
+  const shareProducer = computed(() => {
+    return producersArray.value.find(producer => producer.raw.kind === 'video' && producer.raw.appData.source === 'share')
+  })
+
+  const speakingClients = shallowRef<SpeakingClient[]>([])
 
   watch(signaling.socket, (socket) => {
     if (!socket)
@@ -159,20 +196,35 @@ export const useMediasoup = createSharedComposable(() => {
           producerId,
           kind,
           rtpParameters,
-          streamId: `${socketId}-${appData.source === 'share' ? 'share' : 'mic-webcam'}`,
+          streamId: `${socketId}-${appData.source || 'stream'}`,
           appData: { ...appData, socketId },
         })
 
         if (producerPaused)
           consumer.pause()
 
-        consumer.on('transportclose', () => {
+        consumers.value[consumer.id] = {
-          if (consumers.value.delete(consumer.id))
+          id: consumer.id,
-            triggerRef(consumers)
+          paused: consumer.paused,
+          appData: consumer.appData,
+          raw: markRaw(consumer),
+        }
+
+        consumer.observer.on('resume', () => {
+          consumers.value[consumer.id]!.paused = false
         })
 
-        consumers.value.set(consumer.id, consumer)
+        consumer.observer.on('pause', () => {
-        triggerRef(consumers)
+          consumers.value[consumer.id]!.paused = true
+        })
+
+        consumer.observer.on('close', () => {
+          delete consumers.value[consumer.id]
+        })
+
+        consumer.on('trackended', () => {
+          consumer.close()
+        })
 
         cb()
       },
@@ -183,11 +235,37 @@ export const useMediasoup = createSharedComposable(() => {
       async (
         { consumerId },
       ) => {
-        if (consumers.value.delete(consumerId))
+        const consumer = consumers.value[consumerId]
-          triggerRef(consumers)
+
+        if (!consumer)
+          return
+
+        consumer.raw.close()
       },
     )
 
+    socket.on('consumerPaused', ({ consumerId }) => {
+      const consumer = consumers.value[consumerId]
+
+      if (!consumer)
+        return
+
+      consumer.raw.pause()
+    })
+
+    socket.on('consumerResumed', ({ consumerId }) => {
+      const consumer = consumers.value[consumerId]
+
+      if (!consumer)
+        return
+
+      consumer.raw.resume()
+    })
+
+    socket.on('speakingPeers', (value: SpeakingClient[]) => {
+      speakingClients.value = value
+    })
+
     socket.on('disconnect', () => {
       device.value = undefined
       rtpCapabilities.value = undefined
@@ -198,43 +276,12 @@ export const useMediasoup = createSharedComposable(() => {
       recvTransport.value?.close()
       recvTransport.value = undefined
 
-      micProducer.value = undefined
+      consumers.value = {}
-      cameraProducer.value = undefined
+      producers.value = {}
-      shareProducer.value = undefined
-
-      consumers.value = new Map()
-      producers.value = new Map()
-    })
-
-    socket.on('consumerPaused', ({ consumerId }) => {
-      const consumer = consumers.value.get(consumerId)
-
-      if (!consumer)
-        return
-
-      consumer.pause()
-
-      triggerRef(consumers)
-    })
-
-    socket.on('consumerResumed', ({ consumerId }) => {
-      const consumer = consumers.value.get(consumerId)
-
-      if (!consumer)
-        return
-
-      consumer.resume()
-
-      triggerRef(consumers)
     })
   }, { immediate: true, flush: 'sync' })
 
-  async function enableProducer(type: ProducerType, options: ProducerOptions) {
+  async function createProducer(options: ProducerOptions) {
-    const producer = getProducerByType(type)
-
-    if (producer.value)
-      return
-
     if (!device.value || !sendTransport.value)
       return
 
@@ -244,47 +291,54 @@ export const useMediasoup = createSharedComposable(() => {
     if (!device.value.canProduce(options.track.kind as MediaKind))
       return
 
-    producer.value = await sendTransport.value.produce(options)
+    const producer = await sendTransport.value.produce({ disableTrackOnPause: true, ...options })
 
-    producers.value.set(producer.value.id, producer.value)
+    producers.value[producer.id] = {
-    triggerRef(producers)
+      id: producer.id,
-    triggerRef(producer)
+      paused: producer.paused,
+      appData: producer.appData,
+      raw: markRaw(producer),
+    }
 
-    producer.value.on('transportclose', () => {
+    producer.observer.on('pause', () => {
-      micProducer.value = undefined
+      producers.value[producer.id]!.paused = true
     })
 
-    producer.value.on('trackended', () => {
+    producer.observer.on('resume', () => {
-      disableProducer(type)
+      producers.value[producer.id]!.paused = false
+    })
+
+    producer.observer.on('close', () => {
+      delete producers.value[producer.id]
+    })
+
+    producer.on('trackended', () => {
+      disableProducer(producers.value[producer.id]!)
     })
   }
 
-  async function disableProducer(type: ProducerType) {
+  async function disableProducer(producer: Producer) {
-    const producer = getProducerByType(type)
+    if (!signaling.socket.value)
-
-    if (!signaling.socket.value || !producer.value)
       return
 
-    producers.value.delete(producer.value.id)
-
     try {
-      producer.value.close()
+      producer.raw.close()
 
       await signaling.socket.value.emitWithAck('closeProducer', {
-        producerId: producer.value.id,
+        producerId: producer.id,
       })
     }
     catch {
     }
     finally {
-      triggerRef(producers)
+      delete producers.value[producer.id]
-      triggerRef(producer)
     }
-
-    producer.value = undefined
   }
 
   async function enableMic() {
+    if (micProducer.value)
+      return
+
     const stream = await navigator.mediaDevices.getUserMedia({
       audio: {
         deviceId: { exact: preferences.inputDeviceId.value },
@@ -299,110 +353,125 @@ export const useMediasoup = createSharedComposable(() => {
     if (!track)
       return
 
-    await enableProducer('microphone', {
+    await createProducer({
       track,
+      streamId: 'mic-video',
       codecOptions: {
         opusStereo: true,
         opusDtx: true, // Меньше пакетов летит когда тишина
         opusFec: false, // Фиксит пакет лос
       },
+      appData: {
+        source: 'mic-video',
+      },
     })
   }
 
   async function disableMic() {
-    await disableProducer('microphone')
+    if (!micProducer.value)
+      return
+
+    await disableProducer(micProducer.value)
   }
 
-  async function enableShare() {
+  async function enableVideo() {
+    if (videoProducer.value)
+      return
+
     if (!device.value)
       return
 
-    const stream = await getShareStream()
+    const stream = await navigator.mediaDevices.getUserMedia({
+      video: {
+        deviceId: { exact: preferences.videoDeviceId.value },
+        width: { ideal: 1920 },
+        height: { ideal: 1080 },
+        frameRate: { ideal: 60 },
+      },
+    })
 
     const track = stream.getVideoTracks()[0]
 
     if (!track)
       return
 
-    await enableProducer('share', {
+    await createProducer({
       track,
+      streamId: 'mic-video',
+      // codec: device.value.rtpCapabilities.codecs?.find(
+      //   c => c.mimeType.toLowerCase() === 'video/AV1',
+      // ),
+      // codecOptions: {
+      //   videoGoogleStartBitrate: 1000,
+      // },
+      appData: {
+        source: 'mic-video',
+      },
+    })
+  }
+
+  async function enableShare() {
+    if (shareProducer.value)
+      return
+
+    if (!device.value)
+      return
+
+    const stream = await getShareStream(preferences.shareFps.value)
+
+    const track = stream.getVideoTracks()[0]
+
+    if (!track)
+      return
+
+    await createProducer({
+      track,
+      streamId: 'share',
       codec: device.value.rtpCapabilities.codecs?.find(
-        c => c.mimeType.toLowerCase() === 'video/h264',
+        c => c.mimeType.toLowerCase() === 'video/AV1',
       ),
       codecOptions: {
         videoGoogleStartBitrate: 1000,
       },
+      zeroRtpOnPause: true,
       appData: {
         source: 'share',
       },
     })
   }
 
-  async function pauseProducer(type: ProducerType) {
+  async function pauseProducer(producer: Producer) {
     if (!signaling.socket.value)
       return
 
-    const producer = getProducerByType(type)
+    if (producer.paused)
-
-    if (!producer.value)
-      return
-
-    if (producer.value.paused)
       return
 
     try {
-      producer.value.pause()
+      producer.raw.pause()
 
       await signaling.socket.value.emitWithAck('pauseProducer', {
-        producerId: producer.value.id,
+        producerId: producer.id,
       })
     }
     catch {
-      producer.value.resume()
+      producer.raw.resume()
-    }
-    finally {
-      triggerRef(producers)
-      triggerRef(producer)
     }
   }
 
-  async function resumeProducer(type: ProducerType) {
+  async function resumeProducer(producer: Producer) {
     if (!signaling.socket.value)
       return
 
-    const producer = getProducerByType(type)
-
-    if (!producer.value)
-      return
-
     try {
-      producer.value.resume()
+      producer.raw.resume()
 
       await signaling.socket.value.emitWithAck('resumeProducer', {
-        producerId: producer.value.id,
+        producerId: producer.id,
       })
     }
     catch {
-      producer.value.pause()
+      producer.raw.pause()
-    }
-    finally {
-      triggerRef(producers)
-      triggerRef(producer)
-    }
-  }
-
-  async function init() {
-    signaling.connect()
-  }
-
-  function getProducerByType(type: ProducerType) {
-    switch (type) {
-      case 'microphone':
-        return micProducer
-      case 'camera':
-        return cameraProducer
-      case 'share':
-        return shareProducer
     }
   }
 
@@ -421,18 +490,22 @@ export const useMediasoup = createSharedComposable(() => {
   })
 
   return {
-    init,
     consumers,
+    audioConsumers,
+    videoConsumers,
+    shareConsumers,
     producers,
+    speakingClients,
     sendTransport,
     recvTransport,
     rtpCapabilities,
     device,
     micProducer,
-    cameraProducer,
+    videoProducer,
     shareProducer,
     pauseProducer,
     resumeProducer,
+    enableVideo,
     enableShare,
     disableProducer,
   }

client/app/composables/use-preferences.ts

@@ -14,11 +14,14 @@ export const usePreferences = createGlobalState(() => {
 
   const inputDeviceId = useLocalStorage<MediaDeviceInfo['deviceId']>('INPUT_DEVICE_ID', 'default')
   const outputDeviceId = useLocalStorage<MediaDeviceInfo['deviceId']>('OUTPUT_DEVICE_ID', 'default')
+  const videoDeviceId = useLocalStorage<MediaDeviceInfo['deviceId']>('VIDEO_DEVICE_ID', 'default')
 
   const autoGainControl = useLocalStorage('AUTO_GAIN_CONTROL', false)
   const noiseSuppression = useLocalStorage('NOISE_SUPPRESSION', true)
   const echoCancellation = useLocalStorage('ECHO_CANCELLATION', true)
 
+  const shareFps = useLocalStorage('SHARE_FPS', 30)
+
   const toggleInputHotkey = ref<SyncedPreferences['toggleInputHotkey']>('')
   const toggleOutputHotkey = ref<SyncedPreferences['toggleOutputHotkey']>('')
 
@@ -30,6 +33,10 @@ export const usePreferences = createGlobalState(() => {
     return audioOutputs.value.some(device => device.deviceId === outputDeviceId.value)
   })
 
+  const videoDeviceExist = computed(() => {
+    return videoInputs.value.some(device => device.deviceId === videoDeviceId.value)
+  })
+
   watchDebounced(
     [toggleInputHotkey, toggleOutputHotkey],
     async ([toggleInputHotkey, toggleOutputHotkey]) => {
@@ -54,12 +61,15 @@ export const usePreferences = createGlobalState(() => {
     synced,
     inputDeviceId,
     outputDeviceId,
+    videoDeviceId,
     autoGainControl,
     noiseSuppression,
     echoCancellation,
+    shareFps,
     toggleInputHotkey,
     toggleOutputHotkey,
     inputDeviceExist,
     outputDeviceExist,
+    videoDeviceExist,
   }
 })

client/app/layouts/default.vue

@@ -1,10 +1,10 @@
 <template>
-  <div class="grid grid-cols-[1fr_1.25fr] gap-2 p-2 h-screen grid-rows-[auto_1fr]">
+  <div class="grid grid-cols-[1fr_1.25fr] gap-2 p-2 h-screen grid-rows-[auto_1fr] max-w-full">
     <div
       class="flex items-center justify-between gap-2 rounded-xl p-3 bg-surface-950"
     >
       <div class="inline-flex items-center gap-3">
-        <PrimeBadge v-if="isTauri" class="opacity-50" severity="secondary" :value="version" size="small" />
+        <PrimeBadge class="opacity-50" severity="secondary" :value="version" size="small" />
         <PrimeBadge :severity="connected ? 'success' : 'danger' " />
       </div>
 
@@ -21,6 +21,12 @@
         </PrimeButton>
       </PrimeButtonGroup>
 
+      <PrimeButton :severity="videoEnabled ? 'success' : undefined" outlined @click="toggleVideo">
+        <template #icon>
+          <Component :is="videoEnabled ? CameraOff : Camera" />
+        </template>
+      </PrimeButton>
+
       <PrimeButton :severity="sharingEnabled ? 'success' : undefined" outlined @click="toggleShare">
         <template #icon>
           <Component :is="sharingEnabled ? ScreenShareOff : ScreenShare" />
@@ -44,42 +50,49 @@
       </PrimeSelectButton>
     </div>
 
-    <PrimeScrollPanel class="bg-surface-900 rounded-xl" style="min-height: 0">
+    <PrimeScrollPanel class="bg-surface-900 rounded-xl overflow-hidden" style="min-height: 0">
       <div v-auto-animate class="p-3 space-y-1">
         <ClientRow v-for="client of clients" :key="client.userId" :client="client" />
       </div>
     </PrimeScrollPanel>
 
-    <PrimeScrollPanel class="bg-surface-900 rounded-xl" style="min-height: 0">
+    <PrimeScrollPanel class="bg-surface-900 rounded-xl overflow-hidden" style="min-height: 0">
       <div class="p-3">
         <slot />
       </div>
     </PrimeScrollPanel>
   </div>
+
+  <FullscreenGallery />
 </template>
 
 <script setup lang="ts">
 import {
+  Camera,
+  CameraOff,
   MessageCircle,
   Mic,
   MicOff,
   ScreenShare,
   ScreenShareOff,
   Settings,
+  TvMinimalPlay,
   UserPen,
   Volume2,
   VolumeOff,
 } from 'lucide-vue-next'
 
 const {
-  isTauri,
   version,
   clients,
   inputMuted,
   outputMuted,
+  videoEnabled,
   sharingEnabled,
+  somebodyStreamingVideo,
   toggleInput,
   toggleOutput,
+  toggleVideo,
   toggleShare,
 } = useApp()
 const { connect, connected } = useSignaling()
@@ -92,31 +105,47 @@ interface Tab {
 
 const route = useRoute()
 
-const tabs: Tab[] = [
+const tabs = computed<Tab[]>(() => {
-  {
+  const result = []
-    id: 'Index',
-    icon: MessageCircle,
-    onClick: () => {
-      navigateTo({ name: 'Index' })
-    },
-  },
-  {
-    id: 'Profile',
-    icon: UserPen,
-    onClick: () => {
-      navigateTo({ name: 'Profile' })
-    },
-  },
-  {
-    id: 'Preferences',
-    icon: Settings,
-    onClick: () => {
-      navigateTo({ name: 'Preferences' })
-    },
-  },
-]
 
-const activeTab = ref<Tab>(tabs.find(tab => tab.id === route.name) ?? tabs[0]!)
+  if (somebodyStreamingVideo.value) {
+    result.push({
+      id: 'Gallery',
+      icon: TvMinimalPlay,
+      onClick: () => {
+        navigateTo({ name: 'Gallery' })
+      },
+    })
+  }
+
+  result.push(
+    {
+      id: 'Index',
+      icon: MessageCircle,
+      onClick: () => {
+        navigateTo({ name: 'Index' })
+      },
+    },
+    {
+      id: 'Profile',
+      icon: UserPen,
+      onClick: () => {
+        navigateTo({ name: 'Profile' })
+      },
+    },
+    {
+      id: 'Preferences',
+      icon: Settings,
+      onClick: () => {
+        navigateTo({ name: 'Preferences' })
+      },
+    },
+  )
+
+  return result
+})
+
+const activeTab = ref<Tab>(tabs.value.find(tab => tab.id === route.name) ?? tabs.value.find(tab => tab.id === 'Index')!)
 
 watch(activeTab, (activeTab) => {
   activeTab.onClick()

client/app/pages/gallery.vue

@@ -0,0 +1,65 @@
+<template>
+  <div class="grid grid-cols-[1fr_1fr] gap-2">
+    <GalleryCard
+      v-for="item in gallery"
+      :key="item.client.socketId"
+      :client="item.client"
+      :stream="item.stream"
+      @click="watch(item.stream)"
+    />
+  </div>
+</template>
+
+<script setup lang="ts">
+import type { ChadClient } from '#shared/types'
+
+interface GalleryItem {
+  client: ChadClient
+  stream: MediaStream
+}
+
+definePageMeta({
+  name: 'Gallery',
+})
+
+const { videoProducer, shareProducer } = useMediasoup()
+const { clients, me } = useClients()
+
+const gallery = computed(() => {
+  return clients.value.reduce<GalleryItem[]>(
+    (acc, client) => {
+      const { streaming, videoConsumers, shareConsumers } = useClient(client.socketId)
+
+      if (!streaming.value)
+        return acc
+
+      for (const consumer of [...videoConsumers.value, ...shareConsumers.value]) {
+        acc.push({
+          client,
+          stream: new MediaStream([consumer.raw.track]),
+        })
+      }
+
+      return acc
+    },
+    [videoProducer.value, shareProducer.value].reduce<GalleryItem[]>((acc, producer) => {
+      if (!me.value || !producer || !producer.raw.track)
+        return acc
+
+      acc.push({
+        client: me.value,
+        stream: new MediaStream([producer.raw.track]),
+      })
+
+      return acc
+    }, []),
+  )
+})
+
+watch(gallery, (gallery) => {
+  if (gallery.length > 0)
+    return
+
+  navigateTo({ name: 'Index' })
+})
+</script>

client/app/pages/index.vue

@@ -3,7 +3,7 @@
     <div class="flex items-center justify-center">
       <PrimeCard>
         <template #content>
-          <ChatWidget />
+          The chat is under development.
         </template>
       </PrimeCard>
     </div>

client/app/pages/preferences.vue

@@ -11,6 +11,7 @@
         option-label="label"
         option-value="deviceId"
         input-id="inputDevice"
+        placeholder="No input device"
         fluid
         :invalid="!inputDeviceExist"
       />
@@ -47,6 +48,35 @@
     <!--      <label for="outputDevice">Output device</label> -->
     <!--    </PrimeFloatLabel> -->
 
+    <PrimeDivider align="left">
+      Video
+    </PrimeDivider>
+
+    <PrimeFloatLabel variant="on">
+      <PrimeSelect
+        v-model="videoDeviceId"
+        :options="videoInputs"
+        option-label="label"
+        option-value="deviceId"
+        input-id="videoDevice"
+        placeholder="No video device"
+        fluid
+        :invalid="!videoDeviceExist"
+      />
+      <label for="inputDevice">Input device</label>
+    </PrimeFloatLabel>
+
+    <PrimeDivider align="left">
+      Screen sharing
+    </PrimeDivider>
+
+    <div>
+      <p class="text-sm mb-2 text-center">
+        FPS
+      </p>
+      <PrimeSelectButton v-model="shareFps" :options="[5, 30, 60]" fluid size="small" />
+    </div>
+
     <template v-if="isTauri">
       <PrimeDivider align="left">
         Hotkeys
@@ -106,10 +136,11 @@ definePageMeta({
 })
 const { isTauri, version, commitSha } = useApp()
 const { checking, checkForUpdates, lastUpdate } = useUpdater()
-const { audioInputs, audioOutputs } = useDevices()
+const { audioInputs, audioOutputs, videoInputs } = useDevices()
 const {
   inputDeviceId,
   outputDeviceId,
+  videoDeviceId,
   autoGainControl,
   noiseSuppression,
   echoCancellation,
@@ -117,6 +148,8 @@ const {
   toggleOutputHotkey,
   inputDeviceExist,
   outputDeviceExist,
+  videoDeviceExist,
+  shareFps,
 } = usePreferences()
 
 const setupToggleInputHotkey = (event: KeyboardEvent) => setupHotkey(event, toggleInputHotkey)

client/nuxt.config.ts

@@ -86,8 +86,8 @@ export default defineNuxtConfig({
       strictPort: true,
       proxy: {
         '/api': {
-          target: 'http://localhost:4000/chad',
+          // target: 'http://localhost:4000/chad',
-          // target: 'https://api.koptilnya.xyz/chad',
+          target: 'https://api.koptilnya.xyz/chad',
           ws: true,
           changeOrigin: true,
           rewrite: (path) => {

client/package.json

@@ -4,7 +4,7 @@
   "private": true,
   "scripts": {
     "build": "nuxt build",
-    "dev": "nuxt dev",
+    "dev": "nuxt dev --host",
     "generate": "nuxt generate",
     "preview": "nuxt preview",
     "postinstall": "nuxt prepare"
@@ -20,7 +20,7 @@
     "@vueuse/core": "^13.9.0",
     "hotkeys-js": "^4.0.0",
     "lucide-vue-next": "^0.562.0",
-    "mediasoup-client": "^3.16.7",
+    "mediasoup-client": "^3.18.6",
     "nuxt": "^4.2.2",
     "postcss": "^8.5.6",
     "primeicons": "^7.0.0",

client/shared/types.ts

@@ -1,3 +1,5 @@
+import type { Consumer as MediasoupConsumer, Producer as MediasoupProducer } from 'mediasoup-client/types'
+
 export interface ChadClient {
   socketId: string
   userId: string
@@ -5,6 +7,30 @@ export interface ChadClient {
   displayName: string
   inputMuted?: boolean
   outputMuted?: boolean
+
+  consumers: unknown[]
+  producers: unknown[]
+  volume: number
+  isDominant: boolean
+}
+
+export interface AppData {
+  socketId?: ChadClient['socketId']
+  source?: 'share' | 'mic-video'
+}
+
+export interface Consumer {
+  id: MediasoupConsumer['id']
+  paused: MediasoupConsumer['paused']
+  appData: AppData
+  raw: MediasoupConsumer
+}
+
+export interface Producer {
+  id: MediasoupProducer['id']
+  paused: MediasoupProducer['paused']
+  appData: AppData
+  raw: MediasoupProducer
 }
 
 export type UpdatedClient = Omit<ChadClient, 'socketId' | 'userId' | 'isMe'>

client/src-tauri/src/lib.rs

@@ -4,7 +4,7 @@ pub fn run() {
         .plugin(tauri_plugin_global_shortcut::Builder::new().build())
         .plugin(tauri_plugin_process::init())
         .plugin(tauri_plugin_updater::Builder::new().build())
-//         .plugin(tauri_plugin_single_instance::init(|_, _, _| {}))
+        .plugin(tauri_plugin_single_instance::init(|_, _, _| {}))
         .setup(|app| {
             if cfg!(debug_assertions) {
                 app.handle().plugin(

client/src-tauri/tauri.conf.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
   "productName": "chad",
-  "version": "0.2.17",
+  "version": "0.2.24",
   "identifier": "xyz.koptilnya.chad",
   "build": {
     "frontendDist": "../.output/public",

client/yarn.lock

@@ -4060,7 +4060,7 @@ __metadata:
     eslint-plugin-format: "npm:^1.0.2"
     hotkeys-js: "npm:^4.0.0"
     lucide-vue-next: "npm:^0.562.0"
-    mediasoup-client: "npm:^3.16.7"
+    mediasoup-client: "npm:^3.18.6"
     nuxt: "npm:^4.2.2"
     postcss: "npm:^8.5.6"
     primeicons: "npm:^7.0.0"
@@ -6176,12 +6176,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"h264-profile-level-id@npm:^2.3.1":
+"h264-profile-level-id@npm:^2.3.2":
-  version: 2.3.1
+  version: 2.3.2
-  resolution: "h264-profile-level-id@npm:2.3.1"
+  resolution: "h264-profile-level-id@npm:2.3.2"
   dependencies:
     debug: "npm:^4.4.3"
-  checksum: 10c0/c3459549bb28e456db62428c79885cffd4958ce282099c4181b09576f8e5ad90b42395a77209fff4f20a7cb920aaeb660f73902f08343daead0f5527faeb4015
+  checksum: 10c0/75bd12ff36707ffacf379c31c403d4508f3116ef2065e375deadcfafd4f7d163521cf0c70ae5385ebac970fa0acc07f9dd497c4248cfc1ee5623b4533707731d
   languageName: node
   linkType: hard
 
@@ -7302,9 +7302,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"mediasoup-client@npm:^3.16.7":
+"mediasoup-client@npm:^3.18.6":
-  version: 3.16.7
+  version: 3.18.6
-  resolution: "mediasoup-client@npm:3.16.7"
+  resolution: "mediasoup-client@npm:3.18.6"
   dependencies:
     "@types/debug": "npm:^4.1.12"
     "@types/events-alias": "npm:@types/events@^3.0.3"
@@ -7312,10 +7312,10 @@ __metadata:
     debug: "npm:^4.4.3"
     events-alias: "npm:events@^3.3.0"
     fake-mediastreamtrack: "npm:^2.2.1"
-    h264-profile-level-id: "npm:^2.3.1"
+    h264-profile-level-id: "npm:^2.3.2"
-    sdp-transform: "npm:^2.15.0"
+    sdp-transform: "npm:^3.0.0"
     supports-color: "npm:^10.2.2"
-  checksum: 10c0/da44c6de8889963192c5b0b7907ed628e04d48be73b7bbfbf18012d66b07ede9d7367c0723466e496a87c7002c07f1af432d854c4c5e16cbd0887013870d8abe
+  checksum: 10c0/f5baff9139afccf88de5db767c1139efa5cdd68f4871e2fa9d6ff94d2e71d2365dc40e9ba6e903cde5fbb51a2d82972e738da656be9f6fc7006640fdd82dd5da
   languageName: node
   linkType: hard
 
@@ -9829,12 +9829,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"sdp-transform@npm:^2.15.0":
+"sdp-transform@npm:^3.0.0":
-  version: 2.15.0
+  version: 3.0.0
-  resolution: "sdp-transform@npm:2.15.0"
+  resolution: "sdp-transform@npm:3.0.0"
   bin:
     sdp-verify: checker.js
-  checksum: 10c0/96c060f113a3d5418defa168db609f7e23e5bd7954fa1cf7784f103dbe702e24d667e5310d2ac6d88abdb32322af83d6ebd0df08e07f4f172d5ed5888f921386
+  checksum: 10c0/828a4595041ba64c86b29075aa4007ab384519b1fa29882db59ccb83b54b2b2a33b60848293f8da537fe151c52f5844fc17c8325396cac309fb19e2e81ec5bf4
   languageName: node
   linkType: hard
 

node_modules/.yarn-integrity

@@ -1,12 +0,0 @@
-{
-  "systemParams": "win32-x64-137",
-  "modulesFolders": [
-    "node_modules"
-  ],
-  "flags": [],
-  "linkedModules": [],
-  "topLevelPatterns": [],
-  "lockfileEntries": {},
-  "files": [],
-  "artifacts": {}
-}

server/modules/chat/index.ts

@@ -1,22 +0,0 @@
-import client from '../../prisma/client.ts'
-
-export async function chatInit() {
-  const existing = client.chatChannel.findFirst({
-    where: {
-      id: 0,
-    },
-  })
-
-  if (!existing) {
-    await client.chatChannel.create({
-      create: {
-        id: 0,
-        name: 'Main channel',
-      },
-      update: null,
-      where: {
-        id: 0,
-      },
-    })
-  }
-}

server/plugins/socket.ts

@@ -22,8 +22,8 @@ export default fp<Partial<ServerOptions>>(
       await fastify.io.close()
     })
 
-    fastify.ready(() => {
+    fastify.ready(async () => {
-      registerWebrtcSocket(fastify.io, fastify.mediasoupRouter)
+      await registerWebrtcSocket(fastify.io, fastify.mediasoupRouter)
     })
   },
   { name: 'socket-io', dependencies: ['mediasoup-worker', 'mediasoup-router'] },

server/prisma/migrations/20251226183651_minus_nahuy/migration.sql

@@ -1,20 +0,0 @@
-/*
-  Warnings:
-
-  - You are about to drop the column `volumes` on the `UserPreferences` table. All the data in the column will be lost.
-
-*/
--- RedefineTables
-PRAGMA defer_foreign_keys=ON;
-PRAGMA foreign_keys=OFF;
-CREATE TABLE "new_UserPreferences" (
-    "userId" TEXT NOT NULL PRIMARY KEY,
-    "toggleInputHotkey" TEXT DEFAULT '',
-    "toggleOutputHotkey" TEXT DEFAULT '',
-    CONSTRAINT "UserPreferences_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-INSERT INTO "new_UserPreferences" ("toggleInputHotkey", "toggleOutputHotkey", "userId") SELECT "toggleInputHotkey", "toggleOutputHotkey", "userId" FROM "UserPreferences";
-DROP TABLE "UserPreferences";
-ALTER TABLE "new_UserPreferences" RENAME TO "UserPreferences";
-PRAGMA foreign_keys=ON;
-PRAGMA defer_foreign_keys=OFF;

server/prisma/migrations/20251226190516_chat/migration.sql

@@ -1,18 +0,0 @@
--- CreateTable
-CREATE TABLE "ChatMessage" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "userId" TEXT NOT NULL,
-    "channelId" TEXT NOT NULL,
-    "content" TEXT NOT NULL DEFAULT '',
-    CONSTRAINT "ChatMessage_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
-    CONSTRAINT "ChatMessage_channelId_fkey" FOREIGN KEY ("channelId") REFERENCES "ChatChannel" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-
--- CreateTable
-CREATE TABLE "ChatChannel" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "name" TEXT NOT NULL
-);
-
--- CreateIndex
-CREATE UNIQUE INDEX "ChatChannel_name_key" ON "ChatChannel"("name");

server/prisma/migrations/20251226194252_more_chat_wip/migration.sql

@@ -1,34 +0,0 @@
-/*
-  Warnings:
-
-  - The primary key for the `ChatChannel` table will be changed. If it partially fails, the table could be left without primary key constraint.
-  - You are about to alter the column `id` on the `ChatChannel` table. The data in that column could be lost. The data in that column will be cast from `String` to `Int`.
-  - You are about to alter the column `channelId` on the `ChatMessage` table. The data in that column could be lost. The data in that column will be cast from `String` to `Int`.
-  - Added the required column `createdAt` to the `ChatMessage` table without a default value. This is not possible if the table is not empty.
-
-*/
--- RedefineTables
-PRAGMA defer_foreign_keys=ON;
-PRAGMA foreign_keys=OFF;
-CREATE TABLE "new_ChatChannel" (
-    "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
-    "name" TEXT NOT NULL
-);
-INSERT INTO "new_ChatChannel" ("id", "name") SELECT "id", "name" FROM "ChatChannel";
-DROP TABLE "ChatChannel";
-ALTER TABLE "new_ChatChannel" RENAME TO "ChatChannel";
-CREATE UNIQUE INDEX "ChatChannel_name_key" ON "ChatChannel"("name");
-CREATE TABLE "new_ChatMessage" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "userId" TEXT NOT NULL,
-    "channelId" INTEGER NOT NULL,
-    "content" TEXT NOT NULL DEFAULT '',
-    "createdAt" DATETIME NOT NULL,
-    CONSTRAINT "ChatMessage_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
-    CONSTRAINT "ChatMessage_channelId_fkey" FOREIGN KEY ("channelId") REFERENCES "ChatChannel" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-INSERT INTO "new_ChatMessage" ("channelId", "content", "id", "userId") SELECT "channelId", "content", "id", "userId" FROM "ChatMessage";
-DROP TABLE "ChatMessage";
-ALTER TABLE "new_ChatMessage" RENAME TO "ChatMessage";
-PRAGMA foreign_keys=ON;
-PRAGMA defer_foreign_keys=OFF;

server/prisma/migrations/20251226194927_more_chat_wip/migration.sql

@@ -1,17 +0,0 @@
--- RedefineTables
-PRAGMA defer_foreign_keys=ON;
-PRAGMA foreign_keys=OFF;
-CREATE TABLE "new_ChatMessage" (
-    "id" TEXT NOT NULL PRIMARY KEY,
-    "userId" TEXT NOT NULL,
-    "channelId" INTEGER NOT NULL,
-    "content" TEXT NOT NULL,
-    "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    CONSTRAINT "ChatMessage_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
-    CONSTRAINT "ChatMessage_channelId_fkey" FOREIGN KEY ("channelId") REFERENCES "ChatChannel" ("id") ON DELETE CASCADE ON UPDATE CASCADE
-);
-INSERT INTO "new_ChatMessage" ("channelId", "content", "createdAt", "id", "userId") SELECT "channelId", "content", "createdAt", "id", "userId" FROM "ChatMessage";
-DROP TABLE "ChatMessage";
-ALTER TABLE "new_ChatMessage" RENAME TO "ChatMessage";
-PRAGMA foreign_keys=ON;
-PRAGMA defer_foreign_keys=OFF;

server/prisma/schema.prisma

@@ -18,8 +18,6 @@ model User {
 
   Session           Session[]
   UserPreferences   UserPreferences?
-
-  ChatMessage ChatMessage[]
 }
 
 model Session {
@@ -36,26 +34,7 @@ model UserPreferences {
   userId              String  @id
   toggleInputHotkey   String? @default("")
   toggleOutputHotkey  String? @default("")
+  volumes             Json?   @default("{}")
 
   user                User    @relation(fields: [userId], references: [id], onDelete: Cascade)
-}
-
-model ChatMessage {
-  id String @id
-
-  userId String
-  channelId Int
-  content String
-
-  createdAt DateTime @default(now())
-
-  user User @relation(fields: [userId], references: [id], onDelete: Restrict)
-  channel ChatChannel @relation(fields: [channelId], references: [id], onDelete: Cascade)
-}
-
-model ChatChannel {
-  id Int @id
-  name String @unique
-
-  messages ChatMessage[]
 }

server/routes/chat.ts

@@ -1,23 +0,0 @@
-import type { FastifyInstance } from 'fastify'
-import prisma from '../prisma/client.ts'
-
-export default function (fastify: FastifyInstance) {
-  fastify.get('/chats', async (req, reply) => {
-    if (req.user) {
-      return prisma.chatChannel.findMany()
-    }
-
-    reply.code(401).send(false)
-  })
-
-  fastify.get('/chats/:id', async (req, reply) => {
-    if (req.user) {
-      console.log('Trying to fetch chat with id', req.body.id)
-
-      // return prisma.userPreferences.findFirst({ where: { userId: req.user.id } })
-      return true
-    }
-
-    reply.code(401).send(false)
-  })
-}

server/server.ts

@@ -4,7 +4,6 @@ import FastifyAutoLoad from '@fastify/autoload'
 import FastifyCookie from '@fastify/cookie'
 import FastifyCors from '@fastify/cors'
 import Fastify from 'fastify'
-import { chatInit } from './modules/chat/index.ts'
 import prisma from './prisma/client.ts'
 
 const __filename = fileURLToPath(import.meta.url)
@@ -44,8 +43,6 @@ fastify.register(FastifyAutoLoad, {
 
     await prisma.$connect()
     fastify.log.info('Testing DB Connection. OK')
-
-    await chatInit()
   }
   catch (err) {
     fastify.log.error(err)

server/socket/webrtc.ts

@@ -1,6 +1,7 @@
 import type { types } from 'mediasoup'
 import type { Server as SocketServer } from 'socket.io'
 import type {
+  ChadClient,
   Namespace,
   SomeSocket,
 } from '../types/webrtc.ts'
@@ -8,9 +9,39 @@ import { consola } from 'consola'
 import prisma from '../prisma/client.ts'
 import { socketToClient } from '../utils/socket-to-client.ts'
 
-export default function (io: SocketServer, router: types.Router) {
+export default async function (io: SocketServer, router: types.Router) {
   const namespace: Namespace = io.of('/webrtc')
 
+  const audioLevelObserver = await router.createAudioLevelObserver({
+    maxEntries: 10,
+    threshold: -80,
+    interval: 800,
+  })
+
+  const activeSpeakerObserver = await router.createActiveSpeakerObserver()
+
+  audioLevelObserver.on('volumes', async (volumes: types.AudioLevelObserverVolume[]) => {
+    namespace.emit('speakingPeers', volumes.map(({ producer, volume }) => {
+      const { socketId } = producer.appData as { socketId: ChadClient['socketId'] }
+
+      return {
+        clientId: socketId,
+        volume,
+      }
+    }))
+  })
+
+  audioLevelObserver.on('silence', () => {
+    namespace.emit('speakingPeers', [])
+    namespace.emit('activeSpeaker', undefined)
+  })
+
+  activeSpeakerObserver.on('dominantspeaker', ({ producer }) => {
+    const { socketId } = producer.appData as { socketId: ChadClient['socketId'] }
+
+    namespace.emit('activeSpeaker', socketId)
+  })
+
   namespace.on('connection', async (socket) => {
     consola.info('[WebRtc]', 'Client connected', socket.id)
 
@@ -182,8 +213,8 @@ export default function (io: SocketServer, router: types.Router) {
           )
         }
 
-        // TODO: Add into the AudioLevelObserver and ActiveSpeakerObserver.
+        await audioLevelObserver.addProducer({ producerId: producer.id })
-        // https://github.com/versatica/mediasoup-demo/blob/v3/server/lib/Room.js#L1276
+        await activeSpeakerObserver.addProducer({ producerId: producer.id })
       }
       catch (error) {
         if (error instanceof Error) {

server/types/webrtc.ts

@@ -118,6 +118,8 @@ export interface ServerToClientEvents {
   consumerResumed: (arg: { consumerId: string }) => void
   consumerScore: (arg: { consumerId: string, score: types.ConsumerScore }) => void
   clientChanged: (clientId: ChadClient['socketId'], client: ChadClient) => void
+  speakingPeers: (arg: { clientId: ChadClient['socketId'], volume: types.AudioLevelObserverVolume['volume'] }[]) => void
+  activeSpeaker: (clientId?: ChadClient['socketId']) => void
 }
 
 export interface InterServerEvent {}

yarn.lock

@@ -1,4 +0,0 @@
-# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
-# yarn lockfile v1
-
-