screen sharing

client/app/components/ClientRow.vue

@@ -58,15 +58,15 @@ const volume = useLocalStorage<number>(computed(() => `CLIENT_VOLUME_${props.cli
 const menuRef = useTemplateRef<HTMLAudioElement>('menu')
 
 const menuItems: MenuItem[] = [
-  {
+  // {
-    label: 'Mute',
+  //   label: 'Mute',
-    icon: 'pi pi-headphones',
+  //   icon: 'pi pi-headphones',
-  },
+  // },
-  {
+  // {
-    label: 'DM',
+  //   label: 'DM',
-    icon: 'pi pi-comment',
+  //   icon: 'pi pi-comment',
-    disabled: true,
+  //   disabled: true,
-  },
+  // },
 ]
 
 const isMe = computed(() => {
@@ -98,10 +98,7 @@ const audioTrack = computed(() => {
 const { setGain } = useAudioContext(audioTrack)
 
 watch(volume, (volume) => {
-  if (outputMuted.value)
+  setGain(outputMuted.value ? 0 : (volume * 0.01))
-    return
-
-  setGain(volume * 0.01)
 }, { immediate: true })
 
 watch(outputMuted, (outputMuted) => {

client/app/composables/use-app.ts

@@ -10,78 +10,94 @@ export const useApp = createGlobalState(() => {
   const toast = useToast()
 
   const ready = ref(false)
-
-  const inputMuted = ref(false)
-  const outputMuted = ref(false)
-
-  const previousInputMuted = ref(inputMuted.value)
-
   const isTauri = computed(() => '__TAURI_INTERNALS__' in window)
-
   const commitSha = __COMMIT_SHA__
-
   const version = computedAsync(() => isTauri.value ? getVersion() : 'web', '-')
 
-  watch(inputMuted, async (inputMuted) => {
+  const inputMuted = computed(() => {
-    if (inputMuted) {
+    return !!mediasoup.micProducer.value?.paused
-      await mediasoup.pauseProducer('microphone')
+  })
-    }
+  const previousInputMuted = ref(inputMuted.value)
-    else {
-      if (outputMuted.value) {
-        outputMuted.value = false
-      }
-      await mediasoup.resumeProducer('microphone')
-    }
 
-    const toastText = inputMuted ? 'Microphone muted' : 'Microphone activated'
+  const outputMuted = ref(false)
-    toast.add({ severity: 'info', summary: toastText, closable: false, life: 1000 })
+
+  const sharingEnabled = computed(() => {
+    return !!mediasoup.shareProducer.value
   })
 
-  watch(outputMuted, async (outputMuted) => {
+  async function muteInput() {
-    if (outputMuted) {
+    if (inputMuted.value)
-      previousInputMuted.value = inputMuted.value
+      return
-      muteInput()
+
-    }
+    await mediasoup.pauseProducer('microphone')
-    else {
+
-      inputMuted.value = previousInputMuted.value
+    toast.add({ severity: 'info', summary: 'Microphone muted', closable: false, life: 1000 })
+  }
+
+  async function unmuteInput() {
+    if (!inputMuted.value)
+      return
+
+    if (outputMuted.value) {
+      await unmuteOutput()
     }
 
-    const updatedMe = await signaling.socket.value?.emitWithAck('updateClient', {
+    await mediasoup.resumeProducer('microphone')
-      outputMuted,
+
+    toast.add({ severity: 'info', summary: 'Microphone activated', closable: false, life: 1000 })
+  }
+
+  async function toggleInput() {
+    if (inputMuted.value)
+      await unmuteInput()
+    else
+      await muteInput()
+  }
+
+  async function muteOutput() {
+    if (outputMuted.value)
+      return
+
+    outputMuted.value = true
+
+    previousInputMuted.value = inputMuted.value
+
+    await muteInput()
+
+    await signaling.socket.value?.emitWithAck('updateClient', {
+      outputMuted: false,
     })
 
-    const toastText = outputMuted ? 'Sound muted' : 'Sound resumed'
+    toast.add({ severity: 'info', summary: 'Sound muted', closable: false, life: 1000 })
-    toast.add({ severity: 'info', summary: toastText, closable: false, life: 1000 })
-  })
-
-  function muteInput() {
-    inputMuted.value = true
   }
 
-  function unmuteInput() {
+  async function unmuteOutput() {
-    inputMuted.value = false
-  }
-
-  function toggleInput() {
-    if (inputMuted.value)
-      unmuteInput()
-    else
-      muteInput()
-  }
-
-  function muteOutput() {
-    outputMuted.value = true
-  }
-
-  function unmuteOutput() {
     outputMuted.value = false
+
+    if (!previousInputMuted.value)
+      await unmuteInput()
+
+    await signaling.socket.value?.emitWithAck('updateClient', {
+      outputMuted: false,
+    })
+
+    toast.add({ severity: 'info', summary: 'Sound resumed', closable: false, life: 1000 })
   }
 
-  function toggleOutput() {
+  async function toggleOutput() {
     if (outputMuted.value)
-      unmuteOutput()
+      await unmuteOutput()
     else
-      muteOutput()
+      await muteOutput()
+  }
+
+  async function toggleShare() {
+    if (!mediasoup.shareProducer.value) {
+      await mediasoup.enableShare()
+    }
+    else {
+      await mediasoup.disableProducer('share')
+    }
   }
 
   return {
@@ -98,5 +114,7 @@ export const useApp = createGlobalState(() => {
     version,
     isTauri,
     commitSha,
+    toggleShare,
+    sharingEnabled,
   }
 })

client/app/composables/use-devices.ts

@@ -0,0 +1,28 @@
+import { createGlobalState, useDevicesList } from '@vueuse/core'
+
+export const useDevices = createGlobalState(() => {
+  const {
+    ensurePermissions,
+    permissionGranted,
+    videoInputs,
+    audioInputs,
+    audioOutputs,
+  } = useDevicesList()
+
+  async function getShareStream(fps = 30) {
+    return navigator.mediaDevices.getDisplayMedia({
+      audio: false,
+      video: {
+        displaySurface: 'monitor',
+        frameRate: { max: fps },
+      },
+    })
+  }
+
+  return {
+    videoInputs: computed<MediaDeviceInfo[]>(() => JSON.parse(JSON.stringify(videoInputs.value))),
+    audioInputs: computed<MediaDeviceInfo[]>(() => JSON.parse(JSON.stringify(audioInputs.value))),
+    audioOutputs: computed<MediaDeviceInfo[]>(() => JSON.parse(JSON.stringify(audioOutputs.value))),
+    getShareStream,
+  }
+})

client/app/composables/use-mediasoup.ts

@@ -1,4 +1,5 @@
 import type { ChadClient } from '#shared/types'
+import type { MediaKind, ProducerOptions } from 'mediasoup-client/types'
 import { createSharedComposable } from '@vueuse/core'
 import * as mediasoupClient from 'mediasoup-client'
 import { usePreferences } from '~/composables/use-preferences'
@@ -25,7 +26,6 @@ export const useMediasoup = createSharedComposable(() => {
   const signaling = useSignaling()
   const { addClient, removeClient } = useClients()
   const preferences = usePreferences()
-  const { me } = useAuth()
 
   const device = shallowRef<mediasoupClient.Device>()
   const rtpCapabilities = shallowRef<mediasoupClient.types.RtpCapabilities>()
@@ -158,7 +158,7 @@ export const useMediasoup = createSharedComposable(() => {
           producerId,
           kind,
           rtpParameters,
-          streamId: `${socketId}-${appData.share ? 'share' : 'mic-webcam'}`,
+          streamId: `${socketId}-${appData.source === 'share' ? 'share' : 'mic-webcam'}`,
           appData: { ...appData, socketId },
         })
 
@@ -234,16 +234,62 @@ export const useMediasoup = createSharedComposable(() => {
     return consumers.value.values().filter(consumer => consumer.appData.socketId === socketId)
   }
 
-  async function enableMic() {
+  async function enableProducer(type: ProducerType, options: ProducerOptions) {
-    if (micProducer.value)
+    const producer = getProducerByType(type)
+
+    if (producer.value)
       return
 
     if (!device.value || !sendTransport.value)
       return
 
-    if (!device.value.canProduce('audio'))
+    if (!options.track)
       return
 
+    if (!device.value.canProduce(options.track.kind as MediaKind))
+      return
+
+    producer.value = await sendTransport.value.produce(options)
+
+    producers.value.set(producer.value.id, producer.value)
+    triggerRef(producers)
+    triggerRef(producer)
+
+    producer.value.on('transportclose', () => {
+      micProducer.value = undefined
+    })
+
+    producer.value.on('trackended', () => {
+      disableProducer(type)
+    })
+  }
+
+  async function disableProducer(type: ProducerType) {
+    const producer = getProducerByType(type)
+
+    if (!signaling.socket.value || !producer.value)
+      return
+
+    producers.value.delete(producer.value.id)
+
+    try {
+      producer.value.close()
+
+      await signaling.socket.value.emitWithAck('closeProducer', {
+        producerId: producer.value.id,
+      })
+    }
+    catch {
+    }
+    finally {
+      triggerRef(producers)
+      triggerRef(producer)
+    }
+
+    producer.value = undefined
+  }
+
+  async function enableMic() {
     const stream = await navigator.mediaDevices.getUserMedia({
       audio: {
         deviceId: { exact: preferences.inputDeviceId.value },
@@ -258,7 +304,7 @@ export const useMediasoup = createSharedComposable(() => {
     if (!track)
       return
 
-    micProducer.value = await sendTransport.value.produce({
+    await enableProducer('microphone', {
       track,
       codecOptions: {
         opusStereo: true,
@@ -266,41 +312,41 @@ export const useMediasoup = createSharedComposable(() => {
         opusFec: false, // Фиксит пакет лос
       },
     })
-
-    producers.value.set(micProducer.value.id, micProducer.value)
-    triggerRef(producers)
-    triggerRef(micProducer)
-
-    micProducer.value.on('transportclose', () => {
-      micProducer.value = undefined
-    })
-
-    micProducer.value.on('trackended', () => {
-      disableMic()
-    })
   }
 
   async function disableMic() {
-    if (!signaling.socket.value || !micProducer.value)
+    await disableProducer('microphone')
+  }
+
+  async function enableShare() {
+    if (!device.value)
       return
 
-    producers.value.delete(micProducer.value.id)
+    const stream = await navigator.mediaDevices.getDisplayMedia({
+      audio: false,
+      video: {
+        displaySurface: 'monitor',
+        frameRate: { max: 30 },
+      },
+    })
 
-    try {
+    const track = stream.getVideoTracks()[0]
-      micProducer.value.close()
 
-      await signaling.socket.value.emitWithAck('closeProducer', {
+    if (!track)
-        producerId: micProducer.value.id,
+      return
-      })
-    }
-    catch {
-    }
-    finally {
-      triggerRef(producers)
-      triggerRef(micProducer)
-    }
 
-    micProducer.value = undefined
+    await enableProducer('share', {
+      track,
+      codec: device.value.rtpCapabilities.codecs?.find(
+        c => c.mimeType.toLowerCase() === 'video/h264',
+      ),
+      codecOptions: {
+        videoGoogleStartBitrate: 1000,
+      },
+      appData: {
+        source: 'share',
+      },
+    })
   }
 
   async function pauseProducer(type: ProducerType) {
@@ -371,18 +417,6 @@ export const useMediasoup = createSharedComposable(() => {
     }
   }
 
-  watch(
-    preferences.inputDeviceId,
-    async (inputDeviceId) => {
-      await disableMic()
-
-      if (!inputDeviceId)
-        return
-
-      await enableMic()
-    },
-  )
-
   watch([
     preferences.inputDeviceId,
     preferences.echoCancellation,
@@ -411,5 +445,7 @@ export const useMediasoup = createSharedComposable(() => {
     getClientConsumers,
     pauseProducer,
     resumeProducer,
+    enableShare,
+    disableProducer,
   }
 })

client/app/composables/use-preferences.ts

@@ -1,5 +1,5 @@
 import chadApi from '#shared/chad-api'
-import { createGlobalState, useDevicesList, useLocalStorage, watchDebounced } from '@vueuse/core'
+import { createGlobalState, useLocalStorage, watchDebounced } from '@vueuse/core'
 
 export interface SyncedPreferences {
   toggleInputHotkey: string
@@ -8,6 +8,8 @@ export interface SyncedPreferences {
 }
 
 export const usePreferences = createGlobalState(() => {
+  const { videoInputs, audioInputs, audioOutputs } = useDevices()
+
   const synced = ref(false)
 
   const inputDeviceId = useLocalStorage<MediaDeviceInfo['deviceId']>('INPUT_DEVICE_ID', 'default')
@@ -20,14 +22,6 @@ export const usePreferences = createGlobalState(() => {
   const toggleInputHotkey = ref<SyncedPreferences['toggleInputHotkey']>('')
   const toggleOutputHotkey = ref<SyncedPreferences['toggleOutputHotkey']>('')
 
-  const {
-    ensurePermissions,
-    permissionGranted,
-    videoInputs,
-    audioInputs,
-    audioOutputs,
-  } = useDevicesList()
-
   const inputDeviceExist = computed(() => {
     return audioInputs.value.some(device => device.deviceId === inputDeviceId.value)
   })
@@ -67,8 +61,5 @@ export const usePreferences = createGlobalState(() => {
     toggleOutputHotkey,
     inputDeviceExist,
     outputDeviceExist,
-    videoInputs: computed(() => JSON.parse(JSON.stringify(videoInputs.value))),
-    audioInputs: computed(() => JSON.parse(JSON.stringify(audioInputs.value))),
-    audioOutputs: computed(() => JSON.parse(JSON.stringify(audioOutputs.value))),
   }
 })

client/app/layouts/default.vue

@@ -20,6 +20,12 @@
           </template>
         </PrimeButton>
       </PrimeButtonGroup>
+
+      <PrimeButton outlined @click="toggleShare">
+        <template #icon>
+          <Component :is="sharingEnabled ? ScreenShareOff : ScreenShare" />
+        </template>
+      </PrimeButton>
     </div>
 
     <div
@@ -53,9 +59,19 @@
 </template>
 
 <script setup lang="ts">
-import { MessageCircle, Mic, MicOff, Settings, UserPen, Volume2, VolumeOff } from 'lucide-vue-next'
+import {
+  MessageCircle,
+  Mic,
+  MicOff,
+  ScreenShare,
+  ScreenShareOff,
+  Settings,
+  UserPen,
+  Volume2,
+  VolumeOff,
+} from 'lucide-vue-next'
 
-const { clients, inputMuted, toggleInput, outputMuted, toggleOutput, version, isTauri } = useApp()
+const { clients, inputMuted, toggleInput, outputMuted, toggleOutput, version, isTauri, sharingEnabled, toggleShare } = useApp()
 const { connect, connected } = useSignaling()
 
 interface Tab {

client/app/middleware/01.auth.global.ts

@@ -6,7 +6,9 @@ export default defineNuxtRouteMiddleware(async (to, from) => {
   if (!me.value) {
     try {
       setMe(await chadApi('/me', { method: 'GET' }))
-      return navigateTo({ name: 'Index' })
+
+      if (to.meta.auth !== false)
+        return navigateTo({ name: 'Index' })
     }
     catch {
       if (to.meta.auth !== 'guest') {

client/app/pages/index.vue

@@ -1,15 +1,49 @@
 <template>
-  <div class="flex items-center justify-center">
+  <div>
-    <PrimeCard>
+    <div class="flex items-center justify-center">
-      <template #content>
+      <PrimeCard>
-        The chat is under development.
+        <template #content>
-      </template>
+          The chat is under development.
-    </PrimeCard>
+        </template>
+      </PrimeCard>
+    </div>
+
+    <video
+      v-if="!!shareConsumer"
+      ref="shareVideo"
+      class="w-full aspect-video border border-surface-700 rounded mt-6 cursor-pointer"
+      autoplay
+      playsinline
+      @click="toFullscreen"
+    />
   </div>
 </template>
 
 <script setup lang="ts">
+import { unrefElement } from '@vueuse/core'
+
 definePageMeta({
   name: 'Index',
 })
+
+const shareVideoRef = useTemplateRef('shareVideo')
+
+const { consumers } = useMediasoup()
+
+const shareConsumer = computed(() => {
+  return consumers.value.values().find(consumer => consumer.kind === 'video' && consumer.appData?.source === 'share')
+})
+
+watchEffect(() => {
+  if (!shareVideoRef.value || !shareConsumer.value)
+    return
+
+  const stream = new MediaStream([shareConsumer.value.track])
+
+  unrefElement(shareVideoRef)!.srcObject = stream
+})
+
+function toFullscreen() {
+  unrefElement(shareVideoRef)?.requestFullscreen()
+}
 </script>

client/app/pages/preferences.vue

@@ -106,6 +106,7 @@ definePageMeta({
 })
 const { isTauri, version, commitSha } = useApp()
 const { checking, checkForUpdates, lastUpdate } = useUpdater()
+const { audioInputs, audioOutputs } = useDevices()
 const {
   inputDeviceId,
   outputDeviceId,
@@ -116,8 +117,6 @@ const {
   toggleOutputHotkey,
   inputDeviceExist,
   outputDeviceExist,
-  audioInputs,
-  audioOutputs,
 } = usePreferences()
 
 const setupToggleInputHotkey = (event: KeyboardEvent) => setupHotkey(event, toggleInputHotkey)

client/src-tauri/tauri.conf.json

@@ -1,7 +1,7 @@
 {
   "$schema": "../node_modules/@tauri-apps/cli/config.schema.json",
   "productName": "chad",
-  "version": "0.2.14",
+  "version": "0.2.16",
   "identifier": "xyz.koptilnya.chad",
   "build": {
     "frontendDist": "../.output/public",

server/plugins/mediasoup-router.ts

@@ -25,5 +25,50 @@ export const autoConfig: mediasoup.types.RouterOptions = {
       channels: 2,
       parameters: { useinbandfec: 1, stereo: 1 },
     },
+    {
+      kind: 'video',
+      mimeType: 'video/VP8',
+      clockRate: 90000,
+      parameters: {
+        'x-google-start-bitrate': 1000,
+      },
+    },
+    {
+      kind: 'video',
+      mimeType: 'video/VP9',
+      clockRate: 90000,
+      parameters: {
+        'profile-id': 2,
+        'x-google-start-bitrate': 1000,
+      },
+    },
+    {
+      kind: 'video',
+      mimeType: 'video/h264',
+      clockRate: 90000,
+      parameters: {
+        'packetization-mode': 1,
+        'profile-level-id': '4d0032',
+        'level-asymmetry-allowed': 1,
+        'x-google-start-bitrate': 1000,
+      },
+    },
+    {
+      kind: 'video',
+      mimeType: 'video/h264',
+      clockRate: 90000,
+      parameters: {
+        'packetization-mode': 1,
+        'profile-level-id': '42e01f',
+        'level-asymmetry-allowed': 1,
+        'x-google-start-bitrate': 1000,
+      },
+    },
+    {
+      kind: 'video',
+      mimeType: 'video/AV1',
+      clockRate: 90000,
+      parameters: {},
+    },
   ],
 }

server/socket/webrtc.ts

@@ -148,7 +148,7 @@ export default function (io: SocketServer, router: types.Router) {
       }
     })
 
-    socket.on('produce', async ({ transportId, kind, rtpParameters }, cb) => {
+    socket.on('produce', async ({ transportId, kind, rtpParameters, appData }, cb) => {
       if (!socket.data.joined) {
         consola.error('Peer not joined yet')
         cb({ error: 'Peer not joined yet' })
@@ -166,7 +166,7 @@ export default function (io: SocketServer, router: types.Router) {
       }
 
       try {
-        const producer = await transport.produce({ kind, rtpParameters, appData: { socketId: socket.id } })
+        const producer = await transport.produce({ kind, rtpParameters, appData: { ...appData, socketId: socket.id } })
 
         socket.data.producers.set(producer.id, producer)
 

server/types/webrtc.ts

@@ -55,6 +55,7 @@ export interface ClientToServerEvents {
       transportId: types.WebRtcTransport['id']
       kind: types.MediaKind
       rtpParameters: types.RtpParameters
+      appData: { source: 'share' | string }
     },
     cb: EventCallback<{ id: types.Producer['id'] }>
   ) => void