LiveKit 向けバーチャル背景とノイズキャンセリング

import { Track, TrackProcessor, VideoProcessorOptions } from 'livekit-client';
import { tsvb } from 'effects-sdk';

export class EffectsVideoProcessor implements TrackProcessor<Track.Kind.Video, VideoProcessorOptions> {
  name = 'effects-sdk';
  processedTrack?: MediaStreamTrack;
  sdk = new tsvb('YOUR_CUSTOMER_ID');

  constructor() {
    this.sdk.config({ provider: 'webgpu' });
    this.sdk.preload();
  }

  async init(opts: VideoProcessorOptions) {
    this.sdk.clear();
    this.sdk.useStream(new MediaStream([opts.track]));
    this.processedTrack = this.sdk.getStream()?.getVideoTracks()[0];
    await new Promise((ready) => (this.sdk.onReady = ready));
    this.sdk.run();
  }

  async restart(opts: VideoProcessorOptions) {
    await this.init(opts);
  }

  async destroy() {
    this.sdk.stop();
    this.sdk.clear();
    this.processedTrack = undefined;
  }
}

import { createLocalVideoTrack } from 'livekit-client';

const processor = new EffectsVideoProcessor();
const cameraTrack = await createLocalVideoTrack();
await cameraTrack.setProcessor(processor);
await room.localParticipant.publishTrack(cameraTrack);

// Change effects at any time, without republishing
processor.sdk.setBlur(0.6);
processor.sdk.setBackground('https://example.com/office.jpg');

import { atsvb } from 'audio-effects-sdk';

const sdk = new atsvb('YOUR_CUSTOMER_ID');
sdk.config({ preset: 'speed', sample_rate: 16000 });
sdk.preload();

const mic = await navigator.mediaDevices.getUserMedia({ audio: true });
sdk.useStream(mic);

sdk.onReady = async () => {
  sdk.run();
  const cleanTrack = sdk.getStream().getAudioTracks()[0];
  // Publish the processed track instead of the raw microphone
  await room.localParticipant.publishTrack(cleanTrack);
};

// pubspec.yaml: livekit_client from github.com/EffectsSDK/livekit-client-sdk-flutter
// Web only: load tsvb-web.js in index.html and call this first
VideoEffectsSDKExt.initialize('YOUR_CUSTOMER_ID');

final videoTrack = await LocalVideoTrack.createCameraTrack(
  CameraCaptureOptions(effectsSdkRequired: true),
);
await videoTrack.start();

// Android and iOS: authorize the track
final status = await videoTrack.auth('YOUR_CUSTOMER_ID');
if (status == AuthStatus.active) {
  await videoTrack.setPipelineMode(PipelineMode.blur);
  await videoTrack.setBlurPower(0.6);
}