2nd/10_Wiki/Topics/Coding/Mobile_Spatial_Audio_Video.md

id, title, category, status, source_trust_level, verification_status, created_at, updated_at, tags, tech_stack, applied_in, aliases

id	title	category	status	source_trust_level	verification_status	created_at	updated_at	tags	tech_stack	applied_in	aliases
mobile-spatial-audio-video	Spatial Audio / Video — AVFoundation / ExoPlayer	Coding	draft	B	conceptual	2026-05-09	2026-05-09	mobile audio video spatial vibe-coding	language applicable_to Swift / Kotlin iOS Android		Spatial Audio AirPods Dolby Atmos ExoPlayer AVPlayer HLS DASH picture-in-picture

Spatial Audio / Video

Modern audio (3D / Atmos / Spatial). Modern video (HLS / DASH / PiP / AirPlay). iOS = AVFoundation. Android = ExoPlayer / Media3.

📖 핵심 개념

• Spatial: head-tracked 3D audio (AirPods Pro / Max).
• Atmos: object-based audio.
• HLS / DASH: adaptive bitrate streaming.
• PiP: Picture-in-Picture.

💻 코드 패턴

iOS — AVAudioPlayerNode (positional)

import AVFoundation

class SpatialAudio {
    let engine = AVAudioEngine()
    let player = AVAudioPlayerNode()
    let env = AVAudioEnvironmentNode()
    
    init() {
        engine.attach(player)
        engine.attach(env)
        
        // Listener at origin
        env.listenerPosition = AVAudio3DPoint(x: 0, y: 0, z: 0)
        env.listenerAngularOrientation = AVAudio3DAngularOrientation(yaw: 0, pitch: 0, roll: 0)
        
        // Source
        player.position = AVAudio3DPoint(x: 5, y: 0, z: -10)  // 5m right, 10m forward
        
        // Connect
        engine.connect(player, to: env, format: nil)
        engine.connect(env, to: engine.mainMixerNode, format: env.outputFormat(forBus: 0))
        
        try? engine.start()
    }
    
    func play(url: URL) {
        let file = try! AVAudioFile(forReading: url)
        player.scheduleFile(file, at: nil)
        player.play()
    }
}

Spatial audio in AVPlayer (Atmos / Dolby)

let player = AVPlayer(url: hlsUrl)

// Spatial 자동 적용 — iOS 가 hardware 검사 + apply.
// Apple Music API 가 Atmos track 표시:
let song = try await MusicCatalogResource(id: songId).response()
let isAtmos = song.audioVariants.contains(.dolbyAtmos)

Head tracking (AirPods Pro/Max)

import CoreMotion

let manager = CMHeadphoneMotionManager()
if manager.isDeviceMotionAvailable {
    manager.startDeviceMotionUpdates(to: .main) { motion, _ in
        let yaw = motion?.attitude.yaw ?? 0
        let pitch = motion?.attitude.pitch ?? 0
        // 환경 listener orientation update
    }
}

→ Head 가 turn 시 audio source 정확 위치.

Picture-in-Picture (iOS)

import AVKit

let pipController = AVPictureInPictureController(playerLayer: playerLayer)
pipController.delegate = self
pipController.startPictureInPicture()

// Info.plist
// UIBackgroundModes: [audio]

// 또는 PiP 만
// UIBackgroundModes: [audio, "audio,picture-in-picture"]

HLS playback

let url = URL(string: "https://example.com/video.m3u8")!
let player = AVPlayer(url: url)
let layer = AVPlayerLayer(player: player)
view.layer.addSublayer(layer)
player.play()

// Quality control
player.currentItem?.preferredPeakBitRate = 2_000_000  // 2 Mbps cap

AirPlay

import MediaPlayer

let routePicker = AVRoutePickerView(frame: .zero)
view.addSubview(routePicker)
// 사용자가 AirPlay device 선택

→ iOS 자동.

Background audio (iOS)

// Capability: Background Modes → Audio
import AVFoundation

let session = AVAudioSession.sharedInstance()
try session.setCategory(.playback, mode: .default)
try session.setActive(true)

// Now Playing Info
import MediaPlayer

MPNowPlayingInfoCenter.default().nowPlayingInfo = [
    MPMediaItemPropertyTitle: "Song Title",
    MPMediaItemPropertyArtist: "Artist",
    MPNowPlayingInfoPropertyElapsedPlaybackTime: 0,
    MPMediaItemPropertyPlaybackDuration: 240,
]

// Remote commands (lock screen / control center)
let cc = MPRemoteCommandCenter.shared()
cc.playCommand.addTarget { _ in player.play(); return .success }
cc.pauseCommand.addTarget { _ in player.pause(); return .success }
cc.skipForwardCommand.addTarget { _ in /* +15s */; return .success }

Android — Media3 (modern ExoPlayer)

implementation("androidx.media3:media3-exoplayer:1.4.0")
implementation("androidx.media3:media3-ui:1.4.0")
implementation("androidx.media3:media3-exoplayer-hls:1.4.0")

val player = ExoPlayer.Builder(context).build()
val mediaItem = MediaItem.fromUri("https://example.com/video.m3u8")
player.setMediaItem(mediaItem)
player.prepare()
player.play()

// View
playerView.player = player

Spatial audio (Android Auro / Atmos)

val audioFormat = AudioFormat.Builder()
    .setEncoding(AudioFormat.ENCODING_E_AC3_JOC)  // Atmos
    .setSampleRate(48000)
    .build()

// 자동 — device 가 지원하면 적용

→ Hardware-dependent. Pixel / Samsung 의 spatial.

MediaSession (Android)

val session = MediaSession.Builder(context, player)
    .setCallback(object : MediaSession.Callback {
        override fun onPlaybackResumption(session: MediaSession, controller: MediaSession.ControllerInfo): ListenableFuture<MediaSession.MediaItemsWithStartPosition> {
            // ...
        }
    })
    .build()

// MediaNotification (background)
class PlayerService : MediaSessionService() {
    private var session: MediaSession? = null
    
    override fun onCreate() {
        super.onCreate()
        session = MediaSession.Builder(this, player).build()
    }
    
    override fun onGetSession(controllerInfo: MediaSession.ControllerInfo) = session
}

→ Lock screen + notification controls.

Picture-in-Picture (Android)

<!-- AndroidManifest.xml -->
<activity android:name=".VideoActivity"
    android:supportsPictureInPicture="true"
    android:configChanges="screenSize|smallestScreenSize|screenLayout|orientation">

override fun onUserLeaveHint() {
    super.onUserLeaveHint()
    if (player.isPlaying) {
        enterPipMode()
    }
}

private fun enterPipMode() {
    val params = PictureInPictureParams.Builder()
        .setAspectRatio(Rational(16, 9))
        .build()
    enterPictureInPictureMode(params)
}

HLS (Android)

val mediaItem = MediaItem.Builder()
    .setUri("https://example.com/video.m3u8")
    .setMimeType(MimeTypes.APPLICATION_M3U8)
    .build()

val mediaSource = HlsMediaSource.Factory(DefaultHttpDataSource.Factory()).createMediaSource(mediaItem)
player.setMediaSource(mediaSource)

DRM (premium content)

val drmConfiguration = MediaItem.DrmConfiguration.Builder(C.WIDEVINE_UUID)
    .setLicenseUri("https://license.example.com")
    .build()

val mediaItem = MediaItem.Builder()
    .setUri(videoUri)
    .setDrmConfiguration(drmConfiguration)
    .build()

→ Netflix / Disney+ 같은.

Adaptive bitrate

val trackSelector = DefaultTrackSelector(context).apply {
    parameters = parameters.buildUpon()
        .setMaxVideoBitrate(2_000_000)  // 2 Mbps cap
        .build()
}

ExoPlayer.Builder(context)
    .setTrackSelector(trackSelector)
    .build()

Captions / subtitles

val mediaItem = MediaItem.Builder()
    .setUri(videoUri)
    .setSubtitleConfigurations(listOf(
        MediaItem.SubtitleConfiguration.Builder(subtitleUri)
            .setMimeType(MimeTypes.TEXT_VTT)
            .setLanguage("en")
            .setSelectionFlags(C.SELECTION_FLAG_DEFAULT)
            .build()
    ))
    .build()

Streaming protocols

HLS (Apple): .m3u8 — iOS native, web 호환
DASH:        .mpd — Android / web
WebRTC:      Real-time (low latency)
SRT:         Live broadcast
RTMP:        Legacy (OBS → server)

→ HLS = 가장 호환. DASH 도 일반.

Live streaming

LL-HLS (Low-Latency HLS): 1-3s
WebRTC: < 500ms
SRT: 100-500ms (broadcast quality)

→ 라이브 video chat = WebRTC. Concert / sport = LL-HLS.

Encoding (server-side)

# FFmpeg
ffmpeg -i input.mp4 \
  -c:v libx264 -preset fast \
  -c:a aac \
  -hls_time 4 \
  -hls_list_size 0 \
  -f hls output.m3u8

# Multiple bitrate (adaptive)
ffmpeg -i input.mp4 \
  -map 0:v -map 0:v -map 0:v \
  -map 0:a -map 0:a -map 0:a \
  -filter:v:0 scale=-2:1080 -b:v:0 5M \
  -filter:v:1 scale=-2:720 -b:v:1 2.5M \
  -filter:v:2 scale=-2:480 -b:v:2 1M \
  -var_stream_map "v:0,a:0 v:1,a:1 v:2,a:2" \
  -hls_time 4 -f hls output_%v.m3u8

Audio routing (사용자 device choice)

// iOS
let session = AVAudioSession.sharedInstance()

// Override (speaker forced)
try session.overrideOutputAudioPort(.speaker)

// Bluetooth / AirPods 자동
try session.setCategory(.playAndRecord, options: [.allowBluetooth, .allowBluetoothA2DP])

Audio focus (Android)

val audioManager = getSystemService<AudioManager>()
val request = AudioFocusRequestCompat.Builder(AudioManagerCompat.AUDIOFOCUS_GAIN)
    .setAudioAttributes(AudioAttributesCompat.Builder()
        .setUsage(AudioAttributesCompat.USAGE_MEDIA)
        .setContentType(AudioAttributesCompat.CONTENT_TYPE_MUSIC)
        .build())
    .setOnAudioFocusChangeListener { focusChange ->
        when (focusChange) {
            AudioManager.AUDIOFOCUS_LOSS -> player.pause()
            AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> player.pause()
            AudioManager.AUDIOFOCUS_GAIN -> player.play()
        }
    }
    .build()

AudioManagerCompat.requestAudioFocus(audioManager!!, request)

→ 통화 시 자동 pause 등.

Cast / Chromecast

implementation("androidx.media3:media3-cast:1.4.0")

val castContext = CastContext.getSharedInstance(context)
val castPlayer = CastPlayer(castContext)

YouTube / 외부 video

// React Native
import YoutubePlayer from 'react-native-youtube-iframe';
<YoutubePlayer videoId="..." />

// Or webview
<WebView source={{ uri: 'https://youtube.com/embed/...' }} />

Media buttons / hardware controls

// iOS — automatically MPRemoteCommandCenter

// Android — MediaSession 가 자동

Performance

- 큰 video = adaptive bitrate
- 메모리 — 1 player at a time
- Background = MediaSession + foreground service (Android)
- Hardware decode (h264 / h265 / AV1)

Test

// Robolectric — Android
val player = ExoPlayer.Builder(context).build()
player.setMediaItem(MediaItem.fromUri(testUri))
player.prepare()
shadowOf(Looper.getMainLooper()).idle()

assertThat(player.isPlaying).isTrue()

🤔 의사결정 기준

사용	추천
Music / podcast	AVAudioPlayer / ExoPlayer
Video streaming	AVPlayer / Media3 ExoPlayer
Spatial / 3D	AVAudioEnvironment
Live	LL-HLS / WebRTC
DRM	Widevine / FairPlay
Background	MediaSession + capability
PiP	모두 native API

❌ 안티패턴

• Background audio + capability 없음: 종료.
• MediaSession 없음: lock screen control X.
• Single bitrate: 슬로우 network 깨짐.
• Audio focus 무시: 통화 시 음악 계속.
• Memory leak (player release X): 큰 leak.
• HW decode 안 — software decode: battery / 발열.
• Spatial 가정 + non-spatial source: hardware 이용 X.

🤖 LLM 활용 힌트

• iOS = AVFoundation + AVPlayer.
• Android = Media3 (modern ExoPlayer).
• HLS adaptive bitrate.
• MediaSession / NowPlayingInfo lock screen.
• Spatial = AVAudioEnvironmentNode + head tracking.

🔗 관련 문서

• Android_ExoPlayer_Patterns
• iOS_Charts_Health
• Web_WebRTC_Realtime