wip: in-browser transmux/code

.vscode/settings.json

@@ -33,7 +33,6 @@
   "svelte.plugin.svelte.format.config.singleQuote": true,
   "svelte.plugin.svelte.format.enable": false,
   "typescript.enablePromptUseWorkspaceTsdk": true,
-  "typescript.experimental.expandableHover": true,
   "typescript.preferences.importModuleSpecifierEnding": "minimal",
   "typescript.preferences.quoteStyle": "single",
   "typescript.suggest.autoImports": true,

package.json

@@ -28,6 +28,7 @@
 		"@urql/introspection": "^1.2.1",
 		"autoprefixer": "^10.4.21",
 		"bits-ui": "^0.22.0",
+		"block-iterator": "^1.1.1",
 		"cmdk-sv": "^0.0.19",
 		"eslint-config-standard-universal": "^1.0.9",
 		"gql.tada": "^1.8.13",
@@ -56,6 +57,7 @@
 		"@fontsource-variable/nunito": "^5.2.6",
 		"@fontsource/geist-mono": "^5.2.6",
 		"@prgm/sveltekit-progress-bar": "2.0.0",
+		"@thaunknown/url-file": "^1.0.8",
 		"@thaunknown/web-irc": "^1.0.3",
 		"@urql/core": "^6.0.1",
 		"@urql/exchange-auth": "^3.0.0",

pnpm-lock.yaml

@@ -23,6 +23,9 @@ importers:
       '@prgm/sveltekit-progress-bar':
         specifier: 2.0.0
         version: 2.0.0(@sveltejs/kit@2.37.0(@sveltejs/vite-plugin-svelte@3.1.2(svelte@4.2.19)(vite@5.4.19(terser@5.43.1)))(svelte@4.2.19)(vite@5.4.19(terser@5.43.1)))(svelte@4.2.19)
+      '@thaunknown/url-file':
+        specifier: ^1.0.8
+        version: 1.0.8
       '@thaunknown/web-irc':
         specifier: ^1.0.3
         version: 1.0.3
@@ -174,6 +177,9 @@ importers:
       bits-ui:
         specifier: ^0.22.0
         version: 0.22.0(svelte@4.2.19)
+      block-iterator:
+        specifier: ^1.1.1
+        version: 1.1.1
       cmdk-sv:
         specifier: ^0.0.19
         version: 0.0.19(svelte@4.2.19)
@@ -738,6 +744,9 @@ packages:
   '@thaunknown/simple-websocket@9.1.3':
     resolution: {integrity: sha512-pf/FCJsgWtLJiJmIpiSI7acOZVq3bIQCpnNo222UFc8Ph1lOUOTpe6LoYhhiOSKB9GUaWJEVUtZ+sK1/aBgU5Q==}
 
+  '@thaunknown/url-file@1.0.8':
+    resolution: {integrity: sha512-gacsHaItFqU4armnH32N3uYVzfd9YbDhE0LA3ZiD+2F3Qkmt4OujSn2m/9B2W2gJ1STK5nYTNAwWpcYW31WEPA==}
+
   '@thaunknown/web-irc@1.0.3':
     resolution: {integrity: sha512-gzTDs6+sAfkpuEB1IbVwBfu5btEt2D/0RGP+PwUPITIOZjZOpZLjs3d4ipvzl9eq+76otuudopmIlSD0pUT4Mw==}
 
@@ -1044,6 +1053,9 @@ packages:
     engines: {node: '>=12.20.0'}
     hasBin: true
 
+  block-iterator@1.1.1:
+    resolution: {integrity: sha512-DrjdVWZemVO4iBf4tiOXjUrY5cNesjzy0t7sIiu2rdl8cOCHRxAgKjSJFc3vBZYYMMmshUAxajl8QQh/uxXTKQ==}
+
   bottleneck@2.19.5:
     resolution: {integrity: sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw==}
 
@@ -2006,6 +2018,11 @@ packages:
     resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==}
     engines: {node: '>=8.6'}
 
+  mime@4.1.0:
+    resolution: {integrity: sha512-X5ju04+cAzsojXKes0B/S4tcYtFAJ6tTMuSPBEn9CPGlrWr8Fiw7qYeLT0XyH80HSoAoqWCaz+MWKh22P7G1cw==}
+    engines: {node: '>=16'}
+    hasBin: true
+
   mimic-response@3.1.0:
     resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==}
     engines: {node: '>=10'}
@@ -3362,6 +3379,10 @@ snapshots:
       - supports-color
       - utf-8-validate
 
+  '@thaunknown/url-file@1.0.8':
+    dependencies:
+      mime: 4.1.0
+
   '@thaunknown/web-irc@1.0.3':
     dependencies:
       grapheme-splitter: 1.0.4
@@ -3744,6 +3765,8 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  block-iterator@1.1.1: {}
+
   bottleneck@2.19.5: {}
 
   brace-expansion@1.1.11:
@@ -4833,6 +4856,8 @@ snapshots:
       braces: 3.0.3
       picomatch: 2.3.1
 
+  mime@4.1.0: {}
+
   mimic-response@3.1.0: {}
 
   minimatch@3.1.2:

src/lib/components/ui/player/libav/avio.ts

@@ -0,0 +1,130 @@
+import EventEmitter from 'events'
+
+import { concat } from 'uint8-util'
+
+import type { LazyReader } from './readers/lazy'
+import type { Segment } from './types'
+
+/**
+ * Enum for C-style seek modes.
+ * SEEK_SET is from the start of the file, SEEK_CUR is from the current position, SEEK_END is from the end of the file.
+ */
+const WHENCE = {
+  SEEK_SET: 0,
+  SEEK_CUR: 1,
+  SEEK_END: 2,
+  AVSEEK_SIZE: 65535
+} as const
+
+// Enum for the done flag.
+const DONE = {
+  INIT: 1,
+  SEGMENT: 2,
+  FLUSH: 3,
+  1: 'INIT',
+  2: 'SEGMENT',
+  3: 'FLUSH'
+} as const
+
+// AVIOHandler is a class that is used to handle the input and output from the LibAV muxer.
+// It is used to read data from a file, and write data to a file.
+// It is also used to emit segments of data to the media source.
+export class AVIOHandler extends EventEmitter<{
+  'INIT': [Segment]
+  'SEGMENT': [Segment]
+}> {
+  readonly reader
+  // Fragments should only be combined when all streams are the same as the last write.
+  #fragments: Uint8Array[] = []
+  #audioStream: number | null = null
+  #videoStream: number | null = null
+  #streamsChanged = false
+  #first_pts: bigint | null = null
+  #last_pts: bigint | null = null
+  #done = false
+  constructor (reader: LazyReader) {
+    super()
+
+    this.reader = reader // Complaint reader to read data for Input part of AVIO.
+  }
+
+  // Reads a chunk of the file, returning the data and a done (EOF) flag.
+  // This parameter is passed to the AVIOContext read callback in LibAV.
+  read = (length: number) => this.reader.read(length)
+
+  // Writes a chunk of data to the output buffer.
+  // This parameter is passed to the AVIOContext write callback in LibAV.
+  write = async (data: Uint8Array, length: number, startPts: bigint, endPts: bigint, done: boolean, audioStream: number, videoStream: number): Promise<void> => {
+    this.#first_pts = startPts
+    this.#last_pts = endPts
+    this.#done = done
+
+    // clear fragments if the stream has changed before a done.
+    if (this.#audioStream !== audioStream || this.#videoStream !== videoStream) {
+      this.#audioStream = audioStream
+      this.#videoStream = videoStream
+
+      this.#streamsChanged = true
+      this.#fragments = []
+    }
+    this.#fragments.push(new Uint8Array(data))
+  }
+
+  // Seeks to a specified position in the input file.
+  // This parameter is passed to the AVIOContext input seek callback in LibAV.
+  // The output is a bigint representing the new position in the file.
+  seek = async (offset: number, whence: typeof WHENCE[keyof typeof WHENCE]) => {
+    let position = 0
+
+    switch (whence) {
+      case WHENCE.SEEK_SET:
+        position = offset
+        await this.reader.seek(position)
+        break
+      case WHENCE.SEEK_CUR:
+        position = this.reader._offset + offset
+        await this.reader.seek(position)
+        break
+      case WHENCE.SEEK_END:
+        position = this.reader._file.size + offset
+        await this.reader.seek(position)
+        break
+      case WHENCE.AVSEEK_SIZE:
+        position = this.reader._file.size
+        break
+      default:
+        position = -1
+        break
+    }
+
+    return { value: BigInt(position) }
+  }
+
+  // Flushes the output buffer and emits the segment to the media source when done is called by the LibAV muxer.
+  done = async (type: typeof DONE[keyof typeof DONE]) => {
+    if (!this.#fragments.length) return
+
+    switch (type) {
+      case DONE.INIT:
+      case DONE.SEGMENT: {
+        this.emit(DONE[type], {
+          data: concat(this.#fragments),
+          start_ts: parseInt(this.#first_pts! as unknown as string),
+          end_ts: parseInt(this.#last_pts! as unknown as string),
+          videoStream: this.#videoStream,
+          audioStream: this.#audioStream,
+          streamsChanged: Boolean(this.#streamsChanged),
+          done: !!this.#done
+        })
+        this.#fragments = []
+        this.#first_pts = null
+        this.#last_pts = null
+        this.#streamsChanged = false
+        if (this.#done) this.reader.reset()
+        return
+      }
+      case DONE.FLUSH:
+        this.#fragments = []
+    }
+  }
+}

src/lib/components/ui/player/libav/mse.ts

@@ -0,0 +1,240 @@
+import { AVIOHandler } from './avio.ts'
+import { LazyReader as Reader } from './readers/lazy.ts'
+
+import type Decoder from './muxer/index.js'
+import type { Metadata, Segment } from './types'
+
+export class MSE {
+  mediaSource = new MediaSource()
+  ctrl = new AbortController()
+  readonly muxer
+  readonly avio
+  readonly reader
+
+  waiting: Promise<void> | null = null
+
+  seeking = false
+  bufferedInit: Segment | null = null
+  tryingToSeek: number | null = null
+
+  metadata: Metadata | null = null
+  changingStream = false
+  inflight = false
+  endOfStream = false
+  destroyed = false
+
+  sourceBuffer: SourceBuffer | undefined
+  bufferSize = 30
+  currentTime = 0
+  onhandle
+  onmetadata
+
+  constructor (file: File, muxer: Awaited<ReturnType<typeof Decoder>>, onhandle: ((handle: MediaSourceHandle) => void) = () => {}, onmetadata: ((metadata: Metadata) => void) = () => {}) {
+    this.onhandle = onhandle
+    this.onmetadata = onmetadata
+    this.reader = new Reader(file, 1024 * 1024 * 5)
+    this.avio = new AVIOHandler(this.reader)
+    this.muxer = new muxer.Decoder(this.avio.read, this.avio.seek, this.avio.write, this.avio.done, () => { })
+
+    this.avio.on('INIT', e => this.handleInit(e))
+    this.avio.on('SEGMENT', e => this.handleSegment(e))
+    this.mediaSource.addEventListener('sourceopen', () => this.handleSourceOpen(), { signal: this.ctrl.signal })
+    this.onhandle((this.mediaSource as unknown as MediaSource & { handle: MediaSourceHandle }).handle)
+  }
+
+  selectStream (streamId: number) {
+    this.sourceBuffer?.abort()
+    this.ctrl.abort()
+    if (this.sourceBuffer) this.mediaSource.removeSourceBuffer(this.sourceBuffer)
+    this.sourceBuffer = undefined
+    this.mediaSource = new MediaSource()
+    this.ctrl = new AbortController()
+    this.mediaSource.addEventListener('sourceopen', () => this.handleSourceOpen(), { signal: this.ctrl.signal })
+
+    this.changingStream = true
+    this.seeking = true
+    this.metadata = null
+    this.tryingToSeek = null
+    this.endOfStream = false
+    this.inflight = false
+    this.destroyed = false
+
+    this.waiting = this.muxer.selectStream(streamId, this.currentTime)
+    this.onhandle((this.mediaSource as unknown as MediaSource & { handle: MediaSourceHandle }).handle)
+  }
+
+  async handleInit (event: Segment) {
+    if (this.sourceBuffer) {
+      console.warn('Source buffer already exists.')
+      return
+    }
+
+    if (this.mediaSource.readyState !== 'open') {
+      this.bufferedInit = event
+      return
+    }
+
+    this.inflight = true
+    this.metadata ??= await this.muxer.metadata()
+    this.onmetadata(this.metadata!)
+    this.inflight = false
+
+    const codecs = [this.metadata!.video_codec, this.metadata!.audio_codec].filter(e => e)
+
+    const mimeType = `video/mp4; codecs="${codecs.join(', ')}"`
+    if (!MediaSource.isTypeSupported(mimeType)) {
+      throw new Error('Mimetype not supported!')
+    }
+
+    this.sourceBuffer = this.mediaSource.addSourceBuffer(mimeType)
+    this.sourceBuffer.addEventListener('updateend', this.handleUpdateEnd.bind(this), { signal: this.ctrl.signal })
+
+    this.mediaSource.duration = this.metadata!.duration / 1e6
+    this.sourceBuffer.timestampOffset = Math.max(0, event.start_ts / 1000)
+
+    this.appendBuffer(event.data, event.start_ts, false)
+  }
+
+  handleSegment (event: Segment) {
+    if (!this.sourceBuffer) return
+    this.inflight = false
+
+    this.appendBuffer(
+      event.data,
+      this.seeking ? event.start_ts : undefined,
+      event.done
+    )
+    this.seeking = false
+  }
+
+  async handleSourceOpen () {
+    if (this.inflight || this.endOfStream) return
+
+    if (!this.changingStream) await this.muxer.init()
+    if (this.bufferedInit) {
+      await this.handleInit(this.bufferedInit)
+      this.bufferedInit = null
+    }
+    if (this.changingStream && this.waiting) {
+      await this.waiting
+      this.changingStream = false
+      this.waiting = null
+    } else {
+      await this.muxer.header()
+    }
+  }
+
+  async handleUpdateEnd () {
+    if (this.destroyed) return
+
+    if (this.endOfStream) {
+      if (this.mediaSource.readyState === 'open') this.mediaSource.endOfStream()
+      this.seeking = true
+      await this.muxer.seek(0)
+      return
+    }
+
+    if (this.isBufferFull()) return
+    if (this.sourceBuffer?.updating) return
+    if (this.seeking) return
+    if (this.inflight) return
+
+    if (this.tryingToSeek) {
+      const event = this.tryingToSeek
+      this.tryingToSeek = null
+      return await this.handleSeek(event)
+    }
+
+    this.inflight = true
+    await this.muxer.next()
+  }
+
+  async handleSeek (currentTime: number) {
+    this.currentTime = currentTime
+    if (this.destroyed) return
+    if (this.mediaSource.readyState === 'ended') return
+    if (this.isBuffered(currentTime)) return
+
+    if (this.inflight) {
+      this.tryingToSeek = currentTime
+      return
+    }
+
+    this.seeking = true
+    if (this.mediaSource.readyState === 'open') this.sourceBuffer?.abort()
+
+    this.inflight = true
+    await this.muxer.seek(currentTime)
+    await this.muxer.next()
+  }
+
+  async handleProgress (currentTime: number) {
+    this.currentTime = currentTime
+
+    if (!this.sourceBuffer) return
+    if (this.destroyed) return
+    if (this.isBufferFull(currentTime)) return
+    if (this.sourceBuffer.updating) return
+    if (this.inflight) return
+
+    this.inflight = true
+    await this.muxer.next()
+  }
+
+  appendBuffer (data: Uint8Array, offset: number | undefined, done: boolean) {
+    if (this.sourceBuffer?.updating) throw new Error('Source buffer updating!')
+
+    if (Number.isInteger(offset) && this.sourceBuffer) {
+      this.sourceBuffer.timestampOffset = Math.max(0, offset! / 1000)
+    }
+
+    this.endOfStream = done
+
+    this.sourceBuffer?.appendBuffer(data.buffer as ArrayBuffer)
+
+    return done
+  }
+
+  isBufferFull (current: number = this.currentTime) {
+    if (!this.sourceBuffer?.buffered.length) return false
+
+    for (let i = 0; i < this.sourceBuffer.buffered.length; i++) {
+      const start = this.sourceBuffer.buffered.start(i)
+      const end = this.sourceBuffer.buffered.end(i)
+
+      if (current >= start && current <= end) {
+        if (end >= this.mediaSource.duration) return true
+        return end - current > this.bufferSize
+      }
+    }
+
+    return false
+  }
+
+  isBuffered (time: number) {
+    if (!this.sourceBuffer?.buffered.length) return false
+
+    for (let i = 0; i < this.sourceBuffer.buffered.length; i++) {
+      const start = this.sourceBuffer.buffered.start(i)
+      const end = this.sourceBuffer.buffered.end(i)
+
+      if (time >= start && time <= end) return true
+    }
+
+    return false
+  }
+
+  empty () {
+    if (this.sourceBuffer?.buffered.length) {
+      for (let i = 0; i < this.sourceBuffer.buffered.length; i++) {
+        this.sourceBuffer.remove(this.sourceBuffer.buffered.start(i), this.sourceBuffer.buffered.end(i))
+      }
+    }
+  }
+
+  destroy () {
+    this.destroyed = true
+    if (this.mediaSource.readyState === 'open') this.mediaSource.endOfStream()
+    this.ctrl.abort()
+  }
+}

src/lib/components/ui/player/libav/player.ts

@@ -0,0 +1,58 @@
+import { releaseProxy } from 'abslink'
+import { wrap } from 'abslink/w3c'
+
+import Worker from './worker.ts?worker'
+
+import type { Metadata } from './types'
+import type LibAVWorker from './worker.ts'
+import type { Remote } from 'abslink'
+import type { Track } from 'native'
+
+function toTrack (stream: Metadata['streams'][0]): Track {
+  return {
+    id: stream.index.toString(),
+    kind: stream.type,
+    label: stream.title || `${stream.codec} ${stream.language ? `(${stream.language})` : ''}`,
+    language: stream.language || 'unknown',
+    selected: false,
+    enabled: false
+  }
+}
+
+export default class MSEPlayer {
+  ctrl = new AbortController()
+  worker = new Worker({ name: 'MSEWorker' })
+  instance = wrap<typeof LibAVWorker>(this.worker) as unknown as Remote<typeof LibAVWorker>
+  metadata = this.instance.metadata
+  audioTracks = this.metadata.promise.then(m => m.streams.filter(s => s.type === 'audio').map(toTrack))
+  videoTracks = this.metadata.promise.then(m => m.streams.filter(s => s.type === 'video').map(toTrack))
+  video
+
+  constructor (video: HTMLVideoElement, url: string) {
+    video.addEventListener('seeking', () => this.instance.mse.handleSeek(video.currentTime), { signal: this.ctrl.signal })
+    video.addEventListener('timeupdate', () => this.instance.mse.handleProgress(video.currentTime), { signal: this.ctrl.signal })
+    video.addEventListener('play', () => this.instance.mse.handleProgress(video.currentTime), { signal: this.ctrl.signal })
+    this.video = video
+
+    this.worker.onmessage = ({ data }) => {
+      if (data.type === 'mse-handle') {
+        const time = video.currentTime
+        video.srcObject = data.handle
+        video.currentTime = time
+      }
+    }
+
+    this.instance.play(url)
+  }
+
+  selectTrack (id: string) {
+    this.instance.mse.selectStream(Number(id))
+  }
+
+  async destroy () {
+    this.video.srcObject = null
+    this.ctrl.abort()
+    await this.instance[releaseProxy]()
+    this.worker.terminate()
+  }
+}

src/lib/components/ui/player/libav/readers/lazy.ts

@@ -0,0 +1,90 @@
+/**
+ * LazyReader is a class that reads a file in chunks of a fixed size.
+ * This is useful for reading large files in a memory efficient way, though can be more intensive on the origin.
+ */
+export class LazyReader {
+  _file
+  _offset = 0
+  readonly #bufferSize = 1024 * 1024 * 5 // 5MB
+  #buffer
+  _bufPos = 0
+  _bufEnd = 0
+  ready
+
+  constructor (file: File, bufferSize = 1024 * 1024 * 5) {
+    this._file = file
+    this._offset = 0
+    this.#bufferSize = bufferSize
+    this.#buffer = new Uint8Array(this.#bufferSize)
+    // Should add an abort signal and a way to abort a current load if we seek, close or destroy.
+    this.ready = this._loadBuffer()
+  }
+
+  // Loads a chunk of the file into the buffer.
+  async _loadBuffer () {
+    let sizeToRead = this.#bufferSize
+
+    if (this._offset + this.#bufferSize > this._file.size) {
+      sizeToRead = this._file.size - this._offset // read until EOF
+    }
+
+    const result = await this._file.slice(this._offset, this._offset + sizeToRead).arrayBuffer()
+    this.#buffer.set(new Uint8Array(result), 0)
+    this._bufPos = 0
+    this._bufEnd = result.byteLength
+  }
+
+  // Reads a chunk of the file, returning the data and a done (EOF) flag.
+  async read (length: number) {
+    await this.ready
+    // todo: add better handling around EOF.
+    if (this._bufPos + length > this._bufEnd) {
+      this.ready = this._loadBuffer()
+      await this.ready
+    }
+
+    // when reading before EOF, even if the remaining data is less than length, done = false
+    // when reading at or after EOF should return done = true
+    const done = this._bufEnd === 0
+
+    const end = Math.min(this._bufPos + length, this._bufEnd)
+    const result = this.#buffer.slice(this._bufPos, end)
+    this._bufPos = end
+    this._offset += result.length
+
+    return {
+      done,
+      data: result,
+      length: result.byteLength
+    }
+  }
+
+  // Seeks to a specified position in the file.
+  async seek (position: number) {
+    await this.ready
+
+    this._offset = position
+    if (position < this._bufPos || position >= this._bufPos + this._offset) {
+      this.ready = this._loadBuffer()
+    } else {
+      this._bufPos = position - this._bufPos
+    }
+  }
+
+  // Resets the reader to the beginning of the file.
+  reset () {
+    this._offset = 0
+    this._bufPos = 0
+    this._bufEnd = 0
+    this.#buffer = new Uint8Array(this.#bufferSize)
+    this.ready = this._loadBuffer()
+    return this.ready
+  }
+
+  // Destroys the reader and releases any resources.
+  destroy () {
+    this._offset = 0
+    this._bufPos = 0
+    this._bufEnd = 0
+  }
+}

src/lib/components/ui/player/libav/readers/smart.ts

@@ -0,0 +1,51 @@
+import blockIterator from 'block-iterator'
+import 'fast-readable-async-iterator'
+
+/**
+ * This works by streaming the file and yielding blocks of the specified size.
+ * The stream's backpressure does not work in all browsers and requires a specific cache policy, leading to OOM errors.
+ * This approach allows the browser to cache the file, making seeking and replaying native like.
+ */
+export class SmartReader {
+  _file
+  _pieceSize = 0
+  #iterator: AsyncGenerator<Uint8Array, void, unknown> | undefined
+  _offset = 0
+  constructor (file: File, pieceSize = 1024 * 1024 * 5) {
+    this._file = file
+    this._pieceSize = pieceSize
+  }
+
+  async read (length = this._pieceSize) {
+    if (!this.#iterator) {
+      this.#iterator = blockIterator(this._file.slice(this._offset).stream(), length, { nopad: true })
+    }
+    const { done, value } = await this.#iterator.next(length)
+    this._offset += length
+
+    if (done) {
+      await this.reset()
+    }
+
+    return { data: value, done }
+  }
+
+  async seek (position: number) {
+    this._offset = position
+    const p = this.#iterator?.return()
+    this.#iterator = undefined
+    await p
+  }
+
+  async reset () {
+    this._offset = 0
+    const p = this.#iterator?.return()
+    this.#iterator = undefined
+    await p
+  }
+
+  destroy () {
+    this.#iterator?.return()
+    this.#iterator = undefined
+  }
+}

src/lib/components/ui/player/libav/types.d.ts

@@ -0,0 +1,36 @@
+export interface Metadata {
+  format: string
+  video_stream: number
+  video_codec: string
+  audio_stream: number
+  audio_codec: string
+  duration: number
+  timebase: number
+  streams: Stream[]
+}
+
+interface Stream {
+  index: number
+  codec_id: number
+  codec: string
+  type: string
+  title: string
+  language: string
+  width: number
+  height: number
+  time_base: number
+  duration: string
+  start_time: string
+  avg_frame_rate?: number
+  sample_aspect_ratio: number
+}
+
+export interface Segment {
+  data: Uint8Array
+  start_ts: number
+  end_ts: number
+  videoStream: number | null
+  audioStream: number | null
+  streamsChanged: boolean
+  done: boolean
+}

src/lib/components/ui/player/libav/worker.ts

@@ -0,0 +1,25 @@
+import { fromURL } from '@thaunknown/url-file'
+import { finalizer } from 'abslink'
+import { expose } from 'abslink/w3c'
+
+import { MSE } from './mse.js'
+import Decoder from './muxer'
+
+import type { Metadata } from './types'
+
+export default expose({
+  mse: null as unknown as MSE,
+  metadata: Promise.withResolvers<Metadata>(),
+
+  async play (url: string) {
+    const [muxer, file] = await Promise.all([Decoder(), fromURL(url)])
+    this.mse = new MSE(file, muxer,
+      handle => postMessage({ type: 'mse-handle', handle }, { transfer: [handle] }),
+      meta => this.metadata.resolve(meta)
+    )
+  },
+
+  [finalizer] () {
+    this.mse?.destroy()
+  }
+})

src/lib/components/ui/player/options.svelte

@@ -7,6 +7,7 @@
   import Keybinds from './keybinds.svelte'
   import { normalizeSubs, normalizeTracks, type Chapter } from './util'
 
+  import type MSEPlayer from './libav/player'
   import type PictureInPicture from './pip'
   import type { ResolvedFile } from './resolver'
   import type Subtitles from './subtitles'
@@ -37,6 +38,7 @@
   export let selectFile: (file: ResolvedFile) => void
   export let pip: PictureInPicture
   export let subtitleDelay: number
+  export let mse: MSEPlayer | undefined
 
   $: pipElement = pip.element
 
@@ -109,7 +111,27 @@
         </Keybinds>
       {:else}
         <Tree.Root bind:state={treeState}>
-          {#if 'audioTracks' in HTMLVideoElement.prototype}
+          {#if mse}
+            {#await mse.audioTracks then tracklist}
+              <Tree.Item>
+                <span slot='trigger'>Audio</span>
+                <Tree.Sub>
+                  {#each Object.entries(normalizeTracks(tracklist)) as [lang, tracks] (lang)}
+                    <Tree.Item>
+                      <span slot='trigger' class='capitalize'>{lang}</span>
+                      <Tree.Sub>
+                        {#each tracks as track (track.id)}
+                          <Tree.Item active={track.enabled} on:click={() => { selectAudio(track.id); open = false }}>
+                            <span>{track.label}</span>
+                          </Tree.Item>
+                        {/each}
+                      </Tree.Sub>
+                    </Tree.Item>
+                  {/each}
+                </Tree.Sub>
+              </Tree.Item>
+            {/await}
+          {:else if 'audioTracks' in HTMLVideoElement.prototype}
             <Tree.Item>
               <span slot='trigger'>Audio</span>
               <Tree.Sub>
@@ -128,7 +150,27 @@
               </Tree.Sub>
             </Tree.Item>
           {/if}
-          {#if 'videoTracks' in HTMLVideoElement.prototype}
+          {#if mse}
+            {#await mse.audioTracks then tracklist}
+              <Tree.Item>
+                <span slot='trigger'>Video</span>
+                <Tree.Sub>
+                  {#each Object.entries(normalizeTracks(tracklist)) as [lang, tracks] (lang)}
+                    <Tree.Item>
+                      <span slot='trigger' class='capitalize'>{lang}</span>
+                      <Tree.Sub>
+                        {#each tracks as track (track.id)}
+                          <Tree.Item active={track.enabled} on:click={() => { selectVideo(track.id); open = false }}>
+                            <span>{track.label}</span>
+                          </Tree.Item>
+                        {/each}
+                      </Tree.Sub>
+                    </Tree.Item>
+                  {/each}
+                </Tree.Sub>
+              </Tree.Item>
+            {/await}
+          {:else if 'videoTracks' in HTMLVideoElement.prototype}
             <Tree.Item>
               <span slot='trigger'>Video</span>
               <Tree.Sub>

src/lib/components/ui/player/player.svelte

@@ -31,6 +31,7 @@
   import DownloadStats from './downloadstats.svelte'
   import EpisodesModal from './episodesmodal.svelte'
   import { condition, loadWithDefaults } from './keybinds.svelte'
+  import MSEPlayer from './libav/player'
   import Options from './options.svelte'
   import PictureInPicture from './pip'
   import Seekbar from './seekbar.svelte'
@@ -90,6 +91,7 @@
 
   let subtitles: Subs | undefined
   let deband: VideoDeband | undefined
+  let mse: MSEPlayer | undefined
 
   const pip = new PictureInPicture()
   $: pip._setElements(video, subtitles, deband)
@@ -181,6 +183,7 @@
   }
   function selectAudio (id: string) {
     if (id) {
+      if (mse) return mse.selectTrack(id)
       for (const track of video.audioTracks ?? []) {
         track.enabled = track.id === id
         playAnimation(track.label)
@@ -190,6 +193,7 @@
   }
   function selectVideo (id: string) {
     if (id) {
+      if (mse) return mse.selectTrack(id)
       for (const track of video.videoTracks ?? []) {
         track.selected = track.id === id
         playAnimation(track.label)
@@ -276,6 +280,26 @@
     }
   }
 
+  function customMSE (node: HTMLVideoElement, mseTranscode: boolean) {
+    const createMSE = () => {
+      if (mediaInfo.file.url.endsWith('.mkv')) {
+        mse = new MSEPlayer(node, mediaInfo.file.url)
+      }
+    }
+    if (mseTranscode) createMSE()
+    return {
+      destroy: () => mse?.destroy(),
+      update: (mseTranscode: boolean) => {
+        if (mseTranscode) {
+          createMSE()
+        } else {
+          mse?.destroy()
+          mse = undefined
+        }
+      }
+    }
+  }
+
   let completed = false
   async function checkCompletion () {
     if (!completed && $settings.playerAutocomplete) {
@@ -733,10 +757,11 @@
 <div class='w-full h-full relative content-center bg-black overflow-clip text-left touch-none' class:fitWidth class:seeking class:pip={pictureInPictureElement} bind:this={wrapper} on:navigate={() => resetMove(2000)} on:wheel={handleWheel} on:keydown={stopAnimation} on:focusin={stopAnimation} on:pointerenter={stopAnimation} on:pointermove={stopAnimation}>
   <video class='w-full h-full touch-none' preload='metadata' class:cursor-none={immersed} class:cursor-pointer={isMiniplayer} class:object-cover={fitWidth} class:opacity-0={$settings.playerDeband || seeking || pictureInPictureElement} class:absolute={$settings.playerDeband} class:top-0={$settings.playerDeband}
     use:createSubtitles
+    use:customMSE={$settings.mseTranscode}
     use:createDeband={$settings.playerDeband}
     use:holdToFF={'pointer'}
     crossorigin='anonymous'
-    src={mediaInfo.file.url}
+    src={!$settings.mseTranscode ? mediaInfo.file.url : undefined}
     bind:videoHeight
     bind:videoWidth
     bind:currentTime
@@ -782,7 +807,7 @@
           Subtitle delay: {subtitleDelay} sec
         </div>
       {/if}
-      <Options {wrapper} bind:openSubs {video} {seekTo} {selectAudio} {selectVideo} {fullscreen} {chapters} {subtitles} {videoFiles} {selectFile} {pip} bind:playbackRate={$playbackRate} bind:subtitleDelay id='player-options-button-top'
+      <Options {wrapper} bind:openSubs {video} {seekTo} {selectAudio} {selectVideo} {fullscreen} {chapters} {subtitles} {videoFiles} {selectFile} {pip} {mse} bind:playbackRate={$playbackRate} bind:subtitleDelay id='player-options-button-top'
         class='{($settings.minimalPlayerUI || SUPPORTS.isAndroid) ? 'inline-flex' : 'mobile:inline-flex hidden'} p-3 size-12 absolute z-[1] top-4 left-4 bg-black/20 pointer-events-auto transition-opacity delay-150 select:opacity-100 {immersed && 'opacity-0'}' />
       {#if fastForwarding}
         <div class='absolute top-10 font-bold text-sm animate-[fade-in_.4s_ease] flex items-center leading-none bg-black/60 px-4 py-2 rounded-2xl'>x2 <FastForward class='ml-2' size='12' fill='currentColor' /></div>
@@ -862,7 +887,7 @@
               x{$playbackRate?.toFixed(1)}
             </div>
           {/if}
-          <Options {fullscreen} {wrapper} {seekTo} bind:openSubs {video} {selectAudio} {selectVideo} {chapters} {subtitles} {videoFiles} {selectFile} {pip} bind:playbackRate={$playbackRate} bind:subtitleDelay id='player-options-button' />
+          <Options {fullscreen} {wrapper} {seekTo} bind:openSubs {video} {selectAudio} {selectVideo} {chapters} {subtitles} {videoFiles} {selectFile} {pip} {mse} bind:playbackRate={$playbackRate} bind:subtitleDelay id='player-options-button' />
           {#if subtitles}
             <Button class='p-3 size-12' variant='ghost' on:click={openSubs} on:keydown={keywrap(openSubs)} data-up='#player-seekbar'>
               <Subtitles size='24px' fill='currentColor' strokeWidth='0' />

src/lib/modules/native.ts

@@ -13,7 +13,7 @@ const dummyFiles = [
     type: 'video/webm',
     size: 1234567890,
     path: '/Amebku.webm',
-    url: '/video.mkv',
+    url: 'http://localhost:7344/test3.mkv',
     id: 0
   }
 // {

src/lib/modules/settings/defaults.ts

@@ -42,5 +42,6 @@ export default {
   playerSkip: false,
   playerSkipFiller: false,
   minimalPlayerUI: false,
-  androidStorageType: 'cache'
+  androidStorageType: 'cache',
+  mseTranscode: false
 }

src/routes/app/settings/+page.svelte

@@ -104,4 +104,9 @@
       </div>
     </SettingCard>
   {/if}
+
+  <div class='font-weight-bold text-xl font-bold'>Experimental Settings</div>
+  <SettingCard let:id title='MSE Transcoding' description={'WARNING EXPERIMENTAL!!!\n\nTranscodes video/audio on the fly using on the CPU. This is VERY performance intensive, can cause crashes and video playback errors.\n\nCan enable multi-track audio and video on platforms that don\'t natively support it.'}>
+    <Switch {id} bind:checked={$settings.mseTranscode} />
+  </SettingCard>
 </div>

src/service-worker/index.ts

@@ -7,7 +7,7 @@ import { build, files, prerendered, version } from '$service-worker'
 
 const fallbackURL = '/offline.html'
 
-precacheAndRoute([fallbackURL, ...prerendered, ...build, ...files.filter(e => !['/Ameku.webm', '/video.mkv', '/NotoSansHK.woff2', '/NotoSansJP.woff2', '/NotoSansKR.woff2'].includes(e))].map(url => ({ url, revision: version })))
+precacheAndRoute([fallbackURL, ...prerendered, ...build, ...files.filter(e => !['/Ameku.webm', '/video.mkv', '/test3.mkv', '/NotoSansHK.woff2', '/NotoSansJP.woff2', '/NotoSansKR.woff2'].includes(e))].map(url => ({ url, revision: version })))
 cleanupOutdatedCaches()
 clientsClaim()
 skipWaiting()