Fix --syncTiming

modules/module.merger.ts

@@ -8,13 +8,17 @@ import { AvailableMuxer } from './module.args';
 import { exec } from './sei-helper-fixes';
 import { console } from './log';
 import ffprobe from 'ffprobe';
+import {spawn} from 'node:child_process';
+import {OggOpusDecodedAudio, OggOpusDecoder} from 'ogg-opus-decoder';
+import SynAudio, {MultipleClipMatch, MultipleClipMatchFirst} from 'synaudio';
 
 export type MergerInput = {
   path: string,
   lang: LanguageItem,
-  duration?: number,
+  duration?: OggOpusDecodedAudio,
   delay?: number,
   isPrimary?: boolean,
+  totalDuration?: number,
 }
 
 export type SubtitleInput = {
@@ -58,7 +62,7 @@ export type MergerOptions = {
 }
 
 class Merger {
-  
+
   constructor(private options: MergerOptions) {
     if (this.options.skipSubMux)
       this.options.subtitles = [];
@@ -66,46 +70,107 @@ class Merger {
       this.options.videoTitle = this.options.videoTitle.replace(/"/g, '\'');
   }
 
-  public async createDelays() {
-    //Don't bother scanning it if there is only 1 vna stream
-    if (this.options.videoAndAudio.length > 1) {
-      const bin = await yamlCfg.loadBinCfg();
-      const vnas = this.options.videoAndAudio;
-      //get and set durations on each videoAndAudio Stream
-      for (const [vnaIndex, vna] of vnas.entries()) {
-        const streamInfo = await ffprobe(vna.path, { path: bin.ffprobe as string });
-        const videoInfo = streamInfo.streams.filter(stream => stream.codec_type == 'video');
-        vnas[vnaIndex].duration = parseInt(videoInfo[0].duration as string);
-      }
-      //Sort videoAndAudio streams by duration (shortest first)
-      vnas.sort((a,b) => {
-        if (!a.duration || !b.duration) return -1;
-        return a.duration - b.duration;
+  async convertFile(path: string, ffmpeg: string): Promise<Uint8Array> {
+    return new Promise((resolve, reject) => {
+      const options = [
+        '-t',
+        '60',
+        '-i',
+        path,
+        '-vn',
+        '-c:a',
+        'libopus',
+        '-f',
+        'ogg',
+        'pipe:1'
+      ];
+      console.info(ffmpeg, options.join(' '));
+      const ffmpegSpawn = spawn(ffmpeg, options);
+      const data: number[] = [];
+
+      ffmpegSpawn.stdout.on('data', (chunk) => {
+        data.push(...chunk);
       });
-      //Set Delays
-      const shortestDuration = vnas[0].duration;
-      for (const [vnaIndex, vna] of vnas.entries()) {
-        //Don't calculate the shortestDuration track
-        if (vnaIndex == 0) {
-          if (!vna.isPrimary && vna.isPrimary !== undefined) 
-            console.warn('Shortest video isn\'t primary, this might lead to problems with subtitles. Please report on github or discord if you experience issues.');
-          continue;
+
+      ffmpegSpawn.stderr.on('data', (err) => {
+        console.debug(err.toString().trimEnd());
+      });
+
+      ffmpegSpawn.on('close', (code) => {
+        if (code !== 0) {
+          console.error('Ffmpeg exited with code: ', code);
+          return reject();
         }
-        if (vna.duration && shortestDuration) {
-          //Calculate the tracks delay
-          vna.delay = Math.ceil((vna.duration-shortestDuration) * 1000) / 1000;
-          //TODO: set primary language for audio so it can be used to determine which track needs the delay
-          //The above is a problem in the event that it isn't the dub that needs the delay, but rather the sub.
-          //Alternatively: Might not work: it could be checked if there are multiple of the same video language, and if there is
-          //more than 1 of the same video language, then do the subtitle delay on CC, else normal language.
-          const subtitles = this.options.subtitles.filter(sub => sub.language.code == vna.lang.code);
-          for (const [subIndex, sub] of subtitles.entries()) {
-            if (vna.isPrimary) subtitles[subIndex].delay = vna.delay;
-            else if (sub.closedCaption) subtitles[subIndex].delay = vna.delay;
-          }
+        resolve(new Uint8Array(data));
+      });
+    });
+  }
+
+  async decodeAudio(data: Uint8Array): Promise<OggOpusDecodedAudio> {
+    const decoder = new OggOpusDecoder();
+    await decoder.ready;
+
+    return decoder.decode(data);
+  }
+
+  public async createDelays() {
+    const bin = await yamlCfg.loadBinCfg();
+    const audios = [...this.options.onlyAudio, ...this.options.videoAndAudio];
+    if (audios.length < 2)
+      return;
+
+    if (bin.ffmpeg === undefined || bin.ffmpeg.trim().length === 0)
+      return console.error('Unable to sync timing without ffmpeg!');
+
+    for (const audio of audios) {
+      try {
+        const track = (
+          await this.decodeAudio(
+            await this.convertFile(audio.path, bin.ffmpeg)
+          )
+        );
+        if (track.errors.length > 0) {
+          console.error(`Unable to decode ${audio.path}: ${track.errors}`);
+          return;
         }
+        audio.duration = track;
+        const streamInfo = await ffprobe(audio.path, { path: bin.ffprobe as string });
+        const compare = streamInfo.streams.find(s => s.codec_type === 'video') || streamInfo.streams.find(s => s.codec_type === 'audio');
+        audio.totalDuration = parseInt(compare!.duration!);
+      } catch (e) {
+        console.error(`Unable to generate sync timing because of file ${audio.path}: ${e}`);
+        return;
       }
     }
+
+    audios.sort((a, b) => a.totalDuration! - b.totalDuration!);
+
+    const synAudio = new SynAudio();
+    const audioArray = await synAudio.syncMultiple(
+      audios.map((audio) => {
+        return {
+          name: audio.path,
+          data: {
+            channelData: audio.duration!.channelData,
+            samplesDecoded: audio.duration!.samplesDecoded,
+          }
+        };
+      })
+    );
+
+    // Find max sampleOffset value
+    const maxSampleOffset = Math.max(...audioArray[0].map(item => item.sampleOffset));
+
+    // Invert the sampleOffset values
+    const invertedArray = audioArray[0].map(item => ({
+      ...item,
+      sampleOffset: maxSampleOffset - item.sampleOffset
+    }));
+
+    invertedArray.forEach((clip: MultipleClipMatch | MultipleClipMatchFirst) => {
+      const audio = audios.find(a => a.path === clip.name)!;
+      audio.delay = (clip.sampleOffset || 0 ) / audio.duration!.sampleRate;
+    });
   }
 
   public FFmpeg() : string {
@@ -118,8 +183,9 @@ class Merger {
 
     for (const vid of this.options.videoAndAudio) {
       if (vid.delay && hasVideo) {
+        console.info('Inserting delay vid', vid.delay);
         args.push(
-          `-itsoffset -${Math.ceil(vid.delay*1000)}ms`
+          `-itsoffset ${Math.round(vid.delay * 1000) / 1000}`
         );
       }
       args.push(`-i "${vid.path}"`);
@@ -147,6 +213,10 @@ class Merger {
     }
 
     for (const aud of this.options.onlyAudio) {
+      if (aud.delay) {
+        console.info('Inserting delay', aud.delay);
+        args.push(`-itsoffset -${Math.ceil(aud.delay * 1000) / 1000}`);
+      }
       args.push(`-i "${aud.path}"`);
       metaData.push(`-map ${index}`);
       metaData.push(`-metadata:s:a:${audioIndex} language=${aud.lang.code}`);
@@ -158,7 +228,7 @@ class Merger {
       const sub = this.options.subtitles[index];
       if (sub.delay) {
         args.push(
-          `-itsoffset -${Math.ceil(sub.delay*1000)}ms`
+          `-itsoffset -${Math.round(sub.delay*1000)}ms`
         );
       }
       args.push(`-i "${sub.file}"`);
@@ -229,7 +299,7 @@ class Merger {
       const videoTrackNum = this.options.inverseTrackOrder ? '1' : '0';
       if (vid.delay) {
         args.push(
-          `--sync ${audioTrackNum}:-${Math.ceil(vid.delay*1000)}`
+          `--sync ${audioTrackNum}:-${Math.round(vid.delay*1000)}`
         );
       }
       if (!hasVideo || this.options.keepAllVideos) {
@@ -264,6 +334,11 @@ class Merger {
     }
 
     for (const aud of this.options.onlyAudio) {
+      if (aud.delay) {
+        args.push(
+          `--sync 0:-${Math.round(aud.delay*1000)}`
+        );
+      }
       const trackName = aud.lang.name;
       args.push('--track-name', `0:"${trackName}"`);
       args.push(`--language 0:${aud.lang.code}`);
@@ -283,7 +358,7 @@ class Merger {
       for (const subObj of this.options.subtitles) {
         if (subObj.delay) {
           args.push(
-            `--sync 0:-${Math.ceil(subObj.delay*1000)}`
+            `--sync 0:-${Math.round(subObj.delay*1000)}`
           );
         }
         args.push('--track-name', `0:"${(subObj.language.language || subObj.language.name) + `${subObj.closedCaption === true ? ` ${this.options.ccTag}` : ''}` + `${subObj.signs === true ? ' Signs' : ''}`}"`);
@@ -380,7 +455,7 @@ class Merger {
               path: fontPath,
               mime: mime,
             });
-          } 
+          }
         }
       }
     }

package.json

@@ -55,9 +55,11 @@
     "lookpath": "^1.2.2",
     "m3u8-parsed": "^1.3.0",
     "mpd-parser": "^1.3.0",
+    "ogg-opus-decoder": "^1.6.12",
     "open": "^8.4.2",
     "protobufjs": "^7.3.2",
     "sei-helper": "^3.3.0",
+    "synaudio": "^0.3.5",
     "ws": "^8.17.1",
     "yaml": "^2.4.5",
     "yargs": "^17.7.2"