Creepso/multi-downloader-nx
1
0
Fork 0
Code Issues 38 Pull requests 10 Projects Releases 70 Packages Wiki Activity Actions

Allow --syncTiming to work on all files

Browse source 
Forgive me father for I have sinned (this code with it's "temporary" solutions).

This maps all downloaded "versions" so that each version can have it's delay applied as needed
This commit is contained in:
AnimeDL 2024-10-07 20:53:23 -07:00
parent dfa0a31def
commit 146c5f10a9
6 changed files with 189 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
@types/downloaderTypes.d.ts vendored
View file

@ -29,4 +29,9 @@ export type DownloadedMedia = {
type: 'Subtitle',
signs?: boolean,
cc: boolean
} & sxItem )
} & sxItem )
export type DownloadedMediaMap = {
version: string;
files: DownloadedMedia[];
}

31
adn.ts
View file

@ -28,7 +28,7 @@ import { AvailableFilenameVars } from './modules/module.args';
// Types
import type { ServiceClass } from './@types/serviceClassInterface';
import type { AuthData, AuthResponse, SearchData, SearchResponse, SearchResponseItem } from './@types/messageHandler';
import type { DownloadedMedia, sxItem } from './@types/downloaderTypes';
import type { DownloadedMedia, DownloadedMediaMap, sxItem } from './@types/downloaderTypes';
import type { ADNSearch, ADNSearchShow } from './@types/adnSearch';
import type { ADNVideo, ADNVideos } from './@types/adnVideos';
import type { ADNPlayerConfig } from './@types/adnPlayerConfig';
@ -315,7 +315,7 @@ export default class AnimationDigitalNetwork implements ServiceClass {
return { isOk: true, value: selEpsArr };
}
public async muxStreams(data: DownloadedMedia[], options: yargs.ArgvType) {
public async muxStreams(data: DownloadedMedia[], mediaMap: DownloadedMediaMap[], options: yargs.ArgvType) {
this.cfg.bin = await yamlCfg.loadBinCfg();
let hasAudioStreams = false;
if (options.novids || data.filter(a => a.type === 'Video').length === 0)
@ -324,6 +324,7 @@ export default class AnimationDigitalNetwork implements ServiceClass {
hasAudioStreams = true;
}
const merger = new Merger({
mediaMap,
onlyVid: hasAudioStreams ? data.filter(a => a.type === 'Video').map((a) : MergerInput => {
return {
lang: a.lang,
@ -404,7 +405,7 @@ export default class AnimationDigitalNetwork implements ServiceClass {
return { isOk: false, reason: new Error('Failed to download media list') };
} else {
if (!options.skipmux) {
await this.muxStreams(res.data, { ...options, output: res.fileName });
await this.muxStreams(res.data, res.mediaMap, { ...options, output: res.fileName });
} else {
console.info('Skipping mux');
}
@ -433,6 +434,12 @@ export default class AnimationDigitalNetwork implements ServiceClass {
}
const files: DownloadedMedia[] = [];
const mediaMap: DownloadedMediaMap[] = [];
const fileMap: DownloadedMediaMap = {
version: data.id.toString(),
files: []
};
let dlFailed = false;
let dlVideoOnce = false; // Variable to save if best selected video quality was downloaded
@ -697,6 +704,11 @@ export default class AnimationDigitalNetwork implements ServiceClass {
path: `${tsFile}.ts`,
lang: audDub
});
fileMap.files.push({
type: 'Video',
path: `${tsFile}.ts`,
lang: audDub
});
dlVideoOnce = true;
}
} else{
@ -763,6 +775,11 @@ export default class AnimationDigitalNetwork implements ServiceClass {
lang: langsData.languages.find(a=>a.code=='jpn')!,
type: 'Chapters'
});
fileMap.files.push({
path: `${tsFile}.txt`,
lang: langsData.languages.find(a=>a.code=='jpn')!,
type: 'Chapters'
});
} catch {
console.error('Failed to write chapter file');
}
@ -886,6 +903,11 @@ export default class AnimationDigitalNetwork implements ServiceClass {
...sxData as sxItem,
cc: false
});
fileMap.files.push({
type: 'Subtitle',
...sxData as sxItem,
cc: false
});
}
subIndex++;
}
@ -896,9 +918,12 @@ export default class AnimationDigitalNetwork implements ServiceClass {
console.info('Subtitles downloading skipped!');
}
mediaMap.push(fileMap);
return {
error: dlFailed,
data: files,
mediaMap,
fileName: fileName ? (path.isAbsolute(fileName) ? fileName : path.join(this.cfg.dir.content, fileName)) || './unknown' : './unknown'
};
}

41
ao.ts
View file

@ -32,7 +32,7 @@ import type { AuthData, AuthResponse, SearchData, SearchResponse, SearchResponse
import type { AOSearchResult, AnimeOnegaiSearch } from './@types/animeOnegaiSearch';
import type { AnimeOnegaiSeries } from './@types/animeOnegaiSeries';
import type { AnimeOnegaiSeasons, Episode } from './@types/animeOnegaiSeasons';
import type { DownloadedMedia, sxItem } from './@types/downloaderTypes';
import type { DownloadedMedia, DownloadedMediaMap, sxItem } from './@types/downloaderTypes';
import type { AnimeOnegaiStream } from './@types/animeOnegaiStream';
type parsedMultiDubDownload = {
@ -314,7 +314,7 @@ export default class AnimeOnegai implements ServiceClass {
return false;
} else {
if (!options.skipmux) {
await this.muxStreams(res.data, { ...options, output: res.fileName });
await this.muxStreams(res.data, res.mediaMap, { ...options, output: res.fileName });
} else {
console.info('Skipping mux');
}
@ -326,7 +326,7 @@ export default class AnimeOnegai implements ServiceClass {
return true;
}
public async muxStreams(data: DownloadedMedia[], options: yargs.ArgvType) {
public async muxStreams(data: DownloadedMedia[], mediaMap: DownloadedMediaMap[], options: yargs.ArgvType) {
this.cfg.bin = await yamlCfg.loadBinCfg();
let hasAudioStreams = false;
if (options.novids || data.filter(a => a.type === 'Video').length === 0)
@ -335,6 +335,7 @@ export default class AnimeOnegai implements ServiceClass {
hasAudioStreams = true;
}
const merger = new Merger({
mediaMap,
onlyVid: hasAudioStreams ? data.filter(a => a.type === 'Video').map((a) : MergerInput => {
return {
lang: a.lang,
@ -402,6 +403,7 @@ export default class AnimeOnegai implements ServiceClass {
public async downloadMediaList(medias: parsedMultiDubDownload, options: yargs.ArgvType) : Promise<{
data: DownloadedMedia[],
mediaMap: DownloadedMediaMap[],
fileName: string,
error: boolean
} | undefined> {
@ -421,6 +423,7 @@ export default class AnimeOnegai implements ServiceClass {
}
const files: DownloadedMedia[] = [];
const mediaMap: DownloadedMediaMap[] = [];
let subIndex = 0;
let dlFailed = false;
@ -428,6 +431,11 @@ export default class AnimeOnegai implements ServiceClass {
for (const media of medias.data) {
console.info(`Requesting: [E.${media.episode.ID}] ${mediaName}`);
const fileMap: DownloadedMediaMap = {
version: media.episode.ID.toString(),
files: []
};
const AuthHeaders = {
headers: {
@ -700,6 +708,11 @@ export default class AnimeOnegai implements ServiceClass {
path: `${tsFile}.video.mp4`,
lang: lang
});
fileMap.files.push({
type: 'Video',
path: `${tsFile}.video.mp4`,
lang: lang
});
}
}
@ -721,6 +734,11 @@ export default class AnimeOnegai implements ServiceClass {
path: `${tsFile}.audio.mp4`,
lang: lang
});
fileMap.files.push({
type: 'Audio',
path: `${tsFile}.audio.mp4`,
lang: lang
});
console.info('Decryption done for audio');
}
}
@ -734,6 +752,11 @@ export default class AnimeOnegai implements ServiceClass {
path: `${tsFile}.video.mp4`,
lang: lang
});
fileMap.files.push({
type: 'Video',
path: `${tsFile}.video.mp4`,
lang: lang
});
}
if (audioDownloaded) {
files.push({
@ -741,6 +764,11 @@ export default class AnimeOnegai implements ServiceClass {
path: `${tsFile}.audio.mp4`,
lang: lang
});
fileMap.files.push({
type: 'Audio',
path: `${tsFile}.audio.mp4`,
lang: lang
});
}
}
}
@ -782,6 +810,11 @@ export default class AnimeOnegai implements ServiceClass {
...sxData as sxItem,
cc: false
});
fileMap.files.push({
type: 'Subtitle',
...sxData as sxItem,
cc: false
});
} else{
console.warn(`Failed to download subtitle: ${sxData.file}`);
}
@ -795,11 +828,13 @@ export default class AnimeOnegai implements ServiceClass {
else{
console.info('Subtitles downloading skipped!');
}
mediaMap.push(fileMap);
await this.sleep(options.waittime);
}
return {
error: dlFailed,
data: files,
mediaMap,
fileName: fileName ? (path.isAbsolute(fileName) ? fileName : path.join(this.cfg.dir.content, fileName)) || './unknown' : './unknown'
};
}

58
crunchy.ts
View file

@ -32,7 +32,7 @@ import { exec } from './modules/sei-helper-fixes';
// Types
import type { CrunchyDownloadOptions, CrunchyEpMeta, CrunchyMuxOptions, CrunchyMultiDownload, ParseItem, SeriesSearch, SeriesSearchItem } from './@types/crunchyTypes';
import type { DownloadedMedia, sxItem } from './@types/downloaderTypes';
import type { DownloadedMedia, DownloadedMediaMap, sxItem } from './@types/downloaderTypes';
import type { CrunchySearch } from './@types/crunchySearch';
import type { CrunchyEpisodeList, CrunchyEpisode } from './@types/crunchyEpisodeList';
import type { ObjectInfo } from './@types/objectInfo';
@ -976,7 +976,7 @@ export default class Crunchy implements ServiceClass {
return false;
} else {
if (!options.skipmux) {
await this.muxStreams(res.data, { ...options, output: res.fileName });
await this.muxStreams(res.data, res.mediaMap, { ...options, output: res.fileName });
} else {
console.info('Skipping mux');
}
@ -1186,6 +1186,7 @@ export default class Crunchy implements ServiceClass {
public async downloadMediaList(medias: CrunchyEpMeta, options: CrunchyDownloadOptions) : Promise<{
data: DownloadedMedia[],
mediaMap: DownloadedMediaMap[],
fileName: string,
error: boolean
} | undefined> {
@ -1205,6 +1206,7 @@ export default class Crunchy implements ServiceClass {
}
const files: DownloadedMedia[] = [];
const mediaMap: DownloadedMediaMap[] = [];
if(medias.data.every(a => !a.playback)){
console.warn('Video not available!');
@ -1221,6 +1223,11 @@ export default class Crunchy implements ServiceClass {
// Make sure we have a media id without a : in it
const currentMediaId = (mMeta.mediaId.includes(':') ? mMeta.mediaId.split(':')[1] : mMeta.mediaId);
const fileMap: DownloadedMediaMap = {
version: currentMediaId,
files: []
};
//Make sure token is up-to-date
await this.refreshToken(true, true);
let currentVersion;
@ -1770,6 +1777,12 @@ export default class Crunchy implements ServiceClass {
lang: lang,
isPrimary: isPrimary
});
fileMap.files.push({
type: 'Video',
path: `${tsFile}.video.m4s`,
lang: lang,
isPrimary: isPrimary
});
}
}
@ -1792,6 +1805,12 @@ export default class Crunchy implements ServiceClass {
lang: lang,
isPrimary: isPrimary
});
fileMap.files.push({
type: 'Audio',
path: `${tsFile}.audio.m4s`,
lang: lang,
isPrimary: isPrimary
});
console.info('Decryption done for audio');
}
}
@ -1806,6 +1825,12 @@ export default class Crunchy implements ServiceClass {
lang: lang,
isPrimary: isPrimary
});
fileMap.files.push({
type: 'Video',
path: `${tsFile}.video.m4s`,
lang: lang,
isPrimary: isPrimary
});
}
if (audioDownloaded) {
files.push({
@ -1814,6 +1839,12 @@ export default class Crunchy implements ServiceClass {
lang: lang,
isPrimary: isPrimary
});
fileMap.files.push({
type: 'Audio',
path: `${tsFile}.audio.m4s`,
lang: lang,
isPrimary: isPrimary
});
}
}
} else if (!options.novids) {
@ -1971,6 +2002,12 @@ export default class Crunchy implements ServiceClass {
lang: lang,
isPrimary: isPrimary
});
fileMap.files.push({
type: 'Video',
path: `${tsFile}.ts`,
lang: lang,
isPrimary: isPrimary
});
dlVideoOnce = true;
}
} else{
@ -2008,6 +2045,11 @@ export default class Crunchy implements ServiceClass {
lang: lang,
type: 'Chapters'
});
fileMap.files.push({
path: `${tsFile}.txt`,
lang: lang,
type: 'Chapters'
});
} catch {
console.error('Failed to write chapter file');
}
@ -2095,6 +2137,12 @@ export default class Crunchy implements ServiceClass {
cc: isCC,
signs: isSigns,
});
fileMap.files.push({
type: 'Subtitle',
...sxData as sxItem,
cc: isCC,
signs: isSigns,
});
}
else{
console.warn(`Failed to download subtitle: ${sxData.file}`);
@ -2109,16 +2157,19 @@ export default class Crunchy implements ServiceClass {
console.info('Subtitles downloading skipped!');
}
mediaMap.push(fileMap);
await this.sleep(options.waittime);
}
return {
error: dlFailed,
data: files,
mediaMap,
fileName: fileName ? (path.isAbsolute(fileName) ? fileName : path.join(this.cfg.dir.content, fileName)) || './unknown' : './unknown'
};
}
public async muxStreams(data: DownloadedMedia[], options: CrunchyMuxOptions) {
public async muxStreams(data: DownloadedMedia[], mediaMap: DownloadedMediaMap[], options: CrunchyMuxOptions) {
this.cfg.bin = await yamlCfg.loadBinCfg();
let hasAudioStreams = false;
if (options.novids || data.filter(a => a.type === 'Video').length === 0)
@ -2127,6 +2178,7 @@ export default class Crunchy implements ServiceClass {
hasAudioStreams = true;
}
const merger = new Merger({
mediaMap,
onlyVid: hasAudioStreams ? data.filter(a => a.type === 'Video').map((a) : MergerInput => {
return {
lang: a.lang,

35
hidive.ts
View file

@ -36,7 +36,7 @@ import type { NewHidiveSeries } from './@types/newHidiveSeries';
import type { Episode, NewHidiveEpisodeExtra, NewHidiveSeason, NewHidiveSeriesExtra } from './@types/newHidiveSeason';
import type { NewHidiveEpisode } from './@types/newHidiveEpisode';
import type { NewHidivePlayback, Subtitle } from './@types/newHidivePlayback';
import type { DownloadedMedia, sxItem } from './@types/downloaderTypes';
import type { DownloadedMedia, DownloadedMediaMap, sxItem } from './@types/downloaderTypes';
export default class Hidive implements ServiceClass {
public cfg: yamlCfg.ConfigObject;
@ -546,7 +546,7 @@ export default class Hidive implements ServiceClass {
return { isOk: false, reason: new Error('Failed to download media list') };
} else {
if (!options.skipmux) {
await this.muxStreams(res.data, { ...options, output: res.fileName }, false);
await this.muxStreams(res.data, res.mediaMap, { ...options, output: res.fileName }, false);
} else {
console.info('Skipping mux');
}
@ -635,7 +635,7 @@ export default class Hidive implements ServiceClass {
return { isOk: false, reason: new Error('Failed to download media list') };
} else {
if (!options.skipmux) {
await this.muxStreams(res.data, { ...options, output: res.fileName }, false);
await this.muxStreams(res.data, res.mediaMap, { ...options, output: res.fileName }, false);
} else {
console.info('Skipping mux');
}
@ -650,6 +650,7 @@ export default class Hidive implements ServiceClass {
public async downloadMPD(streamPlaylists: MPDParsed, subs: Subtitle[], selectedEpisode: NewHidiveEpisodeExtra, options: Record<any, any>) {
//let fileName: string;
const files: DownloadedMedia[] = [];
const mediaMap: DownloadedMediaMap[] = [];
const variables: Variable[] = [];
let dlFailed = false;
const subsMargin = 0;
@ -657,6 +658,11 @@ export default class Hidive implements ServiceClass {
let encryptionKeys: KeyContainer[] = [];
if (!canDecrypt) console.warn('Decryption not enabled!');
const fileMap: DownloadedMediaMap = {
version: selectedEpisode.id.toString(),
files: []
};
if (!this.cfg.bin.ffmpeg)
this.cfg.bin = await yamlCfg.loadBinCfg();
@ -838,6 +844,12 @@ export default class Hidive implements ServiceClass {
lang: chosenAudios[0].language,
isPrimary: true
});
fileMap.files.push({
type: 'Video',
path: `${tsFile}.video.m4s`,
lang: chosenAudios[0].language,
isPrimary: true
});
}
} else {
console.warn('mp4decrypt not found, files need decryption. Decryption Keys:', encryptionKeys);
@ -919,6 +931,12 @@ export default class Hidive implements ServiceClass {
lang: chosenAudioSegments.language,
isPrimary: chosenAudioSegments.default
});
fileMap.files.push({
type: 'Audio',
path: `${tsFile}.audio.m4s`,
lang: chosenAudioSegments.language,
isPrimary: chosenAudioSegments.default
});
console.info('Decryption done for audio');
}
} else {
@ -967,6 +985,11 @@ export default class Hidive implements ServiceClass {
...sxData as sxItem,
cc: false
});
fileMap.files.push({
type: 'Subtitle',
...sxData as sxItem,
cc: false
});
} else{
console.warn(`Failed to download subtitle: ${sxData.file}`);
}
@ -980,14 +1003,17 @@ export default class Hidive implements ServiceClass {
console.info('Subtitles downloading skipped!');
}
mediaMap.push(fileMap);
return {
error: dlFailed,
data: files,
mediaMap,
fileName: fileName ? (path.isAbsolute(fileName) ? fileName : path.join(this.cfg.dir.content, fileName)) || './unknown' : './unknown'
};
}
public async muxStreams(data: DownloadedMedia[], options: Record<any, any>, inverseTrackOrder: boolean = true) {
public async muxStreams(data: DownloadedMedia[], mediaMap: DownloadedMediaMap[], options: Record<any, any>, inverseTrackOrder: boolean = true) {
this.cfg.bin = await yamlCfg.loadBinCfg();
let hasAudioStreams = false;
if (options.novids || data.filter(a => a.type === 'Video').length === 0)
@ -996,6 +1022,7 @@ export default class Hidive implements ServiceClass {
hasAudioStreams = true;
}
const merger = new Merger({
mediaMap,
onlyVid: hasAudioStreams ? data.filter(a => a.type === 'Video').map((a) : MergerInput => {
return {
lang: a.lang,

31
modules/module.merger.ts
View file

@ -11,6 +11,7 @@ import ffprobe from 'ffprobe';
import {spawn} from 'node:child_process';
import {OggOpusDecodedAudio, OggOpusDecoder} from 'ogg-opus-decoder';
import SynAudio, {MultipleClipMatch, MultipleClipMatchFirst} from 'synaudio';
import { DownloadedMediaMap } from '../@types/downloaderTypes';
export type MergerInput = {
path: string,
@ -38,6 +39,7 @@ export type ParsedFont = {
}
export type MergerOptions = {
mediaMap?: DownloadedMediaMap[],
videoAndAudio: MergerInput[],
onlyVid: MergerInput[],
onlyAudio: MergerInput[],
@ -115,6 +117,9 @@ class Merger {
public async createDelays() {
const bin = await yamlCfg.loadBinCfg();
const allFiles = [...this.options.onlyAudio, ...this.options.videoAndAudio, ...this.options.onlyVid, ...this.options.subtitles];
if (this.options.chapters)
allFiles.push(...this.options.chapters);
const audios = [...this.options.onlyAudio, ...this.options.videoAndAudio];
if (audios.length < 2)
return;
@ -167,9 +172,35 @@ class Merger {
sampleOffset: maxSampleOffset - item.sampleOffset
}));
//Iterate over found matches
invertedArray.forEach((clip: MultipleClipMatch | MultipleClipMatchFirst) => {
const audio = audios.find(a => a.path === clip.name)!;
audio.delay = (clip.sampleOffset || 0 ) / audio.duration!.sampleRate;
// Iterate over each DownloadedMediaMap in the array
for (const mediaMap of this.options.mediaMap!) {
// Iterate over the files array in each DownloadedMediaMap
for (const media of mediaMap.files) {
// Check if the path matches
if (media.path === clip.name) {
console.log('Matching path found:', media.path);
// Re-iterate over the files array where the match was found
mediaMap.files.forEach((file) => {
const mergerFile = allFiles.find(a =>
('path' in a ? a.path : a.file) === file.path
);
if (mergerFile)
mergerFile.delay = audio.delay;
else
console.error(`File ${file.path} was not found in allFiles array, this shouldn't happen`);
});
// We only want the first match, so break out of the loop
break;
}
}
}
});
}