From 584a36545b14fa6a699e48caa30a2fe84458c83f Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Mon, 3 May 2021 19:49:50 +0200 Subject: [PATCH 01/51] Fixed audio error --- funi.js | 208 ++++++++++++++++++------------------- modules/module.app-args.js | 7 ++ 2 files changed, 107 insertions(+), 108 deletions(-) diff --git a/funi.js b/funi.js index e6b7c7c..3ffd504 100644 --- a/funi.js +++ b/funi.js @@ -526,72 +526,21 @@ async function downloadStreams(){ let dlFailedA = false; - if (!argv.novids) { + video: if (!argv.novids) { // download video let reqVideo = await getData({ url: videoUrl, useProxy: (argv.ssp ? false : true), debug: argv.debug, }); - if (!reqVideo.ok) { return; } + if (!reqVideo.ok) { break video; } let chunkList = m3u8(reqVideo.res.body); - chunkList.baseUrl = videoUrl.split('/').slice(0, -1).join('/') + '/'; let tsFile = path.join(cfg.dir.content, fnOutput); if (chunkList.segments[0].uri.match(/streaming_video_(\d+)_(\d+)_(\d+)\.ts/)) { - if (fs.existsSync(tsFile + '.ts')) { - let rwts = await shlp.question(`[Q] File «${tsFile + '.ts'}» already exists! Rewrite? (y/N)`); - rwts = rwts || 'N'; - if (!['Y', 'y'].includes(rwts[0])) { - return; - } - fs.unlinkSync(tsFile + '.ts') - } - - let chunk = chunkList.segments[0] - - let reqKey = await getData({ - url: chunk.key.uri, - responseType: 'buffer' - }) - if (!reqKey.ok) { return; } - let key = reqKey.res.body; - let iv = Buffer.alloc(16); - let ivs = chunk.key.iv ? chunk.key.iv : [0, 0, 0, 1]; - for (let i in ivs) { - iv.writeUInt32BE(ivs[i], i * 4); - } - key = crypto.createDecipheriv('aes-128-cbc', key, iv); - - let progress, intervall; - - function logInfo() { - if (progress && progress.percent && progress.transferred) - console.log(`[INFO] Downloaded ${progress.percent.toFixed(2) * 100}% (${(progress.transferred/1024).toFixed(0)}kb/${progress.total?(progress.total/1024).toFixed(0) + 'kb':'unknown'})`); - } - - let res = (await got({ - url: chunk.uri, - headers: { - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:70.0) Gecko/20100101 Firefox/70.0', - }, - responseType: 'buffer' - }).on("downloadProgress", (pro) => { - progress = pro - if (intervall === undefined) - intervall = setInterval(logInfo, 2500) - }) - .catch(error => console.log(`[ERROR] ${error.name}: ${error.code||error.message}`))) - clearInterval(intervall) - - if (!res.body) { return; } - let dec = key.update(res.body); - dec = Buffer.concat([dec, key.final()]); - fs.writeFileSync(tsFile + '.ts', dec) - - console.log(`[INFO] Finished ${tsFile}`) + await downloadFile(tsFile, chunkList) } else { let proxyHLS = false; if (argv.proxy && !argv.ssp) { @@ -639,61 +588,10 @@ async function downloadStreams(){ if (!reqAudio.ok) { return; } let chunkListA = m3u8(reqAudio.res.body); - chunkListA.baseUrl = plAud.uri.split('/').slice(0, -1).join('/') + '/'; let tsFileA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`); - if (fs.existsSync(tsFileA + '.ts')) { - let rwts = await shlp.question(`[Q] File «${tsFileA + '.ts'}» already exists! Rewrite? (y/N)`); - rwts = rwts || 'N'; - if (!['Y', 'y'].includes(rwts[0])) { - return; - } - fs.unlinkSync(tsFileA + '.ts') - } - - let chunk = chunkListA.segments[0] - - let reqKey = await getData({ - url: chunk.key.uri, - responseType: 'buffer' - }) - - if (!reqKey.ok) { return; } - let key = reqKey.res.body; - let iv = Buffer.alloc(16); - let ivs = chunk.key.iv ? chunk.key.iv : [0, 0, 0, 1]; - for (let i in ivs) { - iv.writeUInt32BE(ivs[i], i * 4); - } - key = crypto.createDecipheriv('aes-128-cbc', key, iv); - - let progress, intervall; - - function logInfo() { - if (progress && progress.percent && progress.transferred) - console.log(`[INFO] Downloaded ${progress.percent.toFixed(2) * 100}% (${(progress.transferred/1024).toFixed(0)}kb/${progress.total?(progress.total/1024).toFixed(0) + 'kb':'unknown'})`); - } - - let res = (await got({ - url: chunk.uri, - headers: { - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:70.0) Gecko/20100101 Firefox/70.0', - }, - responseType: 'buffer' - }).on("downloadProgress", (pro) => { - progress = pro - if (intervall === undefined) - intervall = setInterval(logInfo, 2500) - }) - .catch(error => console.log(`[ERROR] ${error.name}: ${error.code||error.message}`))) - clearInterval(intervall) - if (!res.body) { return; } - let dec = key.update(res.body); - dec = Buffer.concat([dec, key.final()]); - fs.writeFileSync(tsFileA + '.ts', dec) - - console.log(`[INFO] Finished ${tsFileA}`) + await downloadFile(tsFileA, chunkListA) } // add subs @@ -734,6 +632,7 @@ async function downloadStreams(){ let muxTrg = path.join(cfg.dir.content, fnOutput); let muxTrgA = ''; let tshTrg = path.join(cfg.dir.trash, fnOutput); + let tshTrgA = '' if(!fs.existsSync(`${muxTrg}.ts`) || !fs.statSync(`${muxTrg}.ts`).isFile()){ console.log('\n[INFO] TS file not found, skip muxing video...\n'); @@ -742,6 +641,7 @@ async function downloadStreams(){ if(plAud.uri){ muxTrgA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`); + tshTrgA = path.join(cfg.dir.trash, fnOutput + `.${plAud.language}`) if(!fs.existsSync(`${muxTrgA}.ts`) || !fs.statSync(`${muxTrgA}.ts`).isFile()){ console.log('\n[INFO] TS file not found, skip muxing video...\n'); return; @@ -800,13 +700,13 @@ async function downloadStreams(){ fs.unlinkSync(`${muxTrg}.json`); } else if(usableFFmpeg){ - let ffext = !argv.mp4 ? 'mkv' : 'mp4'; + let ffext = 'mp4'//!argv.mp4 ? 'mkv' : 'mp4'; let ffmux = `-i "${muxTrg}.ts" `; if(plAud.uri){ ffmux += `-i "${muxTrgA}.ts" `; } ffmux += addSubs ? `-i "${muxTrg}${subsExt}" ` : ''; - ffmux += '-map 0 -c:v copy -c:a copy '; + ffmux += '-map 0 -map 1:a -c:v copy -c:a copy '; ffmux += addSubs ? '-map 1 ' : ''; ffmux += addSubs && !argv.mp4 ? '-c:s ass ' : ''; ffmux += addSubs && argv.mp4 ? '-c:s mov_text ' : ''; @@ -826,12 +726,16 @@ async function downloadStreams(){ } else if(argv.nocleanup){ fs.renameSync(muxTrg+'.ts', tshTrg + '.ts'); + if (plAud.uri) + fs.renameSync(muxTrgA+'.ts', tshTrgA + '.ts') if(subsUrl && addSubs){ fs.renameSync(muxTrg +subsExt, tshTrg +subsExt); } } else{ fs.unlinkSync(muxTrg+'.ts'); + if (plAud.uri) + fs.unlinkSync(muxTrgA+'.ts') if(subsUrl && addSubs){ fs.unlinkSync(muxTrg +subsExt); } @@ -839,6 +743,94 @@ async function downloadStreams(){ console.log('\n[INFO] Done!\n'); } +async function downloadFile(filename, chunkList) { + if (fs.existsSync(filename + '.ts')) { + let rwts = await shlp.question(`[Q] File «${filename + '.ts'}» already exists! Rewrite? (y/N)`); + rwts = rwts || 'N'; + if (!['Y', 'y'].includes(rwts[0])) { + return; + } + fs.unlinkSync(filename + '.ts') + } + + let parts = [], start = Date.now(); + + console.log(`[INFO] Started ${filename}.ts`) + for (let i = 0; i < chunkList.segments.length / argv.partsize; i++) { + let cur = [] + for (let a = 0; a < Math.min(argv.partsize, chunkList.segments.length - (i * argv.partsize)); a++) { + cur.push(downloadPart(chunkList.segments[i * argv.partsize + a], i * argv.partsize + a, chunkList.segments.length)) + } + parts = parts.concat(await Promise.all(cur)); + logDownloadInfo(start, (i) * argv.partsize + Math.min(argv.partsize, chunkList.segments.length - (i * argv.partsize)), + chunkList.segments.length, (i) * argv.partsize + Math.min(argv.partsize, chunkList.segments.length - (i * argv.partsize)), + chunkList.segments.length) + } + + if (parts.length !== chunkList.segments.length) { + console.log("[ERROR] Some parts are missing") + return; + } + + for (let i = 0; i < chunkList.segments.length; i++) { + fs.writeFileSync(filename + '.ts', parts[i].content, { flag: 'a' }) + } + + console.log(`[INFO] Finished ${filename}.ts`) +} + +async function downloadPart(chunk, index) { + + let key = await generateCrypto(chunk, index) + + let res = (await got({ + url: chunk.uri, + headers: { + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:70.0) Gecko/20100101 Firefox/70.0', + 'Range': `bytes=${chunk.byterange.offset}-${chunk.byterange.offset+chunk.byterange.length-1}` + }, + responseType: 'buffer' + }).catch(error => console.log(`[ERROR] ${error.name}: ${error.code||error.message}`))) + + if (!res.body) { return; } + + let dec = key.update(res.body); + dec = Buffer.concat([dec, key.final()]); + return { content: dec, index: index} +} + +let keys = {} +async function generateCrypto(chunk, index) { + let key = keys[chunk.key.uri] + if (!key) { + let reqKey = await getData({ + url: chunk.key.uri, + responseType: 'buffer' + }) + + if (!reqKey.ok) { console.log("[ERROR] Can't get key"); return; } + key = reqKey.res.body; + keys[chunk.key.uri] = key; + } + let iv = Buffer.alloc(16); + let ivs = chunk.key.iv ? chunk.key.iv : [0, 0, 0, index]; + for (let i in ivs) { + iv.writeUInt32BE(ivs[i], i * 4); + } + key = crypto.createDecipheriv('aes-128-cbc', key, iv); + return key; +} + +/* Snacked from hls-download */ +function logDownloadInfo (dateStart, partsDL, partsTotal, partsDLRes, partsTotalRes) { + const dateElapsed = Date.now() - dateStart; + const percentFxd = (partsDL / partsTotal * 100).toFixed(); + const percent = percentFxd < 100 ? percentFxd : (partsTotal == partsDL ? 100 : 99); + const revParts = parseInt(dateElapsed * (partsTotal / partsDL - 1)); + const time = shlp.formatTime((revParts / 1000).toFixed()); + console.log(`[INFO] ${partsDLRes} of ${partsTotalRes} parts downloaded [${percent}%] (${time})`); +} + // make proxy URL function buildProxy(proxyBaseUrl, proxyAuth){ if(!proxyBaseUrl.match(/^(https?|socks4|socks5):/)){ diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 16abd6a..46add5d 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -174,6 +174,13 @@ const appArgv = (cfg, langsData) => { describe: 'Show this help', type: 'boolean' }) + .option('partsize', { + alias: 'p', + group: 'Downloading:', + describe: 'The amount of parts that should be downloaded in paralell', + type: 'number', + default: 10 + }) // usage .example([ ['$0 --search "My Hero"', 'search "My Hero" in title'], -- 2.45.2 From 456fc0756785f9ddcb598b8201db2be0864ff8c8 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Mon, 3 May 2021 19:52:00 +0200 Subject: [PATCH 02/51] Removed debug code --- funi.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funi.js b/funi.js index 3ffd504..280200f 100644 --- a/funi.js +++ b/funi.js @@ -700,7 +700,7 @@ async function downloadStreams(){ fs.unlinkSync(`${muxTrg}.json`); } else if(usableFFmpeg){ - let ffext = 'mp4'//!argv.mp4 ? 'mkv' : 'mp4'; + let ffext = !argv.mp4 ? 'mkv' : 'mp4'; let ffmux = `-i "${muxTrg}.ts" `; if(plAud.uri){ ffmux += `-i "${muxTrgA}.ts" `; -- 2.45.2 From a0bb3dfedaf64e18cfecc7c1d18a0e335d220616 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Mon, 3 May 2021 22:38:48 +0200 Subject: [PATCH 03/51] Changes as proposed in #59 Updated module requirement --- funi.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/funi.js b/funi.js index 280200f..83c20fd 100644 --- a/funi.js +++ b/funi.js @@ -21,10 +21,11 @@ const crypto = require("crypto"); const got = require('got'); // extra -const appYargs = require('./modules/module.app-args'); -const getYamlCfg = require('./modules/module.cfg-loader'); -const getData = require('./modules/module.getdata.js'); -const vttConvert = require('./modules/module.vttconvert'); +const moduleFolder = path.join(__dirname, "/modules") +const appYargs = require(path.join(moduleFolder, 'module.app-args')); +const getYamlCfg = require(path.join(moduleFolder, 'module.cfg-loader')); +const getData = require(path.join(moduleFolder, 'module.getdata.js')); +const vttConvert = require(path.join(moduleFolder, 'module.vttconvert')); // new-cfg const cfgFolder = path.join(__dirname, '/config'); -- 2.45.2 From 628fcbeb241aa909e23582b87cac33d3d89ad361 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Tue, 4 May 2021 00:05:16 +0200 Subject: [PATCH 04/51] Added resume files --- funi.js | 49 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/funi.js b/funi.js index 83c20fd..cbca627 100644 --- a/funi.js +++ b/funi.js @@ -745,7 +745,15 @@ async function downloadStreams(){ } async function downloadFile(filename, chunkList) { - if (fs.existsSync(filename + '.ts')) { + let offset = 0; + fileCheck: if (fs.existsSync(filename + '.ts')) { + if (fs.existsSync(filename + '.ts.resume')) { + const resume = JSON.parse(fs.readFileSync(`${filename}.ts.resume`)) + if (resume.total === chunkList.segments.length) { + offset = resume.downloaded + break fileCheck; + } + } let rwts = await shlp.question(`[Q] File «${filename + '.ts'}» already exists! Rewrite? (y/N)`); rwts = rwts || 'N'; if (!['Y', 'y'].includes(rwts[0])) { @@ -754,29 +762,36 @@ async function downloadFile(filename, chunkList) { fs.unlinkSync(filename + '.ts') } - let parts = [], start = Date.now(); + start = Date.now(); console.log(`[INFO] Started ${filename}.ts`) - for (let i = 0; i < chunkList.segments.length / argv.partsize; i++) { + for (let i = offset; i < chunkList.segments.length; i+=argv.partsize) { let cur = [] - for (let a = 0; a < Math.min(argv.partsize, chunkList.segments.length - (i * argv.partsize)); a++) { - cur.push(downloadPart(chunkList.segments[i * argv.partsize + a], i * argv.partsize + a, chunkList.segments.length)) + for (let a = 0; a < Math.min(argv.partsize, chunkList.segments.length - i); a++) { + cur.push(downloadPart(chunkList.segments[i + a], i + a, chunkList.segments.length) + .catch(e => e)) } - parts = parts.concat(await Promise.all(cur)); - logDownloadInfo(start, (i) * argv.partsize + Math.min(argv.partsize, chunkList.segments.length - (i * argv.partsize)), - chunkList.segments.length, (i) * argv.partsize + Math.min(argv.partsize, chunkList.segments.length - (i * argv.partsize)), + + let p = await Promise.all(cur); + if (p.some(el => el instanceof Error)) { + console.log(`[ERROR] An error occured while downloading ${filename}.ts`) + return; + } + + fs.writeFileSync(`${filename}.ts.resume`, JSON.stringify({ total: chunkList.segments.length, downloaded: i + argv.partsize }, null, 4)) + + for (let a = 0; a < p.length; a++) { + fs.writeFileSync(filename + '.ts', p[a].content, { flag: 'a' }) + } + + logDownloadInfo(start, i + Math.min(argv.partsize, chunkList.segments.length - i), + chunkList.segments.length, i + Math.min(argv.partsize, chunkList.segments.length - i), chunkList.segments.length) } - - if (parts.length !== chunkList.segments.length) { - console.log("[ERROR] Some parts are missing") - return; - } - - for (let i = 0; i < chunkList.segments.length; i++) { - fs.writeFileSync(filename + '.ts', parts[i].content, { flag: 'a' }) - } + if (fs.existsSync(`${filename}.ts.resume`)) + fs.unlinkSync(`${filename}.ts.resume`) + console.log(`[INFO] Finished ${filename}.ts`) } -- 2.45.2 From db5a607f7489555c9d64dd21505586a316a9bdbc Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Tue, 4 May 2021 00:49:02 +0200 Subject: [PATCH 05/51] Fixed show search --- modules/module.app-args.js | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 46add5d..4051cc1 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -175,7 +175,6 @@ const appArgv = (cfg, langsData) => { type: 'boolean' }) .option('partsize', { - alias: 'p', group: 'Downloading:', describe: 'The amount of parts that should be downloaded in paralell', type: 'number', -- 2.45.2 From b68d942b5281c4cb528744a4b36bb0a292cb332a Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Tue, 4 May 2021 14:46:06 +0200 Subject: [PATCH 06/51] Fixing eslint errors --- funi.js | 68 +++---- modules/module.app-args.js | 338 +++++++++++++++++------------------ modules/module.cfg-loader.js | 4 +- modules/module.getdata.js | 4 +- modules/module.vttconvert.js | 2 +- 5 files changed, 208 insertions(+), 208 deletions(-) diff --git a/funi.js b/funi.js index cbca627..5a0c12c 100644 --- a/funi.js +++ b/funi.js @@ -17,11 +17,11 @@ const shlp = require('sei-helper'); const { lookpath } = require('lookpath'); const m3u8 = require('m3u8-parsed'); const streamdl = require('hls-download'); -const crypto = require("crypto"); +const crypto = require('crypto'); const got = require('got'); // extra -const moduleFolder = path.join(__dirname, "/modules") +const moduleFolder = path.join(__dirname, '/modules'); const appYargs = require(path.join(moduleFolder, 'module.app-args')); const getYamlCfg = require(path.join(moduleFolder, 'module.cfg-loader')); const getData = require(path.join(moduleFolder, 'module.getdata.js')); @@ -414,16 +414,16 @@ async function downloadStreams(){ plStreams = {}, plLayersStr = [], plLayersRes = {}, - plMaxLayer = 1; - plNewIds = 1; + plMaxLayer = 1, + plNewIds = 1, plAud = { uri: '' }; // new uris let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/; if(plQualityLinkList.playlists[0].uri.match(vplReg)){ if(plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128']){ - let audioData = plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128']; - audioEl = Object.keys(audioData); + let audioData = plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128'], + audioEl = Object.keys(audioData); audioData = audioData[audioEl[0]]; plAud = { ...audioData, ...{ langStr: audioEl[0] } }; } @@ -437,7 +437,7 @@ async function downloadStreams(){ return -1; } return 0; - }) + }); } for(let s of plQualityLinkList.playlists){ @@ -541,7 +541,7 @@ async function downloadStreams(){ let tsFile = path.join(cfg.dir.content, fnOutput); if (chunkList.segments[0].uri.match(/streaming_video_(\d+)_(\d+)_(\d+)\.ts/)) { - await downloadFile(tsFile, chunkList) + await downloadFile(tsFile, chunkList); } else { let proxyHLS = false; if (argv.proxy && !argv.ssp) { @@ -592,10 +592,10 @@ async function downloadStreams(){ let tsFileA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`); - await downloadFile(tsFileA, chunkListA) + await downloadFile(tsFileA, chunkListA); } - // add subs + // add subs let subsUrl = stDlPath; let subsExt = !argv.mp4 || argv.mp4 && !argv.mks && argv.ass ? '.ass' : '.srt'; let addSubs = argv.mks && subsUrl ? true : false; @@ -633,7 +633,7 @@ async function downloadStreams(){ let muxTrg = path.join(cfg.dir.content, fnOutput); let muxTrgA = ''; let tshTrg = path.join(cfg.dir.trash, fnOutput); - let tshTrgA = '' + let tshTrgA = ''; if(!fs.existsSync(`${muxTrg}.ts`) || !fs.statSync(`${muxTrg}.ts`).isFile()){ console.log('\n[INFO] TS file not found, skip muxing video...\n'); @@ -642,7 +642,7 @@ async function downloadStreams(){ if(plAud.uri){ muxTrgA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`); - tshTrgA = path.join(cfg.dir.trash, fnOutput + `.${plAud.language}`) + tshTrgA = path.join(cfg.dir.trash, fnOutput + `.${plAud.language}`); if(!fs.existsSync(`${muxTrgA}.ts`) || !fs.statSync(`${muxTrgA}.ts`).isFile()){ console.log('\n[INFO] TS file not found, skip muxing video...\n'); return; @@ -728,7 +728,7 @@ async function downloadStreams(){ else if(argv.nocleanup){ fs.renameSync(muxTrg+'.ts', tshTrg + '.ts'); if (plAud.uri) - fs.renameSync(muxTrgA+'.ts', tshTrgA + '.ts') + fs.renameSync(muxTrgA+'.ts', tshTrgA + '.ts'); if(subsUrl && addSubs){ fs.renameSync(muxTrg +subsExt, tshTrg +subsExt); } @@ -736,7 +736,7 @@ async function downloadStreams(){ else{ fs.unlinkSync(muxTrg+'.ts'); if (plAud.uri) - fs.unlinkSync(muxTrgA+'.ts') + fs.unlinkSync(muxTrgA+'.ts'); if(subsUrl && addSubs){ fs.unlinkSync(muxTrg +subsExt); } @@ -748,9 +748,9 @@ async function downloadFile(filename, chunkList) { let offset = 0; fileCheck: if (fs.existsSync(filename + '.ts')) { if (fs.existsSync(filename + '.ts.resume')) { - const resume = JSON.parse(fs.readFileSync(`${filename}.ts.resume`)) + const resume = JSON.parse(fs.readFileSync(`${filename}.ts.resume`)); if (resume.total === chunkList.segments.length) { - offset = resume.downloaded + offset = resume.downloaded; break fileCheck; } } @@ -759,45 +759,45 @@ async function downloadFile(filename, chunkList) { if (!['Y', 'y'].includes(rwts[0])) { return; } - fs.unlinkSync(filename + '.ts') + fs.unlinkSync(filename + '.ts'); } - start = Date.now(); + let start = Date.now(); - console.log(`[INFO] Started ${filename}.ts`) + console.log(`[INFO] Started ${filename}.ts`); for (let i = offset; i < chunkList.segments.length; i+=argv.partsize) { - let cur = [] + let cur = []; for (let a = 0; a < Math.min(argv.partsize, chunkList.segments.length - i); a++) { cur.push(downloadPart(chunkList.segments[i + a], i + a, chunkList.segments.length) - .catch(e => e)) + .catch(e => e)); } let p = await Promise.all(cur); if (p.some(el => el instanceof Error)) { - console.log(`[ERROR] An error occured while downloading ${filename}.ts`) + console.log(`[ERROR] An error occured while downloading ${filename}.ts`); return; } - fs.writeFileSync(`${filename}.ts.resume`, JSON.stringify({ total: chunkList.segments.length, downloaded: i + argv.partsize }, null, 4)) + fs.writeFileSync(`${filename}.ts.resume`, JSON.stringify({ total: chunkList.segments.length, downloaded: i + argv.partsize }, null, 4)); for (let a = 0; a < p.length; a++) { - fs.writeFileSync(filename + '.ts', p[a].content, { flag: 'a' }) + fs.writeFileSync(filename + '.ts', p[a].content, { flag: 'a' }); } logDownloadInfo(start, i + Math.min(argv.partsize, chunkList.segments.length - i), chunkList.segments.length, i + Math.min(argv.partsize, chunkList.segments.length - i), - chunkList.segments.length) + chunkList.segments.length); } if (fs.existsSync(`${filename}.ts.resume`)) - fs.unlinkSync(`${filename}.ts.resume`) + fs.unlinkSync(`${filename}.ts.resume`); - console.log(`[INFO] Finished ${filename}.ts`) + console.log(`[INFO] Finished ${filename}.ts`); } async function downloadPart(chunk, index) { - let key = await generateCrypto(chunk, index) + let key = await generateCrypto(chunk, index); let res = (await got({ url: chunk.uri, @@ -806,25 +806,25 @@ async function downloadPart(chunk, index) { 'Range': `bytes=${chunk.byterange.offset}-${chunk.byterange.offset+chunk.byterange.length-1}` }, responseType: 'buffer' - }).catch(error => console.log(`[ERROR] ${error.name}: ${error.code||error.message}`))) + }).catch(error => console.log(`[ERROR] ${error.name}: ${error.code||error.message}`))); if (!res.body) { return; } let dec = key.update(res.body); dec = Buffer.concat([dec, key.final()]); - return { content: dec, index: index} + return { content: dec, index: index}; } -let keys = {} +let keys = {}; async function generateCrypto(chunk, index) { - let key = keys[chunk.key.uri] + let key = keys[chunk.key.uri]; if (!key) { let reqKey = await getData({ url: chunk.key.uri, responseType: 'buffer' - }) + }); - if (!reqKey.ok) { console.log("[ERROR] Can't get key"); return; } + if (!reqKey.ok) { console.log('[ERROR] Can\'t get key'); return; } key = reqKey.res.body; keys[chunk.key.uri] = key; } diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 4051cc1..2143264 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -1,195 +1,195 @@ const yargs = require('yargs'); -const appArgv = (cfg, langsData) => { +const appArgv = (cfg) => { // init return yargs.parserConfiguration({ - "duplicate-arguments-array": false, + 'duplicate-arguments-array': false, }) // main - .wrap(Math.min(120)) // yargs.terminalWidth() - .help(false).version(false) - .usage('Usage: $0 [options]') + .wrap(Math.min(120)) // yargs.terminalWidth() + .help(false).version(false) + .usage('Usage: $0 [options]') // auth - .option('auth', { - group: 'Authentication:', - describe: 'Enter authentication mode', - type: 'boolean', - }) + .option('auth', { + group: 'Authentication:', + describe: 'Enter authentication mode', + type: 'boolean', + }) // search - .option('search', { - alias: 'f', - group: 'Search:', - describe: 'Search show ids', - type: 'string', - }) + .option('search', { + alias: 'f', + group: 'Search:', + describe: 'Search show ids', + type: 'string', + }) // select show and eps - .option('s', { - group: 'Downloading:', - describe: 'Sets the show id', - type: 'number', - }) - .option('e', { - group: 'Downloading:', - describe: 'Select episode ids (comma-separated, hyphen-sequence)', - type: 'string', - }) + .option('s', { + group: 'Downloading:', + describe: 'Sets the show id', + type: 'number', + }) + .option('e', { + group: 'Downloading:', + describe: 'Select episode ids (comma-separated, hyphen-sequence)', + type: 'string', + }) // quality - .option('q', { - group: 'Downloading:', - describe: 'Select video layer (0 is max)', - choices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], - default: cfg.videoLayer || 7, - type: 'number', - }) + .option('q', { + group: 'Downloading:', + describe: 'Select video layer (0 is max)', + choices: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + default: cfg.videoLayer || 7, + type: 'number', + }) // alt listing - .option('alt', { - group: 'Downloading:', - describe: 'Alternative episode listing (if available)', - default: cfg.altList || false, - type: 'boolean', - }) + .option('alt', { + group: 'Downloading:', + describe: 'Alternative episode listing (if available)', + default: cfg.altList || false, + type: 'boolean', + }) // switch to subs - .option('dub', { - group: 'Downloading:', - describe: 'Download non-Japanese Dub (English Dub mode by default)', - choices: [ 'enUS', 'esLA', 'ptBR' ], - default: cfg.dub || 'enUS', - type: 'string', - }) - .option('sub', { - group: 'Downloading:', - describe: 'Japanese Dub with subtitles mode (English Dub mode by default)', - default: cfg.subsMode || false, - type: 'boolean', - }) + .option('dub', { + group: 'Downloading:', + describe: 'Download non-Japanese Dub (English Dub mode by default)', + choices: [ 'enUS', 'esLA', 'ptBR' ], + default: cfg.dub || 'enUS', + type: 'string', + }) + .option('sub', { + group: 'Downloading:', + describe: 'Japanese Dub with subtitles mode (English Dub mode by default)', + default: cfg.subsMode || false, + type: 'boolean', + }) // simulcast - .option('simul', { - group: 'Downloading:', - describe: 'Force downloading simulcast ver. instead of uncut ver. (if uncut ver. available)', - default: cfg.forceSimul || false, - type: 'boolean', - }) + .option('simul', { + group: 'Downloading:', + describe: 'Force downloading simulcast ver. instead of uncut ver. (if uncut ver. available)', + default: cfg.forceSimul || false, + type: 'boolean', + }) // server number - .option('x', { - alias: 'server', - group: 'Downloading:', - describe: 'Select server', - choices: [1, 2, 3, 4], - default: cfg.nServer || 1, - type: 'number', - }) + .option('x', { + alias: 'server', + group: 'Downloading:', + describe: 'Select server', + choices: [1, 2, 3, 4], + default: cfg.nServer || 1, + type: 'number', + }) // skip - .option('novids', { - group: 'Downloading:', - alias: 'skipdl', - describe: 'Skip downloading video (for downloading subtitles only)', - type: 'boolean', - }) - .option('nosubs', { - group: 'Downloading:', - describe: 'Skip downloading subtitles for English Dub (if available)', - type: 'boolean', - }) + .option('novids', { + group: 'Downloading:', + alias: 'skipdl', + describe: 'Skip downloading video (for downloading subtitles only)', + type: 'boolean', + }) + .option('nosubs', { + group: 'Downloading:', + describe: 'Skip downloading subtitles for English Dub (if available)', + type: 'boolean', + }) // proxy - .option('proxy', { - group: 'Proxy:', - describe: 'Set http(s)/socks proxy WHATWG url', - default: cfg.proxy || false, - hidden: true, - }) - .option('proxy-auth', { - group: 'Proxy:', - describe: 'Colon-separated username and password for proxy', - default: cfg.proxy_auth || false, - hidden: true, - }) - .option('ssp', { - group: 'Proxy:', - describe: 'Don\'t use proxy for stream and subtitles downloading', - default: cfg.proxy_ssp || false, - hidden: true, - type: 'boolean', - }) + .option('proxy', { + group: 'Proxy:', + describe: 'Set http(s)/socks proxy WHATWG url', + default: cfg.proxy || false, + hidden: true, + }) + .option('proxy-auth', { + group: 'Proxy:', + describe: 'Colon-separated username and password for proxy', + default: cfg.proxy_auth || false, + hidden: true, + }) + .option('ssp', { + group: 'Proxy:', + describe: 'Don\'t use proxy for stream and subtitles downloading', + default: cfg.proxy_ssp || false, + hidden: true, + type: 'boolean', + }) // muxing - .option('skipmux', { - group: 'Muxing:', - describe: 'Skip muxing video and subtitles', - type: 'boolean', - }) - .option('mp4', { - group: 'Muxing:', - describe: 'Mux into mp4', - default: cfg.mp4mux || false, - type: 'boolean' - }) - .option('mks', { - group: 'Muxing:', - describe: 'Add subtitles to mkv/mp4 (if available)', - default: cfg.muxSubs || false, - type: 'boolean' - }) + .option('skipmux', { + group: 'Muxing:', + describe: 'Skip muxing video and subtitles', + type: 'boolean', + }) + .option('mp4', { + group: 'Muxing:', + describe: 'Mux into mp4', + default: cfg.mp4mux || false, + type: 'boolean' + }) + .option('mks', { + group: 'Muxing:', + describe: 'Add subtitles to mkv/mp4 (if available)', + default: cfg.muxSubs || false, + type: 'boolean' + }) // filenaming - .option('a', { - alias: 'grouptag', - group: 'Filename Template:', - describe: 'Release group', - default: cfg.releaseGroup || 'Funimation', - type: 'string' - }) - .option('t', { - alias: 'title', - group: 'Filename Template:', - describe: 'Series title override', - type: 'string' - }) - .option('ep', { - group: 'Filename Template:', - describe: 'Episode number override (ignored in batch mode)', - type: 'string' - }) - .option('suffix', { - group: 'Filename Template:', - describe: 'Filename suffix override (first "SIZEp" will be replaced with actual video size)', - default: cfg.fileSuffix || 'SIZEp', - type: 'string' - }) + .option('a', { + alias: 'grouptag', + group: 'Filename Template:', + describe: 'Release group', + default: cfg.releaseGroup || 'Funimation', + type: 'string' + }) + .option('t', { + alias: 'title', + group: 'Filename Template:', + describe: 'Series title override', + type: 'string' + }) + .option('ep', { + group: 'Filename Template:', + describe: 'Episode number override (ignored in batch mode)', + type: 'string' + }) + .option('suffix', { + group: 'Filename Template:', + describe: 'Filename suffix override (first "SIZEp" will be replaced with actual video size)', + default: cfg.fileSuffix || 'SIZEp', + type: 'string' + }) // util - .option('nocleanup', { - group: 'Utilities:', - describe: 'Move temporary files to trash folder instead of deleting', - default: cfg.noCleanUp || false, - type: 'boolean' - }) - .option('notrashfolder', { - implies: ['nocleanup'], - group: 'Utilities:', - describe: 'Don\'t move temporary files to trash folder (Used with --nocleanup)', - default: cfg.noTrashFolder || false, - type: 'boolean' - }) + .option('nocleanup', { + group: 'Utilities:', + describe: 'Move temporary files to trash folder instead of deleting', + default: cfg.noCleanUp || false, + type: 'boolean' + }) + .option('notrashfolder', { + implies: ['nocleanup'], + group: 'Utilities:', + describe: 'Don\'t move temporary files to trash folder (Used with --nocleanup)', + default: cfg.noTrashFolder || false, + type: 'boolean' + }) // help - .option('help', { - alias: 'h', - group: 'Help:', - describe: 'Show this help', - type: 'boolean' - }) - .option('partsize', { - group: 'Downloading:', - describe: 'The amount of parts that should be downloaded in paralell', - type: 'number', - default: 10 - }) + .option('help', { + alias: 'h', + group: 'Help:', + describe: 'Show this help', + type: 'boolean' + }) + .option('partsize', { + group: 'Downloading:', + describe: 'The amount of parts that should be downloaded in paralell', + type: 'number', + default: 10 + }) // usage - .example([ - ['$0 --search "My Hero"', 'search "My Hero" in title'], - ['$0 -s 124389 -e 1,2,3', 'download episodes 1-3 from show with id 124389'], - ['$0 -s 124389 -e 1-3,2-7,s1-2', 'download episodes 1-7 and "S"-episodes 1-2 from show with id 124389'], - ]) + .example([ + ['$0 --search "My Hero"', 'search "My Hero" in title'], + ['$0 -s 124389 -e 1,2,3', 'download episodes 1-3 from show with id 124389'], + ['$0 -s 124389 -e 1-3,2-7,s1-2', 'download episodes 1-7 and "S"-episodes 1-2 from show with id 124389'], + ]) // -- - .argv; -} + .argv; +}; const showHelp = yargs.showHelp; diff --git a/modules/module.cfg-loader.js b/modules/module.cfg-loader.js index e85241b..3b5d207 100644 --- a/modules/module.cfg-loader.js +++ b/modules/module.cfg-loader.js @@ -4,7 +4,7 @@ const existsFile = fs.existsSync; const loadYamlFile = (file) => { return yaml.parse(fs.readFileSync(file, 'utf8')); -} +}; const loadYamlCfg = (file) => { if(existsFile(`${file}.user.yml`)){ @@ -21,6 +21,6 @@ const loadYamlCfg = (file) => { } } return {}; -} +}; module.exports = loadYamlCfg; diff --git a/modules/module.getdata.js b/modules/module.getdata.js index dde50e4..fa65fef 100644 --- a/modules/module.getdata.js +++ b/modules/module.getdata.js @@ -10,7 +10,7 @@ const getData = async (options) => { } }; if(options.responseType) { - gOptions.responseType = options.responseType + gOptions.responseType = options.responseType; } if(options.baseUrl){ gOptions.prefixUrl = options.baseUrl; @@ -71,6 +71,6 @@ const getData = async (options) => { error, }; } -} +}; module.exports = getData; diff --git a/modules/module.vttconvert.js b/modules/module.vttconvert.js index ae6c437..52aae6b 100644 --- a/modules/module.vttconvert.js +++ b/modules/module.vttconvert.js @@ -141,7 +141,7 @@ function toSubsTime(str, srtFormat) { let hLen = srtFormat ? 2 : 1; x[3] = '0.' + ('' + x[3]).padStart(3, '0'); - sx = x[0]*60*60 + x[1]*60 + x[2] + Number(x[3]) + sx = x[0]*60*60 + x[1]*60 + x[2] + Number(x[3]); sx = sx.toFixed(msLen).split('.'); -- 2.45.2 From 6e5b31edf6406c693827faf9133edf00cdaeecf6 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Tue, 4 May 2021 18:41:21 +0200 Subject: [PATCH 07/51] Removed hls-download The new code to download files can handle both the new and the old version --- funi.js | 56 ++++++++++++---------------------------------------- package.json | 1 - 2 files changed, 13 insertions(+), 44 deletions(-) diff --git a/funi.js b/funi.js index 5a0c12c..47a9cde 100644 --- a/funi.js +++ b/funi.js @@ -16,7 +16,6 @@ const yaml = require('yaml'); const shlp = require('sei-helper'); const { lookpath } = require('lookpath'); const m3u8 = require('m3u8-parsed'); -const streamdl = require('hls-download'); const crypto = require('crypto'); const got = require('got'); @@ -539,41 +538,7 @@ async function downloadStreams(){ let chunkList = m3u8(reqVideo.res.body); let tsFile = path.join(cfg.dir.content, fnOutput); - - if (chunkList.segments[0].uri.match(/streaming_video_(\d+)_(\d+)_(\d+)\.ts/)) { - await downloadFile(tsFile, chunkList); - } else { - let proxyHLS = false; - if (argv.proxy && !argv.ssp) { - try { - proxyHLS = {}; - proxyHLS.url = buildProxyUrl(argv.proxy, argv['proxy-auth']); - } - catch(e){ - console.log(`\n[WARN] Not valid proxy URL${e.input?' ('+e.input+')':''}!`); - console.log('[WARN] Skiping...'); - proxyHLS = false; - } - } - - let streamdlParams = { - fn: tsFile + '.ts', - m3u8json: chunkList, - baseurl: chunkList.baseUrl, - pcount: 10, - proxy: (proxyHLS ? proxyHLS : false) - }; - - let dldata = await new streamdl(streamdlParams).download(); - if(!dldata.ok){ - fs.writeFileSync(`${tsFile}.ts.resume`, JSON.stringify(dldata.parts)); - console.log(`[ERROR] DL Stats: ${JSON.stringify(dldata.parts)}\n`); - dlFailed = true; - } - else if(fs.existsSync(`${tsFile}.ts.resume`) && dldata.ok){ - fs.unlinkSync(`${tsFile}.ts.resume`); - } - } + dlFailed = !await downloadFile(tsFile, chunkList); } else{ console.log('[INFO] Skip video downloading...\n'); @@ -592,7 +557,7 @@ async function downloadStreams(){ let tsFileA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`); - await downloadFile(tsFileA, chunkListA); + let dlFailedA = !await downloadFile(tsFileA, chunkListA); } // add subs @@ -757,7 +722,7 @@ async function downloadFile(filename, chunkList) { let rwts = await shlp.question(`[Q] File «${filename + '.ts'}» already exists! Rewrite? (y/N)`); rwts = rwts || 'N'; if (!['Y', 'y'].includes(rwts[0])) { - return; + return false; } fs.unlinkSync(filename + '.ts'); } @@ -775,7 +740,7 @@ async function downloadFile(filename, chunkList) { let p = await Promise.all(cur); if (p.some(el => el instanceof Error)) { console.log(`[ERROR] An error occured while downloading ${filename}.ts`); - return; + return false; } fs.writeFileSync(`${filename}.ts.resume`, JSON.stringify({ total: chunkList.segments.length, downloaded: i + argv.partsize }, null, 4)); @@ -793,18 +758,23 @@ async function downloadFile(filename, chunkList) { fs.unlinkSync(`${filename}.ts.resume`); console.log(`[INFO] Finished ${filename}.ts`); + return true; } async function downloadPart(chunk, index) { let key = await generateCrypto(chunk, index); + let headers = { + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:70.0) Gecko/20100101 Firefox/70.0' + }; + + if (chunk.byterange) + headers.Range = `bytes=${chunk.byterange.offset}-${chunk.byterange.offset+chunk.byterange.length-1}`; + let res = (await got({ url: chunk.uri, - headers: { - 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:70.0) Gecko/20100101 Firefox/70.0', - 'Range': `bytes=${chunk.byterange.offset}-${chunk.byterange.offset+chunk.byterange.length-1}` - }, + headers, responseType: 'buffer' }).catch(error => console.log(`[ERROR] ${error.name}: ${error.code||error.message}`))); diff --git a/package.json b/package.json index 1044f72..cfa10eb 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,6 @@ "dependencies": { "form-data": "^3.0.0", "got": "^11.7.0", - "hls-download": "^2.5.3", "lookpath": "^1.1.0", "m3u8-parsed": "^1.3.0", "sei-helper": "^3.3.0", -- 2.45.2 From 0fbb2004173c8cef09dd6a280871e02cd157c3c3 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Tue, 4 May 2021 20:18:41 +0200 Subject: [PATCH 08/51] Added functionality to download all episodes as requested in #33 --- funi.js | 7 ++++++- modules/module.app-args.js | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/funi.js b/funi.js index 47a9cde..872134e 100644 --- a/funi.js +++ b/funi.js @@ -214,7 +214,12 @@ async function getShow(){ let showStrId = eps[e].ids.externalShowId; let epStrId = eps[e].ids.externalEpisodeId.replace(new RegExp('^'+showStrId),''); // select - if(epSelList.includes(epStrId.replace(/^(?:([A-Z]+)|)(0+)/,'$1'))){ + if (argv.all) { + fnSlug.push({title:eps[e].item.titleSlug,episode:eps[e].item.episodeSlug}) + epSelEps.push(epStrId) + is_selected = true + } + else if(epSelList.includes(epStrId.replace(/^(?:([A-Z]+)|)(0+)/,'$1'))){ fnSlug.push({title:eps[e].item.titleSlug,episode:eps[e].item.episodeSlug}); epSelEps.push(epStrId); is_selected = true; diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 2143264..7e69dcb 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -33,6 +33,12 @@ const appArgv = (cfg) => { describe: 'Select episode ids (comma-separated, hyphen-sequence)', type: 'string', }) + .option('all', { + group: 'Downloading:', + describe: 'Used to download all episodes from the show', + type: 'boolean', + default: false + }) // quality .option('q', { group: 'Downloading:', -- 2.45.2 From b519ecf4f35008301c4960438fa23421d685a62d Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Tue, 4 May 2021 20:31:27 +0200 Subject: [PATCH 09/51] Error handling --- funi.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/funi.js b/funi.js index 872134e..6dc0c7e 100644 --- a/funi.js +++ b/funi.js @@ -783,11 +783,12 @@ async function downloadPart(chunk, index) { responseType: 'buffer' }).catch(error => console.log(`[ERROR] ${error.name}: ${error.code||error.message}`))); - if (!res.body) { return; } - - let dec = key.update(res.body); - dec = Buffer.concat([dec, key.final()]); - return { content: dec, index: index}; + if (!res.body) { return new Error("Invalid State"); } + try { + let dec = key.update(res.body); + dec = Buffer.concat([dec, key.final()]); + return { content: dec, index: index}; + } catch (e) { return e } } let keys = {}; -- 2.45.2 From 066fd0df640ef9cbe4a727017ff608615484e2e3 Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 4 May 2021 18:52:27 -0400 Subject: [PATCH 10/51] fixed empty token file handling --- funi.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/funi.js b/funi.js index e6b7c7c..5868ff5 100644 --- a/funi.js +++ b/funi.js @@ -42,11 +42,19 @@ let cfg = { // token let token = getYamlCfg(tokenFile); -token = token.token ? token.token : false; +let emptyToken = false; +try{ + token = token.token ? token.token : false; +} +catch(error){ + token = false; + emptyToken = true; +} // info if token not set if(!token){ - console.log('[INFO] Token not set!\n'); + if(emptyToken) console.log('[WARN] token.yml is empty!\n'); + else console.log('[INFO] Token not set!\n'); } // cli -- 2.45.2 From e0bafdd9b4837d4a0858578ce8918bf4366d4e9a Mon Sep 17 00:00:00 2001 From: Jake Date: Tue, 4 May 2021 22:52:30 -0400 Subject: [PATCH 11/51] added handling for empty and non-existant config files --- funi.js | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/funi.js b/funi.js index 5868ff5..d3b8a8a 100644 --- a/funi.js +++ b/funi.js @@ -40,21 +40,42 @@ let cfg = { cli: getYamlCfg(cliCfgFile), }; +// make sure cfg params aren't empty +if (cfg.bin === null){ + cfg.bin = {}; + console.log('[WARN] bin-path.yml is empty or does not exist!\n'); +} +if (!cfg.bin.hasOwnProperty('ffmpeg')) cfg.bin.ffmpeg = './bin/ffmpeg'; +if (!cfg.bin.hasOwnProperty('mkvmerge')) cfg.bin.mkvmerge = './bin/mkvmerge'; + +if (cfg.dir === null){ + cfg.dir = {}; + console.log('[WARN] dir-path.yml is empty or does not exist!\n'); +} +if (!cfg.dir.hasOwnProperty('content')) cfg.dir.content = './videos/'; +if (!cfg.dir.hasOwnProperty('trash')) cfg.dir.trash = "./videos/_trash/"; + +if (cfg.cli === null){ + cfg.cli = {}; + console.log('[WARN] cli-defaults.yml is empty or does not exist!\n'); +} +if (!cfg.cli.hasOwnProperty('releaseGroup')) cfg.cli.releaseGroup = "Funimation"; +if (!cfg.cli.hasOwnProperty('videoLayer')) cfg.cli.videoLayer = 7; +if (!cfg.cli.hasOwnProperty('fileSuffix')) cfg.cli.fileSuffix = "SIZEp"; +if (!cfg.cli.hasOwnProperty('nServer')) cfg.cli.nServer = 1; +if (!cfg.cli.hasOwnProperty('mp4mux')) cfg.cli.mp4mux = false; +if (!cfg.cli.hasOwnProperty('muxSubs')) cfg.cli.muxSubs = false; +if (!cfg.cli.hasOwnProperty('noCleanUp')) cfg.cli.noCleanUp = false; + // token let token = getYamlCfg(tokenFile); -let emptyToken = false; -try{ - token = token.token ? token.token : false; -} -catch(error){ - token = false; - emptyToken = true; -} +if (token === null) token = false; +else if (token.token === null) token = false; +else token = token.token; // info if token not set if(!token){ - if(emptyToken) console.log('[WARN] token.yml is empty!\n'); - else console.log('[INFO] Token not set!\n'); + console.log('[INFO] Token not set!\n'); } // cli -- 2.45.2 From 16ad400669ae087949ebfaa62d8b434dfbab8f52 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Wed, 5 May 2021 11:59:37 +0200 Subject: [PATCH 12/51] Added multiple language support as requested in #63 --- funi.js | 28 ++++++++++++++++++---------- package.json | 1 + 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/funi.js b/funi.js index 6dc0c7e..d21face 100644 --- a/funi.js +++ b/funi.js @@ -18,6 +18,7 @@ const { lookpath } = require('lookpath'); const m3u8 = require('m3u8-parsed'); const crypto = require('crypto'); const got = require('got'); +const iso639 = require('iso-639'); // extra const moduleFolder = path.join(__dirname, '/modules'); @@ -215,9 +216,9 @@ async function getShow(){ let epStrId = eps[e].ids.externalEpisodeId.replace(new RegExp('^'+showStrId),''); // select if (argv.all) { - fnSlug.push({title:eps[e].item.titleSlug,episode:eps[e].item.episodeSlug}) - epSelEps.push(epStrId) - is_selected = true + fnSlug.push({title:eps[e].item.titleSlug,episode:eps[e].item.episodeSlug}); + epSelEps.push(epStrId); + is_selected = true; } else if(epSelList.includes(epStrId.replace(/^(?:([A-Z]+)|)(0+)/,'$1'))){ fnSlug.push({title:eps[e].item.titleSlug,episode:eps[e].item.episodeSlug}); @@ -562,7 +563,7 @@ async function downloadStreams(){ let tsFileA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`); - let dlFailedA = !await downloadFile(tsFileA, chunkListA); + dlFailedA = !await downloadFile(tsFileA, chunkListA); } // add subs @@ -618,7 +619,15 @@ async function downloadStreams(){ return; } } - + + let langCode; + for (let lang in iso639.iso_639_2) { + let langObj = iso639.iso_639_2[lang]; + if (langObj.hasOwnProperty('639-1') && langObj['639-1'] === plAud['language']) { + langCode = langObj['639-2']; + } + } + // usage let usableMKVmerge = true; let usableFFmpeg = true; @@ -640,7 +649,6 @@ async function downloadStreams(){ // ftag argv.ftag = argv.ftag ? argv.ftag : argv.a; argv.ftag = shlp.cleanupFilename(argv.ftag); - // select muxer if(!argv.mp4 && usableMKVmerge){ // mux to mkv @@ -648,7 +656,7 @@ async function downloadStreams(){ mkvmux.push('-o',`${muxTrg}.mkv`); mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data'); mkvmux.push('--track-name',`0:[${argv.ftag}]`); - mkvmux.push('--language',`1:${argv.sub?'jpn':''}`); + mkvmux.push('--language',`1:${langCode}`); if(plAud.uri){ mkvmux.push('--video-tracks','0','--no-audio'); mkvmux.push('--no-subtitles','--no-attachments'); @@ -682,7 +690,7 @@ async function downloadStreams(){ ffmux += addSubs && !argv.mp4 ? '-c:s ass ' : ''; ffmux += addSubs && argv.mp4 ? '-c:s mov_text ' : ''; ffmux += '-metadata encoding_tool="no_variable_data" '; - ffmux += `-metadata:s:v:0 title="[${argv.a}]" -metadata:s:a:0 language=${argv.sub?'jpn':''} `; + ffmux += `-metadata:s:v:0 title="[${argv.a}]" -metadata:s:a:0 language=${langCode} `; ffmux += addSubs ? '-metadata:s:s:0 language=eng ' : ''; ffmux += `"${muxTrg}.${ffext}"`; // mux to mkv @@ -783,12 +791,12 @@ async function downloadPart(chunk, index) { responseType: 'buffer' }).catch(error => console.log(`[ERROR] ${error.name}: ${error.code||error.message}`))); - if (!res.body) { return new Error("Invalid State"); } + if (!res.body) { return new Error('Invalid State'); } try { let dec = key.update(res.body); dec = Buffer.concat([dec, key.final()]); return { content: dec, index: index}; - } catch (e) { return e } + } catch (e) { return e; } } let keys = {}; diff --git a/package.json b/package.json index cfa10eb..6bf246c 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "dependencies": { "form-data": "^3.0.0", "got": "^11.7.0", + "iso-639": "^0.2.2", "lookpath": "^1.1.0", "m3u8-parsed": "^1.3.0", "sei-helper": "^3.3.0", -- 2.45.2 From 5d34c2152e22086feab71fe8765f2ebdb9e433b5 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Wed, 5 May 2021 17:23:32 +0200 Subject: [PATCH 13/51] Fixed mkvmerge in #63 and cli-defaults in #62 --- funi.js | 2 +- modules/module.app-args.js | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/funi.js b/funi.js index d21face..ea1ab15 100644 --- a/funi.js +++ b/funi.js @@ -656,7 +656,7 @@ async function downloadStreams(){ mkvmux.push('-o',`${muxTrg}.mkv`); mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data'); mkvmux.push('--track-name',`0:[${argv.ftag}]`); - mkvmux.push('--language',`1:${langCode}`); + mkvmux.push('--language',`0:${langCode}`); if(plAud.uri){ mkvmux.push('--video-tracks','0','--no-audio'); mkvmux.push('--no-subtitles','--no-attachments'); diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 7e69dcb..1811776 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -37,7 +37,13 @@ const appArgv = (cfg) => { group: 'Downloading:', describe: 'Used to download all episodes from the show', type: 'boolean', - default: false + default: cfg.all || false + }) + .option('partsize', { + group: 'Downloading:', + describe: 'The amount of parts that should be downloaded in paralell', + type: 'number', + default: cfg.partsize || 10 }) // quality .option('q', { @@ -180,12 +186,6 @@ const appArgv = (cfg) => { describe: 'Show this help', type: 'boolean' }) - .option('partsize', { - group: 'Downloading:', - describe: 'The amount of parts that should be downloaded in paralell', - type: 'number', - default: 10 - }) // usage .example([ ['$0 --search "My Hero"', 'search "My Hero" in title'], -- 2.45.2 From f268e616f9e2ebf049631595ace2a198477e1028 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Wed, 5 May 2021 17:55:49 +0200 Subject: [PATCH 14/51] Fixed language issue --- config/bin-path.yml | 2 +- funi.js | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/config/bin-path.yml b/config/bin-path.yml index 8a8e1a4..239b00c 100644 --- a/config/bin-path.yml +++ b/config/bin-path.yml @@ -1,2 +1,2 @@ ffmpeg: "./bin/ffmpeg" -mkvmerge: "./bin/mkvmerge" +mkvmerge: "./bin/mkvtoolnix/mkvmerge.exe" diff --git a/funi.js b/funi.js index ea1ab15..2082d3a 100644 --- a/funi.js +++ b/funi.js @@ -627,6 +627,8 @@ async function downloadStreams(){ langCode = langObj['639-2']; } } + if (!langCode) + langCode = argv.sub ? 'jpn' : 'eng' // usage let usableMKVmerge = true; @@ -645,7 +647,7 @@ async function downloadStreams(){ console.log('[WARN] FFmpeg not found, skip using this...'); usableFFmpeg = false; } - + // ftag argv.ftag = argv.ftag ? argv.ftag : argv.a; argv.ftag = shlp.cleanupFilename(argv.ftag); @@ -656,16 +658,18 @@ async function downloadStreams(){ mkvmux.push('-o',`${muxTrg}.mkv`); mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data'); mkvmux.push('--track-name',`0:[${argv.ftag}]`); - mkvmux.push('--language',`0:${langCode}`); + if(plAud.uri){ mkvmux.push('--video-tracks','0','--no-audio'); mkvmux.push('--no-subtitles','--no-attachments'); mkvmux.push(`${muxTrg}.ts`); + mkvmux.push('--language',`0:${langCode}`); mkvmux.push('--no-video','--audio-tracks','0'); mkvmux.push('--no-subtitles','--no-attachments'); mkvmux.push(`${muxTrgA}.ts`); } else{ + mkvmux.push('--language',`1:${langCode}`); mkvmux.push('--video-tracks','0','--audio-tracks','1'); mkvmux.push('--no-subtitles','--no-attachments'); mkvmux.push(`${muxTrg}.ts`); -- 2.45.2 From c907510e20cc72d495e24500b4f063eedbd168b2 Mon Sep 17 00:00:00 2001 From: KX-Apple <57068530+AnAppleforlife@users.noreply.github.com> Date: Fri, 7 May 2021 17:21:45 +0200 Subject: [PATCH 15/51] Fixed build error --- funi.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/funi.js b/funi.js index 2082d3a..236aba5 100644 --- a/funi.js +++ b/funi.js @@ -21,11 +21,10 @@ const got = require('got'); const iso639 = require('iso-639'); // extra -const moduleFolder = path.join(__dirname, '/modules'); -const appYargs = require(path.join(moduleFolder, 'module.app-args')); -const getYamlCfg = require(path.join(moduleFolder, 'module.cfg-loader')); -const getData = require(path.join(moduleFolder, 'module.getdata.js')); -const vttConvert = require(path.join(moduleFolder, 'module.vttconvert')); +const appYargs = require('./modules/module.app-args'); +const getYamlCfg = require('./modules/module.cfg-loader'); +const getData = require('./modules/module.getdata.js'); +const vttConvert = require('./modules/module.vttconvert'); // new-cfg const cfgFolder = path.join(__dirname, '/config'); -- 2.45.2 From b1bd11235e5687f39c2291aa9a5ad78c77260e43 Mon Sep 17 00:00:00 2001 From: Apple Date: Mon, 10 May 2021 15:29:01 +0200 Subject: [PATCH 16/51] Resolved #71 and #72 --- funi.js | 12 ++++++++---- modules/module.app-args.js | 7 ++++++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/funi.js b/funi.js index 236aba5..ef2d03f 100644 --- a/funi.js +++ b/funi.js @@ -549,7 +549,7 @@ async function downloadStreams(){ console.log('[INFO] Skip video downloading...\n'); } - if (!argv.novids && plAud.uri) { + if (!argv.noaudio && plAud.uri) { // download audio let reqAudio = await getData({ url: plAud.uri, @@ -627,7 +627,7 @@ async function downloadStreams(){ } } if (!langCode) - langCode = argv.sub ? 'jpn' : 'eng' + langCode = argv.sub ? 'jpn' : 'eng'; // usage let usableMKVmerge = true; @@ -646,6 +646,9 @@ async function downloadStreams(){ console.log('[WARN] FFmpeg not found, skip using this...'); usableFFmpeg = false; } + if ( argv.novids ){ + console.log('[INFO] Video not downloaded. Skip muxing video.'); + } // ftag argv.ftag = argv.ftag ? argv.ftag : argv.a; @@ -686,10 +689,11 @@ async function downloadStreams(){ let ffmux = `-i "${muxTrg}.ts" `; if(plAud.uri){ ffmux += `-i "${muxTrgA}.ts" `; + ffmux += '-map 1:a '; } ffmux += addSubs ? `-i "${muxTrg}${subsExt}" ` : ''; - ffmux += '-map 0 -map 1:a -c:v copy -c:a copy '; - ffmux += addSubs ? '-map 1 ' : ''; + ffmux += '-map 0 -c:v copy -c:a copy '; + ffmux += addSubs ? `-map ${plAud.uri ? 2 : 1} ` : ''; ffmux += addSubs && !argv.mp4 ? '-c:s ass ' : ''; ffmux += addSubs && argv.mp4 ? '-c:s mov_text ' : ''; ffmux += '-metadata encoding_tool="no_variable_data" '; diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 1811776..492f4ae 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -91,10 +91,15 @@ const appArgv = (cfg) => { type: 'number', }) // skip + .option('noaudio', { + group: 'Downloading:', + describe: 'Skip downloading audio', + type: 'boolean' + }) .option('novids', { group: 'Downloading:', alias: 'skipdl', - describe: 'Skip downloading video (for downloading subtitles only)', + describe: 'Skip downloading video', type: 'boolean', }) .option('nosubs', { -- 2.45.2 From 1817a3d75a887d51128d9a900c68fd09dc4a92a7 Mon Sep 17 00:00:00 2001 From: Apple Date: Tue, 11 May 2021 22:25:30 +0200 Subject: [PATCH 17/51] Added Chinese Mandarin support and fixed an ffmpeg error @STRATOS99K This should help you download your episodes. About the subtitles, I haven't found a show with like spanish subtitles. If you can provide me with an id I can look into it. --- funi.js | 3 ++- modules/module.app-args.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/funi.js b/funi.js index ef2d03f..1abc1ac 100644 --- a/funi.js +++ b/funi.js @@ -318,6 +318,7 @@ async function getEpisode(fnSlug){ 'enUS': 'English', 'esLA': 'Spanish (Latin Am)', 'ptBR': 'Portuguese (Brazil)', + 'zhMN': 'Chinese (Mandarin, PRC)' }; // select @@ -697,7 +698,7 @@ async function downloadStreams(){ ffmux += addSubs && !argv.mp4 ? '-c:s ass ' : ''; ffmux += addSubs && argv.mp4 ? '-c:s mov_text ' : ''; ffmux += '-metadata encoding_tool="no_variable_data" '; - ffmux += `-metadata:s:v:0 title="[${argv.a}]" -metadata:s:a:0 language=${langCode} `; + ffmux += `-metadata:s:v:0 title="[${argv.a}]" -metadata:s:a:${plAud.uri?1:0} language=${langCode} `; ffmux += addSubs ? '-metadata:s:s:0 language=eng ' : ''; ffmux += `"${muxTrg}.${ffext}"`; // mux to mkv diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 492f4ae..7c290db 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -64,7 +64,7 @@ const appArgv = (cfg) => { .option('dub', { group: 'Downloading:', describe: 'Download non-Japanese Dub (English Dub mode by default)', - choices: [ 'enUS', 'esLA', 'ptBR' ], + choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN' ], default: cfg.dub || 'enUS', type: 'string', }) -- 2.45.2 From 09d774c22987165edf14f329393160d5bab74b9c Mon Sep 17 00:00:00 2001 From: Apple Date: Wed, 12 May 2021 01:44:59 +0200 Subject: [PATCH 18/51] Added subtitle language support So @STRATOS99K that should be the second part of your request. Can you test it on your end? If something is not quite working feel free to open an issue in my fork or comment here. I will fix it tomorrow since its already 1:45 AM here. --- funi.js | 10 +++++++++- modules/module.app-args.js | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/funi.js b/funi.js index 1abc1ac..01b8c3f 100644 --- a/funi.js +++ b/funi.js @@ -387,10 +387,18 @@ function getSubsUrl(m){ if(argv.nosubs && !argv.sub){ return false; } + + let subLangAvailable = m.some(a => a.ext == 'vtt' && a.languages && a.languages[0].code === argv.subLang); + + if (!subLangAvailable) { + console.log(`[WARN] Unable to find subtitle language '${argv.subLang}'. Defaulting to English.`); + argv.subLang = 'en'; + } + for(let i in m){ let fpp = m[i].filePath.split('.'); let fpe = fpp[fpp.length-1]; - if(fpe == 'vtt'){ // dfxp (TTML), srt, vtt + if(fpe == 'vtt' && (( !m[i].languages ) || (m[i].languages[0].code === argv.subLang))){ // dfxp (TTML), srt, vtt return m[i].filePath; } } diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 7c290db..126966e 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -74,6 +74,13 @@ const appArgv = (cfg) => { default: cfg.subsMode || false, type: 'boolean', }) + .option('subLang', { + group: 'Downloading:', + describe: 'Set the subtitle language (English is default and fallback)', + default: cfg.subLang || 'en', + choices: ['en','es','pt'], + type: 'string' + }) // simulcast .option('simul', { group: 'Downloading:', -- 2.45.2 From be9e53da04dfa0dc02cd5ebfd8d15d6225b3ad30 Mon Sep 17 00:00:00 2001 From: Apple Date: Wed, 12 May 2021 14:24:20 +0200 Subject: [PATCH 19/51] Updated Subtitle Downloading @STRATOS99K This should include the things you requested. Once again if you have any issues just bring them up. --- funi.js | 14 +++++++++----- modules/module.vttconvert.js | 8 ++++---- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/funi.js b/funi.js index 01b8c3f..688d1ca 100644 --- a/funi.js +++ b/funi.js @@ -66,7 +66,7 @@ let fnTitle = '', fnSuffix = '', fnOutput = '', tsDlPath = false, - stDlPath = false, + stDlPath = undefined, batchDL = false; // select mode @@ -399,7 +399,10 @@ function getSubsUrl(m){ let fpp = m[i].filePath.split('.'); let fpe = fpp[fpp.length-1]; if(fpe == 'vtt' && (( !m[i].languages ) || (m[i].languages[0].code === argv.subLang))){ // dfxp (TTML), srt, vtt - return m[i].filePath; + return { + path: m[i].filePath, + ext: `.${(m[i].languages ? m[i].languages[0].code : 'en')}` + }; } } return false; @@ -575,7 +578,7 @@ async function downloadStreams(){ } // add subs - let subsUrl = stDlPath; + let subsUrl = stDlPath ? stDlPath.path : false; let subsExt = !argv.mp4 || argv.mp4 && !argv.mks && argv.ass ? '.ass' : '.srt'; let addSubs = argv.mks && subsUrl ? true : false; @@ -589,8 +592,9 @@ async function downloadStreams(){ debug: argv.debug, }); if(subsSrc.ok){ - let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false)); - let assFile = path.join(cfg.dir.content, fnOutput) + subsExt; + let langName = iso639.iso_639_1[argv.subLang].name; + let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), langName ? langName : undefined ); + let assFile = path.join(cfg.dir.content, fnOutput) + stDlPath.ext + subsExt; fs.writeFileSync(assFile, assData); console.log('[INFO] Subtitles downloaded!'); } diff --git a/modules/module.vttconvert.js b/modules/module.vttconvert.js index 52aae6b..9af46f7 100644 --- a/modules/module.vttconvert.js +++ b/modules/module.vttconvert.js @@ -35,10 +35,10 @@ function loadVtt(vttStr) { } // ass specific -function convertToAss(vttStr){ +function convertToAss(vttStr, lang){ let ass = [ '\ufeff[Script Info]', - 'Title: English', + `Title: ${lang}`, 'ScriptType: v4.00+', 'PlayResX: 1280', 'PlayResY: 720', @@ -160,7 +160,7 @@ function padTimeNum(sep, input, pad){ } // export module -module.exports = (vttStr, toSrt) => { +module.exports = (vttStr, toSrt, lang = 'English') => { const convert = toSrt ? convertToSrt : convertToAss; - return convert(vttStr); + return convert(vttStr, lang); }; -- 2.45.2 From e1db81e38dc3c25b01499a5562e66adb0b20a2fc Mon Sep 17 00:00:00 2001 From: Apple Date: Wed, 12 May 2021 14:25:24 +0200 Subject: [PATCH 20/51] Updated version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6bf246c..08822eb 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "funimation-downloader-nx", "short_name": "funi", - "version": "4.8.0-beta.1", + "version": "4.8.0", "description": "Download videos from Funimation via cli.", "keywords": [ "download", -- 2.45.2 From 116a820c32c35acc95dec891486133d154008326 Mon Sep 17 00:00:00 2001 From: Apple Date: Thu, 20 May 2021 19:55:44 +0200 Subject: [PATCH 21/51] Resolved subtitle merging issue --- funi.js | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/funi.js b/funi.js index 688d1ca..a13ed41 100644 --- a/funi.js +++ b/funi.js @@ -581,7 +581,9 @@ async function downloadStreams(){ let subsUrl = stDlPath ? stDlPath.path : false; let subsExt = !argv.mp4 || argv.mp4 && !argv.mks && argv.ass ? '.ass' : '.srt'; let addSubs = argv.mks && subsUrl ? true : false; - + let subFile; + let subTrashFile; + // download subtitles if(subsUrl){ console.log('[INFO] Downloading subtitles...'); @@ -594,8 +596,9 @@ async function downloadStreams(){ if(subsSrc.ok){ let langName = iso639.iso_639_1[argv.subLang].name; let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), langName ? langName : undefined ); - let assFile = path.join(cfg.dir.content, fnOutput) + stDlPath.ext + subsExt; - fs.writeFileSync(assFile, assData); + subFile = path.join(cfg.dir.content, fnOutput) + stDlPath.ext + subsExt; + subTrashFile = path.join(cfg.dir.trash, fnOutput) + stDlPath.ext + subsExt; + fs.writeFileSync(subFile, assData); console.log('[INFO] Subtitles downloaded!'); } else{ @@ -691,7 +694,7 @@ async function downloadStreams(){ } if(addSubs){ mkvmux.push('--language','0:eng'); - mkvmux.push(`${muxTrg}${subsExt}`); + mkvmux.push(`${subFile ? subFile : muxTrg + subsExt}`); } fs.writeFileSync(`${muxTrg}.json`,JSON.stringify(mkvmux,null,' ')); shlp.exec('mkvmerge',`"${mkvmergebinfile}"`,`@"${muxTrg}.json"`); @@ -700,11 +703,9 @@ async function downloadStreams(){ else if(usableFFmpeg){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; let ffmux = `-i "${muxTrg}.ts" `; - if(plAud.uri){ - ffmux += `-i "${muxTrgA}.ts" `; - ffmux += '-map 1:a '; - } - ffmux += addSubs ? `-i "${muxTrg}${subsExt}" ` : ''; + ffmux += plAud.uri ? `-i "${muxTrgA}.ts" ` : ''; + ffmux += addSubs ? `-i "${subFile ? subFile : muxTrg + subsExt}" ` : ''; + ffmux += plAud.uri ? '-map 1:a ' : ''; ffmux += '-map 0 -c:v copy -c:a copy '; ffmux += addSubs ? `-map ${plAud.uri ? 2 : 1} ` : ''; ffmux += addSubs && !argv.mp4 ? '-c:s ass ' : ''; @@ -728,7 +729,7 @@ async function downloadStreams(){ if (plAud.uri) fs.renameSync(muxTrgA+'.ts', tshTrgA + '.ts'); if(subsUrl && addSubs){ - fs.renameSync(muxTrg +subsExt, tshTrg +subsExt); + fs.renameSync(subFile ? subFile : muxTrg+subsExt, subTrashFile ? subTrashFile : tshTrg + subsExt); } } else{ @@ -736,7 +737,7 @@ async function downloadStreams(){ if (plAud.uri) fs.unlinkSync(muxTrgA+'.ts'); if(subsUrl && addSubs){ - fs.unlinkSync(muxTrg +subsExt); + fs.unlinkSync(subFile ? subFile : muxTrg + subsExt); } } console.log('\n[INFO] Done!\n'); -- 2.45.2 From 810d8c703b7cf44d3e350b17ed3f38d7c5731111 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Tue, 25 May 2021 17:31:49 +0200 Subject: [PATCH 22/51] Fixed sub download --- funi.js | 28 +++++++++++++++++++--------- modules/module.app-args.js | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/funi.js b/funi.js index a13ed41..d0fb258 100644 --- a/funi.js +++ b/funi.js @@ -340,7 +340,7 @@ async function getEpisode(fnSlug){ stDlPath = m.subtitles; selected = true; } - console.log(`[#${m.id}] ${dub_type} [${m.version}]`,(selected?'(selected)':'')); + console.log(`[#${m.id}] ${dub_type} [${m.version}]`,(selected?'(selected)':''),(m.subtitles.subLangAvailable?'':'(defaulted to Englisch subtitles)')); } } @@ -388,20 +388,31 @@ function getSubsUrl(m){ return false; } - let subLangAvailable = m.some(a => a.ext == 'vtt' && a.languages && a.languages[0].code === argv.subLang); + let subLang = argv.subLang + + const subType = { + 'enUS': 'English', + 'esLA': 'Spanish (Latin Am)', + 'ptBR': 'Portuguese (Brazil)' + }; + + let subLangAvailable = m.some(a => { + return a.ext == 'vtt' && a.language === subType[subLang] + }); if (!subLangAvailable) { - console.log(`[WARN] Unable to find subtitle language '${argv.subLang}'. Defaulting to English.`); - argv.subLang = 'en'; + subLang = 'enUS'; } - + for(let i in m){ let fpp = m[i].filePath.split('.'); let fpe = fpp[fpp.length-1]; - if(fpe == 'vtt' && (( !m[i].languages ) || (m[i].languages[0].code === argv.subLang))){ // dfxp (TTML), srt, vtt + if(fpe == 'vtt' && m[i].language === subType[subLang]) { return { path: m[i].filePath, - ext: `.${(m[i].languages ? m[i].languages[0].code : 'en')}` + ext: `.${subLang}`, + langName: subType[subLang], + subLangAvailable: subLangAvailable }; } } @@ -594,8 +605,7 @@ async function downloadStreams(){ debug: argv.debug, }); if(subsSrc.ok){ - let langName = iso639.iso_639_1[argv.subLang].name; - let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), langName ? langName : undefined ); + let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), stDlPath.langName ); subFile = path.join(cfg.dir.content, fnOutput) + stDlPath.ext + subsExt; subTrashFile = path.join(cfg.dir.trash, fnOutput) + stDlPath.ext + subsExt; fs.writeFileSync(subFile, assData); diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 126966e..5c4d11b 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -78,7 +78,7 @@ const appArgv = (cfg) => { group: 'Downloading:', describe: 'Set the subtitle language (English is default and fallback)', default: cfg.subLang || 'en', - choices: ['en','es','pt'], + choices: [ 'enUS', 'esLA', 'ptBR' ], type: 'string' }) // simulcast -- 2.45.2 From 23540d5ef00109b7c70b1091bb110f426f4791ac Mon Sep 17 00:00:00 2001 From: Oujiii <34745265+Oujiii@users.noreply.github.com> Date: Tue, 25 May 2021 13:10:52 -0300 Subject: [PATCH 23/51] Fixing typo on line 343 Just changing from Englisch to English, minor typo. --- funi.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funi.js b/funi.js index d0fb258..b603dfd 100644 --- a/funi.js +++ b/funi.js @@ -340,7 +340,7 @@ async function getEpisode(fnSlug){ stDlPath = m.subtitles; selected = true; } - console.log(`[#${m.id}] ${dub_type} [${m.version}]`,(selected?'(selected)':''),(m.subtitles.subLangAvailable?'':'(defaulted to Englisch subtitles)')); + console.log(`[#${m.id}] ${dub_type} [${m.version}]`,(selected?'(selected)':''),(m.subtitles.subLangAvailable?'':'(defaulted to English subtitles)')); } } -- 2.45.2 From b2fb0e7260e19e9e32e11b67f2dc3752474fd8de Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Tue, 25 May 2021 19:07:16 +0200 Subject: [PATCH 24/51] Subtitle font size support --- funi.js | 4 ++-- modules/module.app-args.js | 6 ++++++ modules/module.vttconvert.js | 10 +++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/funi.js b/funi.js index b603dfd..685dc54 100644 --- a/funi.js +++ b/funi.js @@ -395,7 +395,7 @@ function getSubsUrl(m){ 'esLA': 'Spanish (Latin Am)', 'ptBR': 'Portuguese (Brazil)' }; - + let subLangAvailable = m.some(a => { return a.ext == 'vtt' && a.language === subType[subLang] }); @@ -605,7 +605,7 @@ async function downloadStreams(){ debug: argv.debug, }); if(subsSrc.ok){ - let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), stDlPath.langName ); + let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), stDlPath.langName, argv.fontSize); subFile = path.join(cfg.dir.content, fnOutput) + stDlPath.ext + subsExt; subTrashFile = path.join(cfg.dir.trash, fnOutput) + stDlPath.ext + subsExt; fs.writeFileSync(subFile, assData); diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 5c4d11b..a0b4925 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -81,6 +81,12 @@ const appArgv = (cfg) => { choices: [ 'enUS', 'esLA', 'ptBR' ], type: 'string' }) + .option('fontSize', { + group: 'Downloading:', + describe: 'Used to set the fontsize of the subtitles', + default: cfg.fontSize || 55, + type: 'number' + }) // simulcast .option('simul', { group: 'Downloading:', diff --git a/modules/module.vttconvert.js b/modules/module.vttconvert.js index 9af46f7..dcdfba9 100644 --- a/modules/module.vttconvert.js +++ b/modules/module.vttconvert.js @@ -35,7 +35,7 @@ function loadVtt(vttStr) { } // ass specific -function convertToAss(vttStr, lang){ +function convertToAss(vttStr, lang, fontSize){ let ass = [ '\ufeff[Script Info]', `Title: ${lang}`, @@ -49,8 +49,8 @@ function convertToAss(vttStr, lang){ 'Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, ' + 'Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, ' + 'BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding', - 'Style: Main,Noto Sans,55,&H00FFFFFF,&H000000FF,&H00020713,&H00000000,0,0,0,0,100,100,0,0,1,3,0,2,10,10,10,1', - 'Style: MainTop,Noto Sans,55,&H00FFFFFF,&H000000FF,&H00020713,&H00000000,0,0,0,0,100,100,0,0,1,3,0,8,10,10,10,1', + `Style: Main,Noto Sans,${fontSize},&H00FFFFFF,&H000000FF,&H00020713,&H00000000,0,0,0,0,100,100,0,0,1,3,0,2,10,10,10,1`, + `Style: MainTop,Noto Sans,${fontSize},&H00FFFFFF,&H000000FF,&H00020713,&H00000000,0,0,0,0,100,100,0,0,1,3,0,8,10,10,10`, '', '[Events]', 'Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text', @@ -160,7 +160,7 @@ function padTimeNum(sep, input, pad){ } // export module -module.exports = (vttStr, toSrt, lang = 'English') => { +module.exports = (vttStr, toSrt, lang = 'English', fontSize) => { const convert = toSrt ? convertToSrt : convertToAss; - return convert(vttStr, lang); + return convert(vttStr, lang, fontSize); }; -- 2.45.2 From 225d81e9ce9f15bc92ad4e8c1c0855fd78ecd6c2 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Tue, 25 May 2021 20:32:41 +0200 Subject: [PATCH 25/51] Fixed build files --- funi.js | 9 ++++----- modules/module.app-args.js | 2 +- package.json | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/funi.js b/funi.js index 685dc54..a2cacfa 100644 --- a/funi.js +++ b/funi.js @@ -27,11 +27,10 @@ const getData = require('./modules/module.getdata.js'); const vttConvert = require('./modules/module.vttconvert'); // new-cfg -const cfgFolder = path.join(__dirname, '/config'); -const binCfgFile = path.join(cfgFolder, 'bin-path'); -const dirCfgFile = path.join(cfgFolder, 'dir-path'); -const cliCfgFile = path.join(cfgFolder, 'cli-defaults'); -const tokenFile = path.join(cfgFolder, 'token'); +const binCfgFile = path.join('config', 'bin-path'); +const dirCfgFile = path.join('config', 'dir-path'); +const cliCfgFile = path.join('config', 'cli-defaults'); +const tokenFile = path.join('config', 'token'); // params let cfg = { diff --git a/modules/module.app-args.js b/modules/module.app-args.js index a0b4925..6838203 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -77,7 +77,7 @@ const appArgv = (cfg) => { .option('subLang', { group: 'Downloading:', describe: 'Set the subtitle language (English is default and fallback)', - default: cfg.subLang || 'en', + default: cfg.subLang || 'enUS', choices: [ 'enUS', 'esLA', 'ptBR' ], type: 'string' }) diff --git a/package.json b/package.json index 08822eb..a0bbdb7 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "funimation-downloader-nx", "short_name": "funi", - "version": "4.8.0", + "version": "4.8.1", "description": "Download videos from Funimation via cli.", "keywords": [ "download", -- 2.45.2 From 3aa4c414986226fecab85f28ede6b4303f2d2116 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Tue, 1 Jun 2021 15:27:00 +0200 Subject: [PATCH 26/51] Resolved issue where application won't run from outside --- config/bin-path.yml | 2 +- funi.js | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config/bin-path.yml b/config/bin-path.yml index 239b00c..039122d 100644 --- a/config/bin-path.yml +++ b/config/bin-path.yml @@ -1,2 +1,2 @@ -ffmpeg: "./bin/ffmpeg" +ffmpeg: "C:\\Program Files\\ffmpeg\\bin\\ffmpeg.exe" mkvmerge: "./bin/mkvtoolnix/mkvmerge.exe" diff --git a/funi.js b/funi.js index a2cacfa..2dd91a7 100644 --- a/funi.js +++ b/funi.js @@ -27,10 +27,13 @@ const getData = require('./modules/module.getdata.js'); const vttConvert = require('./modules/module.vttconvert'); // new-cfg -const binCfgFile = path.join('config', 'bin-path'); -const dirCfgFile = path.join('config', 'dir-path'); -const cliCfgFile = path.join('config', 'cli-defaults'); -const tokenFile = path.join('config', 'token'); + +const pathToFile = process.pkg ? '' : __dirname; + +const binCfgFile = path.join(pathToFile, 'config', 'bin-path'); +const dirCfgFile = path.join(pathToFile, 'config', 'dir-path'); +const cliCfgFile = path.join(pathToFile, 'config', 'cli-defaults'); +const tokenFile = path.join(pathToFile, 'config', 'token'); // params let cfg = { -- 2.45.2 From e9e7112763bd4027cde108bcc94faf9f6db5931b Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Fri, 11 Jun 2021 21:41:01 +0200 Subject: [PATCH 27/51] Reverted accidental config change --- config/bin-path.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/bin-path.yml b/config/bin-path.yml index 039122d..6131ee9 100644 --- a/config/bin-path.yml +++ b/config/bin-path.yml @@ -1,2 +1,2 @@ -ffmpeg: "C:\\Program Files\\ffmpeg\\bin\\ffmpeg.exe" +ffmpeg: "./bin/ffmpeg/ffmpeg.exe" mkvmerge: "./bin/mkvtoolnix/mkvmerge.exe" -- 2.45.2 From e98fb1588ea2ef4082a10655b81a3a977a36de93 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Tue, 22 Jun 2021 19:48:11 +0200 Subject: [PATCH 28/51] Better file naming Now you can set the name of the output file as coustom as you want --- funi.js | 72 ++++++++++++++++++++++++++++++-------- modules/module.app-args.js | 42 +++++++++++----------- package.json | 2 +- 3 files changed, 79 insertions(+), 37 deletions(-) diff --git a/funi.js b/funi.js index 2dd91a7..f3d6ff7 100644 --- a/funi.js +++ b/funi.js @@ -25,7 +25,6 @@ const appYargs = require('./modules/module.app-args'); const getYamlCfg = require('./modules/module.cfg-loader'); const getData = require('./modules/module.getdata.js'); const vttConvert = require('./modules/module.vttconvert'); - // new-cfg const pathToFile = process.pkg ? '' : __dirname; @@ -63,10 +62,12 @@ else{ } // fn variables -let fnTitle = '', - fnEpNum = '', +let title = '', + showTitle = '', + fnEpNum = 0, fnSuffix = '', fnOutput = '', + season = 0, tsDlPath = false, stDlPath = undefined, batchDL = false; @@ -273,12 +274,13 @@ async function getEpisode(fnSlug){ if(!episodeData.ok){return;} let ep = JSON.parse(episodeData.res.body).items[0], streamId = 0; // build fn - fnTitle = argv.t ? argv.t : ep.parent.title; - ep.number = isNaN(ep.number) ? ep.number : ( parseInt(ep.number, 10) < 10 ? '0' + ep.number : ep.number ); + showTitle = ep.parent.title; + title = ep.title; + season = parseInt(ep.parent.seasonNumber) if(ep.mediaCategory != 'Episode'){ ep.number = ep.number !== '' ? ep.mediaCategory+ep.number : ep.mediaCategory+'#'+ep.id; } - fnEpNum = argv.ep && !batchDL ? ( parseInt(argv.ep, 10) < 10 ? '0' + argv.ep : argv.ep ) : ep.number; + fnEpNum = parseInt(ep.number); // is uncut let uncut = { @@ -496,13 +498,13 @@ async function downloadStreams(){ plStreams[plServer][plLayerId] = plUrlDl; } // set plLayersStr - let plResolution = `${s.attributes.RESOLUTION.height}p`; + let plResolution = s.attributes.RESOLUTION; plLayersRes[plLayerId] = plResolution; let plBandwidth = Math.round(s.attributes.BANDWIDTH/1024); if(plLayerId<10){ plLayerId = plLayerId.toString().padStart(2,' '); } - let qualityStrAdd = `${plLayerId}: ${plResolution} (${plBandwidth}KiB/s)`; + let qualityStrAdd = `${plLayerId}: ${plResolution.width}x${plResolution.height} (${plBandwidth}KiB/s)`; let qualityStrRegx = new RegExp(qualityStrAdd.replace(/(:|\(|\)|\/)/g,'\\$1'),'m'); let qualityStrMatch = !plLayersStr.join('\r\n').match(qualityStrRegx); if(qualityStrMatch){ @@ -537,10 +539,10 @@ async function downloadStreams(){ console.log(`[INFO] Available qualities:\n\t${plLayersStr.join('\n\t')}`); if(videoUrl != ''){ - console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q]}) @ ${plSelectedServer}`); + console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q].width}x${plLayersRes[argv.q].height}) @ ${plSelectedServer}`); console.log('[INFO] Stream URL:',videoUrl); - fnSuffix = argv.suffix.replace('SIZEp',plLayersRes[argv.q]); - fnOutput = shlp.cleanupFilename(`[${argv.a}] ${fnTitle} - ${fnEpNum} [${fnSuffix}]`); + + fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height) console.log(`[INFO] Output filename: ${fnOutput}.ts`); } else if(argv.x > plServerList.length){ @@ -678,16 +680,13 @@ async function downloadStreams(){ console.log('[INFO] Video not downloaded. Skip muxing video.'); } - // ftag - argv.ftag = argv.ftag ? argv.ftag : argv.a; - argv.ftag = shlp.cleanupFilename(argv.ftag); // select muxer if(!argv.mp4 && usableMKVmerge){ // mux to mkv let mkvmux = []; mkvmux.push('-o',`${muxTrg}.mkv`); mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data'); - mkvmux.push('--track-name',`0:[${argv.ftag}]`); + mkvmux.push('--track-name',`0:[Funimation]`); if(plAud.uri){ mkvmux.push('--video-tracks','0','--no-audio'); @@ -894,3 +893,46 @@ function buildProxy(proxyBaseUrl, proxyAuth){ return proxyStr; } + +/** + * @param {string} input + * @param {string} title + * @param {number} episode + * @param {string} showTitle + * @param {number} season + * @param {number} width + * @param {number} height + * @returns {string} + */ +function parseFileName(input, title, episode, showTitle, season, width, height) { + const varRegex = /\${[A-Za-z1-9]+}/g + const vars = input.match(varRegex) + for (let i = 0; i < vars.length; i++) { + const type = vars[i] + switch (type.slice(2, -1).toLowerCase()) { + case "title": + input = input.replace(vars[i], title) + break; + case "episode": + let len = episode.toFixed(0).toString().length + input = input.replace(vars[i], len < argv.numbers ? '0'.repeat(argv.numbers - len) + episode : episode) + break; + case "showtitle": + input = input.replace(vars[i], showTitle) + break; + case "season": + let len1 = season.toFixed(0).toString().length + input = input.replace(vars[i], len1 < argv.numbers ? '0'.repeat(argv.numbers - len1) + season : season) + break; + case "width": + input = input.replace(vars[i], width) + break; + case "height": + input = input.replace(vars[i], height) + break; + default: + break; + } + } + return shlp.cleanupFilename(input) +} \ No newline at end of file diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 6838203..220e97d 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -1,5 +1,14 @@ const yargs = require('yargs'); +const availableFilenameVars = [ + 'title', + 'episode', + 'showTitle', + 'season', + 'width', + 'height' +] + const appArgv = (cfg) => { // init return yargs.parserConfiguration({ @@ -159,29 +168,19 @@ const appArgv = (cfg) => { type: 'boolean' }) // filenaming - .option('a', { - alias: 'grouptag', + .option('fileName', { group: 'Filename Template:', - describe: 'Release group', - default: cfg.releaseGroup || 'Funimation', - type: 'string' + describe: `Set the filename template. Use \${variable_name} to insert variables.\nYou may use ${availableFilenameVars + .map(a => `'${a}'`).join(', ')} as variables.`, + type: 'string', + default: cfg.fileName || '[Funimation] ${showTitle} - ${episode} [${height}p]' }) - .option('t', { - alias: 'title', + .option('numbers', { group: 'Filename Template:', - describe: 'Series title override', - type: 'string' - }) - .option('ep', { - group: 'Filename Template:', - describe: 'Episode number override (ignored in batch mode)', - type: 'string' - }) - .option('suffix', { - group: 'Filename Template:', - describe: 'Filename suffix override (first "SIZEp" will be replaced with actual video size)', - default: cfg.fileSuffix || 'SIZEp', - type: 'string' + describe: `Set how long a number in the title should be at least.\n${[[3, 5, "005"], [2, 1, "01"], [1, 20, "20"]] + .map(val => `Set in config: ${val[0]}; Episode number: ${val[1]}; Output: ${val[2]}`).join('\n')}`, + type: 'number', + default: cfg.numbers || 2 }) // util .option('nocleanup', { @@ -219,5 +218,6 @@ const showHelp = yargs.showHelp; module.exports = { appArgv, - showHelp + showHelp, + availableFilenameVars }; diff --git a/package.json b/package.json index a0bbdb7..c69d494 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "funimation-downloader-nx", "short_name": "funi", - "version": "4.8.1", + "version": "4.8.2", "description": "Download videos from Funimation via cli.", "keywords": [ "download", -- 2.45.2 From 4b76c75c7c54c4ddd7c799bc46a6f532ce6b586c Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Tue, 22 Jun 2021 19:51:09 +0200 Subject: [PATCH 29/51] Updated package.json urls to lead to this fork --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index c69d494..be42410 100644 --- a/package.json +++ b/package.json @@ -12,14 +12,14 @@ "utility", "cli" ], - "author": "AniDL", - "homepage": "https://github.com/anidl/funimation-downloader-nx", + "author": "AniDL/Izu-co", + "homepage": "https://github.com/izu-co/funimation-downloader-nx", "repository": { "type": "git", - "url": "https://github.com/anidl/funimation-downloader-nx.git" + "url": "https://github.com/izu-co/funimation-downloader-nx.git" }, "bugs": { - "url": "https://github.com/anidl/funimation-downloader-nx/issues" + "url": "https://github.com/izu-co/funimation-downloader-nx/issues" }, "license": "MIT", "main": "funi.js", -- 2.45.2 From 6aa231536d5a48bb8984b1b84dbb22cf8c2be31d Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Tue, 22 Jun 2021 19:54:57 +0200 Subject: [PATCH 30/51] Eslint fix --- funi.js | 55 ++++++++++++++++++-------------------- modules/module.app-args.js | 4 +-- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/funi.js b/funi.js index f3d6ff7..805ee6b 100644 --- a/funi.js +++ b/funi.js @@ -65,12 +65,10 @@ else{ let title = '', showTitle = '', fnEpNum = 0, - fnSuffix = '', fnOutput = '', season = 0, tsDlPath = false, - stDlPath = undefined, - batchDL = false; + stDlPath = undefined; // select mode if(argv.auth){ @@ -247,9 +245,6 @@ async function getShow(){ conOut += eps.length-1 == e ? '\n' : ''; console.log(conOut); } - if(fnSlug.length>1){ - batchDL = true; - } if(fnSlug.length<1){ console.log('[INFO] Episodes not selected!\n'); process.exit(); @@ -276,7 +271,7 @@ async function getEpisode(fnSlug){ // build fn showTitle = ep.parent.title; title = ep.title; - season = parseInt(ep.parent.seasonNumber) + season = parseInt(ep.parent.seasonNumber); if(ep.mediaCategory != 'Episode'){ ep.number = ep.number !== '' ? ep.mediaCategory+ep.number : ep.mediaCategory+'#'+ep.id; } @@ -392,7 +387,7 @@ function getSubsUrl(m){ return false; } - let subLang = argv.subLang + let subLang = argv.subLang; const subType = { 'enUS': 'English', @@ -401,7 +396,7 @@ function getSubsUrl(m){ }; let subLangAvailable = m.some(a => { - return a.ext == 'vtt' && a.language === subType[subLang] + return a.ext == 'vtt' && a.language === subType[subLang]; }); if (!subLangAvailable) { @@ -542,7 +537,7 @@ async function downloadStreams(){ console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q].width}x${plLayersRes[argv.q].height}) @ ${plSelectedServer}`); console.log('[INFO] Stream URL:',videoUrl); - fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height) + fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height); console.log(`[INFO] Output filename: ${fnOutput}.ts`); } else if(argv.x > plServerList.length){ @@ -686,7 +681,7 @@ async function downloadStreams(){ let mkvmux = []; mkvmux.push('-o',`${muxTrg}.mkv`); mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data'); - mkvmux.push('--track-name',`0:[Funimation]`); + mkvmux.push('--track-name','0:[Funimation]'); if(plAud.uri){ mkvmux.push('--video-tracks','0','--no-audio'); @@ -905,34 +900,36 @@ function buildProxy(proxyBaseUrl, proxyAuth){ * @returns {string} */ function parseFileName(input, title, episode, showTitle, season, width, height) { - const varRegex = /\${[A-Za-z1-9]+}/g - const vars = input.match(varRegex) + const varRegex = /\${[A-Za-z1-9]+}/g; + const vars = input.match(varRegex); for (let i = 0; i < vars.length; i++) { - const type = vars[i] + const type = vars[i]; switch (type.slice(2, -1).toLowerCase()) { - case "title": - input = input.replace(vars[i], title) + case 'title': + input = input.replace(vars[i], title); break; - case "episode": - let len = episode.toFixed(0).toString().length - input = input.replace(vars[i], len < argv.numbers ? '0'.repeat(argv.numbers - len) + episode : episode) + case 'episode': { + let len = episode.toFixed(0).toString().length; + input = input.replace(vars[i], len < argv.numbers ? '0'.repeat(argv.numbers - len) + episode : episode); break; - case "showtitle": - input = input.replace(vars[i], showTitle) + } + case 'showtitle': + input = input.replace(vars[i], showTitle); break; - case "season": - let len1 = season.toFixed(0).toString().length - input = input.replace(vars[i], len1 < argv.numbers ? '0'.repeat(argv.numbers - len1) + season : season) + case 'season': { + let len = season.toFixed(0).toString().length; + input = input.replace(vars[i], len < argv.numbers ? '0'.repeat(argv.numbers - len) + season : season); break; - case "width": - input = input.replace(vars[i], width) + } + case 'width': + input = input.replace(vars[i], width); break; - case "height": - input = input.replace(vars[i], height) + case 'height': + input = input.replace(vars[i], height); break; default: break; } } - return shlp.cleanupFilename(input) + return shlp.cleanupFilename(input); } \ No newline at end of file diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 220e97d..f3531d4 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -7,7 +7,7 @@ const availableFilenameVars = [ 'season', 'width', 'height' -] +]; const appArgv = (cfg) => { // init @@ -177,7 +177,7 @@ const appArgv = (cfg) => { }) .option('numbers', { group: 'Filename Template:', - describe: `Set how long a number in the title should be at least.\n${[[3, 5, "005"], [2, 1, "01"], [1, 20, "20"]] + describe: `Set how long a number in the title should be at least.\n${[[3, 5, '005'], [2, 1, '01'], [1, 20, '20']] .map(val => `Set in config: ${val[0]}; Episode number: ${val[1]}; Output: ${val[2]}`).join('\n')}`, type: 'number', default: cfg.numbers || 2 -- 2.45.2 From 7cc717dd93360602741788c5b925d8f3b915afa4 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Tue, 22 Jun 2021 19:55:51 +0200 Subject: [PATCH 31/51] Update cli-defaults.yml --- config/cli-defaults.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/cli-defaults.yml b/config/cli-defaults.yml index 08a6b4b..e3a894c 100644 --- a/config/cli-defaults.yml +++ b/config/cli-defaults.yml @@ -1,6 +1,5 @@ releaseGroup: Funimation videoLayer: 7 -fileSuffix: SIZEp nServer: 1 mp4mux: false muxSubs: false -- 2.45.2 From 987f64ee0e80b8e84ba2e0ac47e29710f6059598 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Mon, 28 Jun 2021 19:32:01 +0200 Subject: [PATCH 32/51] Bug Fixes - Fixed an issue where the app coudnt be used outside the directory where the files live --- funi.js | 20 ++++++++++++-------- package.json | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/funi.js b/funi.js index 805ee6b..ab2ce6d 100644 --- a/funi.js +++ b/funi.js @@ -26,14 +26,11 @@ const getYamlCfg = require('./modules/module.cfg-loader'); const getData = require('./modules/module.getdata.js'); const vttConvert = require('./modules/module.vttconvert'); // new-cfg - -const pathToFile = process.pkg ? '' : __dirname; - -const binCfgFile = path.join(pathToFile, 'config', 'bin-path'); -const dirCfgFile = path.join(pathToFile, 'config', 'dir-path'); -const cliCfgFile = path.join(pathToFile, 'config', 'cli-defaults'); -const tokenFile = path.join(pathToFile, 'config', 'token'); - +const workingDir = process.pkg ? path.dirname(process.execPath) : __dirname; +const binCfgFile = path.join(workingDir, 'config', 'bin-path'); +const dirCfgFile = path.join(workingDir, 'config', 'dir-path'); +const cliCfgFile = path.join(workingDir, 'config', 'cli-defaults'); +const tokenFile = path.join(workingDir, 'config', 'token'); // params let cfg = { bin: getYamlCfg(binCfgFile), @@ -41,6 +38,13 @@ let cfg = { cli: getYamlCfg(cliCfgFile), }; +/* Normalise paths for use outside the current directory */ +for (let key of Object.keys(cfg.dir)) { + if (!path.isAbsolute(cfg.dir[key])) { + cfg.dir[key] = path.join(workingDir, cfg.dir[key]) + } +} + // token let token = getYamlCfg(tokenFile); token = token.token ? token.token : false; diff --git a/package.json b/package.json index be42410..9ab9f31 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "funimation-downloader-nx", "short_name": "funi", - "version": "4.8.2", + "version": "4.8.3", "description": "Download videos from Funimation via cli.", "keywords": [ "download", -- 2.45.2 From 6e19808585182b398e463bca27c4442603ce0155 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Mon, 28 Jun 2021 20:00:02 +0200 Subject: [PATCH 33/51] Resolve bin paths --- funi.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/funi.js b/funi.js index ab2ce6d..232abc1 100644 --- a/funi.js +++ b/funi.js @@ -45,6 +45,12 @@ for (let key of Object.keys(cfg.dir)) { } } +for (let key of Object.keys(cfg.bin)) { + if (!path.isAbsolute(cfg.bin[key])) { + cfg.bin[key] = path.join(workingDir, cfg.bin[key]) + } +} + // token let token = getYamlCfg(tokenFile); token = token.token ? token.token : false; -- 2.45.2 From 366501dce3a355ff7b800bee03249bc0781b3fd0 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Thu, 1 Jul 2021 18:55:56 +0200 Subject: [PATCH 34/51] Update argv to handle array input --- modules/module.app-args.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/modules/module.app-args.js b/modules/module.app-args.js index f3531d4..5055204 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -11,8 +11,9 @@ const availableFilenameVars = [ const appArgv = (cfg) => { // init - return yargs.parserConfiguration({ - 'duplicate-arguments-array': false, + const argv = yargs.parserConfiguration({ + 'duplicate-arguments-array': true, + "camel-case-expansion": false }) // main .wrap(Math.min(120)) // yargs.terminalWidth() @@ -212,6 +213,15 @@ const appArgv = (cfg) => { // -- .argv; + + // Resolve unwanted arrays + for (let key in argv) { + if (argv[key] instanceof Array && !(key === "subLang" || key === "dub")) { + argv[key] = argv[key].pop() + } + } + + return argv; }; const showHelp = yargs.showHelp; -- 2.45.2 From c194a9c6b798781461d9c81b6af1c354dda385de Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Thu, 1 Jul 2021 20:01:45 +0200 Subject: [PATCH 35/51] Convert Input to array --- modules/module.app-args.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 5055204..d706175 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -76,7 +76,7 @@ const appArgv = (cfg) => { describe: 'Download non-Japanese Dub (English Dub mode by default)', choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN' ], default: cfg.dub || 'enUS', - type: 'string', + type: 'array', }) .option('sub', { group: 'Downloading:', @@ -89,7 +89,7 @@ const appArgv = (cfg) => { describe: 'Set the subtitle language (English is default and fallback)', default: cfg.subLang || 'enUS', choices: [ 'enUS', 'esLA', 'ptBR' ], - type: 'string' + type: 'array' }) .option('fontSize', { group: 'Downloading:', @@ -129,6 +129,7 @@ const appArgv = (cfg) => { group: 'Downloading:', describe: 'Skip downloading subtitles for English Dub (if available)', type: 'boolean', + default: false }) // proxy .option('proxy', { @@ -220,7 +221,6 @@ const appArgv = (cfg) => { argv[key] = argv[key].pop() } } - return argv; }; -- 2.45.2 From ef28f4e62488cfb1c98ada0c6c4723c7a2145e4f Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Fri, 2 Jul 2021 17:56:24 +0200 Subject: [PATCH 36/51] Ffmpeg working with multiple subtitle files Please note that MkvMerge will follow in the next commit --- config/bin-path.yml | 2 +- funi.js | 137 ++++++++++++++++++++++++-------------------- modules/merger.js | 52 +++++++++++++++++ 3 files changed, 128 insertions(+), 63 deletions(-) create mode 100644 modules/merger.js diff --git a/config/bin-path.yml b/config/bin-path.yml index 6131ee9..039122d 100644 --- a/config/bin-path.yml +++ b/config/bin-path.yml @@ -1,2 +1,2 @@ -ffmpeg: "./bin/ffmpeg/ffmpeg.exe" +ffmpeg: "C:\\Program Files\\ffmpeg\\bin\\ffmpeg.exe" mkvmerge: "./bin/mkvtoolnix/mkvmerge.exe" diff --git a/funi.js b/funi.js index 232abc1..cb389bc 100644 --- a/funi.js +++ b/funi.js @@ -62,6 +62,14 @@ if(!token){ // cli const argv = appYargs.appArgv(cfg.cli); +module.exports = { + argv, + cfg +} + +// Import merger after argv has been exported + +const merger = require('./modules/merger') // check page if(!isNaN(parseInt(argv.p, 10)) && parseInt(argv.p, 10) > 0){ @@ -349,7 +357,9 @@ async function getEpisode(fnSlug){ stDlPath = m.subtitles; selected = true; } - console.log(`[#${m.id}] ${dub_type} [${m.version}]`,(selected?'(selected)':''),(m.subtitles.subLangAvailable?'':'(defaulted to English subtitles)')); + console.log(`[#${m.id}] ${dub_type} [${m.version}]${(selected?' (selected)':'')}${ + stDlPath && selected ? ` (using ${stDlPath.map(a => `'${a.langName}'`).join(', ')} for subtitles)` : '' + }`); } } @@ -397,7 +407,7 @@ function getSubsUrl(m){ return false; } - let subLang = argv.subLang; + let subLangs = argv.subLang; const subType = { 'enUS': 'English', @@ -405,27 +415,30 @@ function getSubsUrl(m){ 'ptBR': 'Portuguese (Brazil)' }; - let subLangAvailable = m.some(a => { - return a.ext == 'vtt' && a.language === subType[subLang]; - }); + let subLangAvailable = m.some(a => subLangs.some(subLang => a.ext == 'vtt' && a.language === subType[subLang])); if (!subLangAvailable) { - subLang = 'enUS'; + subLangs = [ 'enUS' ]; } + let found = [] + for(let i in m){ let fpp = m[i].filePath.split('.'); let fpe = fpp[fpp.length-1]; - if(fpe == 'vtt' && m[i].language === subType[subLang]) { - return { - path: m[i].filePath, - ext: `.${subLang}`, - langName: subType[subLang], - subLangAvailable: subLangAvailable - }; + for (let lang of subLangs) { + if(fpe == 'vtt' && m[i].language === subType[lang]) { + found.push({ + path: m[i].filePath, + ext: `.${lang}`, + langName: subType[lang], + language: m[i]?.languages[0]?.code ?? lang.slice(0, 2) + }); + } } } - return false; + + return found; } async function downloadStreams(){ @@ -598,35 +611,36 @@ async function downloadStreams(){ } // add subs - let subsUrl = stDlPath ? stDlPath.path : false; let subsExt = !argv.mp4 || argv.mp4 && !argv.mks && argv.ass ? '.ass' : '.srt'; - let addSubs = argv.mks && subsUrl ? true : false; - let subFile; - let subTrashFile; + let addSubs = argv.mks && tsDlPath ? true : false; // download subtitles - if(subsUrl){ + if(stDlPath){ console.log('[INFO] Downloading subtitles...'); - console.log(subsUrl); - let subsSrc = await getData({ - url: subsUrl, - useProxy: true, - debug: argv.debug, - }); - if(subsSrc.ok){ - let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), stDlPath.langName, argv.fontSize); - subFile = path.join(cfg.dir.content, fnOutput) + stDlPath.ext + subsExt; - subTrashFile = path.join(cfg.dir.trash, fnOutput) + stDlPath.ext + subsExt; - fs.writeFileSync(subFile, assData); + for (let subObject of stDlPath) { + console.log(subObject); + let subsSrc = await getData({ + url: subObject.path, + useProxy: true, + debug: argv.debug, + }); + if(subsSrc.ok){ + let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.langName, argv.fontSize); + subObject.file = path.join(cfg.dir.content, fnOutput) + subObject.ext + subsExt; + subObject.trashFile = path.join(cfg.dir.trash, fnOutput) + subObject.ext + subsExt; + fs.writeFileSync(subObject.file, assData); + } + else{ + console.log('[ERROR] Failed to download subtitles!'); + addSubs = false; + break; + } + } + if (addSubs) console.log('[INFO] Subtitles downloaded!'); - } - else{ - console.log('[ERROR] Failed to download subtitles!'); - addSubs = false; - } } - if(dlFailed || dlFailedA){ + if(false && (dlFailed || dlFailedA)){ console.log('\n[INFO] TS file not fully downloaded, skip muxing video...\n'); return; } @@ -671,7 +685,7 @@ async function downloadStreams(){ // check exec path let mkvmergebinfile = await lookpath(path.join(cfg.bin.mkvmerge)); let ffmpegbinfile = await lookpath(path.join(cfg.bin.ffmpeg)); - + // check exec if( !argv.mp4 && !mkvmergebinfile ){ console.log('[WARN] MKVMerge not found, skip using this...'); @@ -686,57 +700,53 @@ async function downloadStreams(){ } // select muxer + usableMKVmerge = false + /* TODO Remake mkvmerge */ if(!argv.mp4 && usableMKVmerge){ + /* // mux to mkv let mkvmux = []; mkvmux.push('-o',`${muxTrg}.mkv`); mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data'); mkvmux.push('--track-name','0:[Funimation]'); - if(plAud.uri){ mkvmux.push('--video-tracks','0','--no-audio'); - mkvmux.push('--no-subtitles','--no-attachments'); mkvmux.push(`${muxTrg}.ts`); mkvmux.push('--language',`0:${langCode}`); mkvmux.push('--no-video','--audio-tracks','0'); - mkvmux.push('--no-subtitles','--no-attachments'); mkvmux.push(`${muxTrgA}.ts`); } else{ mkvmux.push('--language',`1:${langCode}`); mkvmux.push('--video-tracks','0','--audio-tracks','1'); - mkvmux.push('--no-subtitles','--no-attachments'); mkvmux.push(`${muxTrg}.ts`); } if(addSubs){ - mkvmux.push('--language','0:eng'); - mkvmux.push(`${subFile ? subFile : muxTrg + subsExt}`); + for (let index in stDlPath) { + subObj = stDlPath[index] + mkvmux.push('--language',`${parseInt(index) + 2}:${getLanguageCode(subObj.language)}`); + mkvmux.push(`${subObj.file ? subObj.file : muxTrg + subsExt}`); + } + } else { + mkvmux.push('--no-subtitles') + mkvmux.push('--no-attachments'); } fs.writeFileSync(`${muxTrg}.json`,JSON.stringify(mkvmux,null,' ')); shlp.exec('mkvmerge',`"${mkvmergebinfile}"`,`@"${muxTrg}.json"`); - fs.unlinkSync(`${muxTrg}.json`); + // fs.unlinkSync(`${muxTrg}.json`); + */ } else if(usableFFmpeg){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; - let ffmux = `-i "${muxTrg}.ts" `; - ffmux += plAud.uri ? `-i "${muxTrgA}.ts" ` : ''; - ffmux += addSubs ? `-i "${subFile ? subFile : muxTrg + subsExt}" ` : ''; - ffmux += plAud.uri ? '-map 1:a ' : ''; - ffmux += '-map 0 -c:v copy -c:a copy '; - ffmux += addSubs ? `-map ${plAud.uri ? 2 : 1} ` : ''; - ffmux += addSubs && !argv.mp4 ? '-c:s ass ' : ''; - ffmux += addSubs && argv.mp4 ? '-c:s mov_text ' : ''; - ffmux += '-metadata encoding_tool="no_variable_data" '; - ffmux += `-metadata:s:v:0 title="[${argv.a}]" -metadata:s:a:${plAud.uri?1:0} language=${langCode} `; - ffmux += addSubs ? '-metadata:s:s:0 language=eng ' : ''; - ffmux += `"${muxTrg}.${ffext}"`; - // mux to mkv - shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,ffmux); + let command = merger.buildCommandFFmpeg(`${muxTrg}.ts`, plAud, stDlPath, `${muxTrg}.${ffext}`) + shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,command) } else{ console.log('\n[INFO] Done!\n'); return; } + if (argv.nocleanup) + return; if(argv.notrashfolder && argv.nocleanup){ // don't move or delete temp files } @@ -744,16 +754,19 @@ async function downloadStreams(){ fs.renameSync(muxTrg+'.ts', tshTrg + '.ts'); if (plAud.uri) fs.renameSync(muxTrgA+'.ts', tshTrgA + '.ts'); - if(subsUrl && addSubs){ - fs.renameSync(subFile ? subFile : muxTrg+subsExt, subTrashFile ? subTrashFile : tshTrg + subsExt); + if(addSubs){ + for (let subObj of stDlPath) { + fs.renameSync(subObj.file ? subObj.file : muxTrg+subsExt, subObj.trashFile ? subObj.trashFile : tshTrg + subsExt); + } } } else{ fs.unlinkSync(muxTrg+'.ts'); if (plAud.uri) fs.unlinkSync(muxTrgA+'.ts'); - if(subsUrl && addSubs){ - fs.unlinkSync(subFile ? subFile : muxTrg + subsExt); + if(addSubs){ + for (let subObj of stDlPath) + fs.unlinkSync(subObj.file ? subObj.file : muxTrg + subsExt); } } console.log('\n[INFO] Done!\n'); diff --git a/modules/merger.js b/modules/merger.js new file mode 100644 index 0000000..3f4dd5c --- /dev/null +++ b/modules/merger.js @@ -0,0 +1,52 @@ +const iso639 = require('iso-639'); +const argv = require('../funi').argv + +/** + * @param {string} videoFile + * @param {object} audioFile + * @param {Array} subtitles + * @returns {string} + */ +const buildCommandFFmpeg = (videoFile, audioSettings, subtitles, output) => { + let args = [] + args.push(`-i "${videoFile}"`) + + if (audioSettings.uri) + args.push(`-i "${audioSettings.uri}"`) + for (let index in subtitles) { + let sub = subtitles[index] + args.push(`-i "${sub.file}"`) + } + + args.push('-map 0') + if (audioSettings.uri) + args.push( `-map 1`) + + args.push(...subtitles.map((_, index) => `-map ${index + (audioSettings.uri ? 2 : 1)}`)) + args.push( + '-metadata:s:v:0 title="[Funimation]"', + `-metadata:s:a:0 language=${getLanguageCode(audioSettings.language, argv.sub ? 'jpn' : 'eng')}`, + `-c:v copy`, + `-c:a copy`, + `-c:s mov_text`, + `-c:s ass` + ) + args.push(...subtitles.map((sub, index) => `-metadata:s:${index + 2} language=${getLanguageCode(sub.language)}`)) + args.push(`"${output}"`) + return args.join(" ") +} + +const getLanguageCode = (from, _default = 'eng') => { + for (let lang in iso639.iso_639_2) { + let langObj = iso639.iso_639_2[lang]; + if (langObj.hasOwnProperty('639-1') && langObj['639-1'] === from) { + return langObj['639-2']; + } + } + return _default +} + +module.exports = { + buildCommandFFmpeg, + getLanguageCode +} \ No newline at end of file -- 2.45.2 From 9ea16caffce8e6970305493f22e2f2db8ed6c117 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Fri, 2 Jul 2021 20:04:52 +0200 Subject: [PATCH 37/51] Mergin files with mkvmerge --- funi.js | 42 +++++------------------------------ modules/merger.js | 56 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 37 deletions(-) diff --git a/funi.js b/funi.js index cb389bc..7b93f8f 100644 --- a/funi.js +++ b/funi.js @@ -618,7 +618,6 @@ async function downloadStreams(){ if(stDlPath){ console.log('[INFO] Downloading subtitles...'); for (let subObject of stDlPath) { - console.log(subObject); let subsSrc = await getData({ url: subObject.path, useProxy: true, @@ -640,6 +639,7 @@ async function downloadStreams(){ console.log('[INFO] Subtitles downloaded!'); } + /* TODO Remove */ if(false && (dlFailed || dlFailedA)){ console.log('\n[INFO] TS file not fully downloaded, skip muxing video...\n'); return; @@ -699,42 +699,11 @@ async function downloadStreams(){ console.log('[INFO] Video not downloaded. Skip muxing video.'); } - // select muxer - usableMKVmerge = false - /* TODO Remake mkvmerge */ if(!argv.mp4 && usableMKVmerge){ - /* - // mux to mkv - let mkvmux = []; - mkvmux.push('-o',`${muxTrg}.mkv`); - mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data'); - mkvmux.push('--track-name','0:[Funimation]'); - if(plAud.uri){ - mkvmux.push('--video-tracks','0','--no-audio'); - mkvmux.push(`${muxTrg}.ts`); - mkvmux.push('--language',`0:${langCode}`); - mkvmux.push('--no-video','--audio-tracks','0'); - mkvmux.push(`${muxTrgA}.ts`); - } - else{ - mkvmux.push('--language',`1:${langCode}`); - mkvmux.push('--video-tracks','0','--audio-tracks','1'); - mkvmux.push(`${muxTrg}.ts`); - } - if(addSubs){ - for (let index in stDlPath) { - subObj = stDlPath[index] - mkvmux.push('--language',`${parseInt(index) + 2}:${getLanguageCode(subObj.language)}`); - mkvmux.push(`${subObj.file ? subObj.file : muxTrg + subsExt}`); - } - } else { - mkvmux.push('--no-subtitles') - mkvmux.push('--no-attachments'); - } - fs.writeFileSync(`${muxTrg}.json`,JSON.stringify(mkvmux,null,' ')); - shlp.exec('mkvmerge',`"${mkvmergebinfile}"`,`@"${muxTrg}.json"`); - // fs.unlinkSync(`${muxTrg}.json`); - */ + let ffext = !argv.mp4 ? 'mkv' : 'mp4'; + let command = merger.buildCommandMkvMerge(`${muxTrg}.ts`, plAud, stDlPath, `${muxTrg}.${ffext}`) + console.log(command) + shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command) } else if(usableFFmpeg){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; @@ -745,6 +714,7 @@ async function downloadStreams(){ console.log('\n[INFO] Done!\n'); return; } + /* TODO Remove */ if (argv.nocleanup) return; if(argv.notrashfolder && argv.nocleanup){ diff --git a/modules/merger.js b/modules/merger.js index 3f4dd5c..3d0fbf0 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -36,6 +36,59 @@ const buildCommandFFmpeg = (videoFile, audioSettings, subtitles, output) => { return args.join(" ") } +/** + * @param {string} videoFile + * @param {object} audioFile + * @param {Array} subtitles + * @returns {string} + */ +const buildCommandMkvMerge = (videoFile, audioSettings, subtitles, output) => { + let args = [] + args.push(`-o "${output}"`) + args.push( + '--no-date', + '--disable-track-statistics-tags', + '--engage no_variable_data', + '--track-name 0:[Funimation]' + ) + + if (audioSettings.uri) { + args.push( + '--video-tracks 0', + '--no-audio' + ); + args.push(`"${videoFile}"`); + args.push(`--language 0:${getLanguageCode(audioSettings.language, argv.sub ? 'jpn' : 'eng')}`); + args.push( + '--no-video', + '--audio-tracks 0' + ); + args.push(`"${audioSettings.uri}"`); + } else{ + args.push(`--language 1:${argv.sub ? 'jpn' : 'eng'}`); + args.push( + '--video-tracks 0', + '--audio-tracks 1' + ); + args.push(`"${videoFile}"`); + } + + if(subtitles.length > 0){ + for (let index in subtitles) { + subObj = subtitles[index] + args.push('--language',`${/*parseInt(index) + (audioSettings.uri ? 2 : 1)*/0}:${getLanguageCode(subObj.language)}`); + args.push(`"${subObj.file}"`); + } + } else { + args.push( + '--no-subtitles', + '--no-attachments' + ) + } + + return args.join(" ") +} + const getLanguageCode = (from, _default = 'eng') => { for (let lang in iso639.iso_639_2) { let langObj = iso639.iso_639_2[lang]; @@ -48,5 +101,6 @@ const getLanguageCode = (from, _default = 'eng') => { module.exports = { buildCommandFFmpeg, - getLanguageCode + getLanguageCode, + buildCommandMkvMerge } \ No newline at end of file -- 2.45.2 From 6edf34a768e6c88db4caadba92755b465aa94b8c Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Fri, 2 Jul 2021 20:09:31 +0200 Subject: [PATCH 38/51] Eslint style and fixes --- funi.js | 33 ++++++++-------------- modules/merger.js | 58 +++++++++++++++++++------------------- modules/module.app-args.js | 6 ++-- 3 files changed, 43 insertions(+), 54 deletions(-) diff --git a/funi.js b/funi.js index 7b93f8f..f05e5ea 100644 --- a/funi.js +++ b/funi.js @@ -18,7 +18,6 @@ const { lookpath } = require('lookpath'); const m3u8 = require('m3u8-parsed'); const crypto = require('crypto'); const got = require('got'); -const iso639 = require('iso-639'); // extra const appYargs = require('./modules/module.app-args'); @@ -41,13 +40,13 @@ let cfg = { /* Normalise paths for use outside the current directory */ for (let key of Object.keys(cfg.dir)) { if (!path.isAbsolute(cfg.dir[key])) { - cfg.dir[key] = path.join(workingDir, cfg.dir[key]) + cfg.dir[key] = path.join(workingDir, cfg.dir[key]); } } for (let key of Object.keys(cfg.bin)) { if (!path.isAbsolute(cfg.bin[key])) { - cfg.bin[key] = path.join(workingDir, cfg.bin[key]) + cfg.bin[key] = path.join(workingDir, cfg.bin[key]); } } @@ -65,11 +64,11 @@ const argv = appYargs.appArgv(cfg.cli); module.exports = { argv, cfg -} +}; // Import merger after argv has been exported -const merger = require('./modules/merger') +const merger = require('./modules/merger'); // check page if(!isNaN(parseInt(argv.p, 10)) && parseInt(argv.p, 10) > 0){ @@ -421,7 +420,7 @@ function getSubsUrl(m){ subLangs = [ 'enUS' ]; } - let found = [] + let found = []; for(let i in m){ let fpp = m[i].filePath.split('.'); @@ -432,7 +431,7 @@ function getSubsUrl(m){ path: m[i].filePath, ext: `.${lang}`, langName: subType[lang], - language: m[i]?.languages[0]?.code ?? lang.slice(0, 2) + language: m[i].languages[0].code ?? lang.slice(0, 2) }); } } @@ -668,16 +667,6 @@ async function downloadStreams(){ } } - let langCode; - for (let lang in iso639.iso_639_2) { - let langObj = iso639.iso_639_2[lang]; - if (langObj.hasOwnProperty('639-1') && langObj['639-1'] === plAud['language']) { - langCode = langObj['639-2']; - } - } - if (!langCode) - langCode = argv.sub ? 'jpn' : 'eng'; - // usage let usableMKVmerge = true; let usableFFmpeg = true; @@ -701,14 +690,14 @@ async function downloadStreams(){ if(!argv.mp4 && usableMKVmerge){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; - let command = merger.buildCommandMkvMerge(`${muxTrg}.ts`, plAud, stDlPath, `${muxTrg}.${ffext}`) - console.log(command) - shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command) + let command = merger.buildCommandMkvMerge(`${muxTrg}.ts`, plAud, stDlPath, `${muxTrg}.${ffext}`); + console.log(command); + shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command); } else if(usableFFmpeg){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; - let command = merger.buildCommandFFmpeg(`${muxTrg}.ts`, plAud, stDlPath, `${muxTrg}.${ffext}`) - shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,command) + let command = merger.buildCommandFFmpeg(`${muxTrg}.ts`, plAud, stDlPath, `${muxTrg}.${ffext}`); + shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,command); } else{ console.log('\n[INFO] Done!\n'); diff --git a/modules/merger.js b/modules/merger.js index 3d0fbf0..36f5487 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -1,5 +1,5 @@ const iso639 = require('iso-639'); -const argv = require('../funi').argv +const argv = require('../funi').argv; /** * @param {string} videoFile @@ -8,33 +8,33 @@ const argv = require('../funi').argv * @returns {string} */ const buildCommandFFmpeg = (videoFile, audioSettings, subtitles, output) => { - let args = [] - args.push(`-i "${videoFile}"`) + let args = []; + args.push(`-i "${videoFile}"`); if (audioSettings.uri) - args.push(`-i "${audioSettings.uri}"`) + args.push(`-i "${audioSettings.uri}"`); for (let index in subtitles) { - let sub = subtitles[index] - args.push(`-i "${sub.file}"`) + let sub = subtitles[index]; + args.push(`-i "${sub.file}"`); } - args.push('-map 0') + args.push('-map 0'); if (audioSettings.uri) - args.push( `-map 1`) + args.push( '-map 1'); - args.push(...subtitles.map((_, index) => `-map ${index + (audioSettings.uri ? 2 : 1)}`)) + args.push(...subtitles.map((_, index) => `-map ${index + (audioSettings.uri ? 2 : 1)}`)); args.push( '-metadata:s:v:0 title="[Funimation]"', `-metadata:s:a:0 language=${getLanguageCode(audioSettings.language, argv.sub ? 'jpn' : 'eng')}`, - `-c:v copy`, - `-c:a copy`, - `-c:s mov_text`, - `-c:s ass` - ) - args.push(...subtitles.map((sub, index) => `-metadata:s:${index + 2} language=${getLanguageCode(sub.language)}`)) - args.push(`"${output}"`) - return args.join(" ") -} + '-c:v copy', + '-c:a copy', + '-c:s mov_text', + '-c:s ass' + ); + args.push(...subtitles.map((sub, index) => `-metadata:s:${index + 2} language=${getLanguageCode(sub.language)}`)); + args.push(`"${output}"`); + return args.join(' '); +}; /** * @param {string} videoFile @@ -43,14 +43,14 @@ const buildCommandFFmpeg = (videoFile, audioSettings, subtitles, output) => { * @returns {string} */ const buildCommandMkvMerge = (videoFile, audioSettings, subtitles, output) => { - let args = [] - args.push(`-o "${output}"`) + let args = []; + args.push(`-o "${output}"`); args.push( '--no-date', '--disable-track-statistics-tags', '--engage no_variable_data', '--track-name 0:[Funimation]' - ) + ); if (audioSettings.uri) { args.push( @@ -75,7 +75,7 @@ const buildCommandMkvMerge = (videoFile, audioSettings, subtitles, output) => { if(subtitles.length > 0){ for (let index in subtitles) { - subObj = subtitles[index] + let subObj = subtitles[index]; args.push('--language',`${/*parseInt(index) + (audioSettings.uri ? 2 : 1)*/0}:${getLanguageCode(subObj.language)}`); args.push(`"${subObj.file}"`); } @@ -83,24 +83,24 @@ const buildCommandMkvMerge = (videoFile, audioSettings, subtitles, output) => { args.push( '--no-subtitles', '--no-attachments' - ) + ); } - return args.join(" ") -} + return args.join(' '); +}; const getLanguageCode = (from, _default = 'eng') => { for (let lang in iso639.iso_639_2) { let langObj = iso639.iso_639_2[lang]; - if (langObj.hasOwnProperty('639-1') && langObj['639-1'] === from) { + if (Object.prototype.hasOwnProperty.call(langObj, '639-1') && langObj['639-1'] === from) { return langObj['639-2']; } } - return _default -} + return _default; +}; module.exports = { buildCommandFFmpeg, getLanguageCode, buildCommandMkvMerge -} \ No newline at end of file +}; \ No newline at end of file diff --git a/modules/module.app-args.js b/modules/module.app-args.js index d706175..8b7f141 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -13,7 +13,7 @@ const appArgv = (cfg) => { // init const argv = yargs.parserConfiguration({ 'duplicate-arguments-array': true, - "camel-case-expansion": false + 'camel-case-expansion': false }) // main .wrap(Math.min(120)) // yargs.terminalWidth() @@ -217,8 +217,8 @@ const appArgv = (cfg) => { // Resolve unwanted arrays for (let key in argv) { - if (argv[key] instanceof Array && !(key === "subLang" || key === "dub")) { - argv[key] = argv[key].pop() + if (argv[key] instanceof Array && !(key === 'subLang' || key === 'dub')) { + argv[key] = argv[key].pop(); } } return argv; -- 2.45.2 From 5a95739e3307bc104b6ec3a65b97423a6af1c7bc Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Fri, 2 Jul 2021 22:39:38 +0200 Subject: [PATCH 39/51] FFmpeg with multiple language files --- config/dir-path.yml | 1 - docs/README.md | 2 +- funi.js | 501 +++++++++++++++++++------------------ modules/build.js | 1 - modules/merger.js | 59 +++-- modules/module.app-args.js | 23 +- videos/_trash/.gitkeep | 0 7 files changed, 307 insertions(+), 280 deletions(-) delete mode 100644 videos/_trash/.gitkeep diff --git a/config/dir-path.yml b/config/dir-path.yml index f9c4877..43961c8 100644 --- a/config/dir-path.yml +++ b/config/dir-path.yml @@ -1,2 +1 @@ content: ./videos/ -trash: ./videos/_trash/ diff --git a/docs/README.md b/docs/README.md index 9b5e748..a7b1edd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -70,7 +70,7 @@ After installing NodeJS with NPM go to directory with `package.json` file and ty ### Utility -* `--nocleanup` move unnecessary files to trash folder after completion instead of deleting +* `--nocleanup` don't delete the input files after the muxing * `-h`, `--help` show all options ## Filename Template diff --git a/funi.js b/funi.js index f05e5ea..3dfeebe 100644 --- a/funi.js +++ b/funi.js @@ -84,8 +84,8 @@ let title = '', fnEpNum = 0, fnOutput = '', season = 0, - tsDlPath = false, - stDlPath = undefined; + tsDlPath = [], + stDlPath = []; // select mode if(argv.auth){ @@ -284,7 +284,7 @@ async function getEpisode(fnSlug){ debug: argv.debug, }); if(!episodeData.ok){return;} - let ep = JSON.parse(episodeData.res.body).items[0], streamId = 0; + let ep = JSON.parse(episodeData.res.body).items[0], streamIds = []; // build fn showTitle = ep.parent.title; title = ep.title; @@ -334,7 +334,8 @@ async function getEpisode(fnSlug){ 'enUS': 'English', 'esLA': 'Spanish (Latin Am)', 'ptBR': 'Portuguese (Brazil)', - 'zhMN': 'Chinese (Mandarin, PRC)' + 'zhMN': 'Chinese (Mandarin, PRC)', + 'jpJP': 'Japanese' }; // select @@ -343,55 +344,72 @@ async function getEpisode(fnSlug){ let selected = false; if(m.id > 0 && m.type == 'Non-Encrypted'){ let dub_type = m.language; + let localSubs = [] let selUncut = !argv.simul && uncut[dub_type] && m.version.match(/uncut/i) ? true : (!uncut[dub_type] || argv.simul && m.version.match(/simulcast/i) ? true : false); - if(dub_type == 'Japanese' && argv.sub && selUncut){ - streamId = m.id; - stDlPath = m.subtitles; - selected = true; - } - else if(dub_type == dubType[argv.dub] && !argv.sub && selUncut){ - streamId = m.id; - stDlPath = m.subtitles; - selected = true; + for (let curDub of argv.dub) { + if(dub_type == dubType[curDub] && selUncut){ + streamIds.push({ + id: m.id, + lang: merger.getLanguageCode(dubType[curDub], curDub.slice(0, -2)) + }); + stDlPath.push(...m.subtitles); + localSubs = m.subtitles + selected = true; + } } console.log(`[#${m.id}] ${dub_type} [${m.version}]${(selected?' (selected)':'')}${ - stDlPath && selected ? ` (using ${stDlPath.map(a => `'${a.langName}'`).join(', ')} for subtitles)` : '' + localSubs && localSubs.length > 0 && selected ? ` (using ${localSubs.map(a => `'${a.langName}'`).join(', ')} for subtitles)` : '' }`); } } + + let already = [] + stDlPath = stDlPath.filter(a => { + if (already.includes(a.language)) { + return false; + } else { + already.push(a.language) + return true; + } + }) - if(streamId<1){ + if(streamIds.length <1){ console.log('[ERROR] Track not selected\n'); return; } else{ - let streamData = await getData({ - baseUrl: api_host, - url: `/source/catalog/video/${streamId}/signed`, - token: token, - dinstid: 'uuid', - useToken: true, - useProxy: true, - debug: argv.debug, - }); - if(!streamData.ok){return;} - streamData = JSON.parse(streamData.res.body); - tsDlPath = false; - if(streamData.errors){ - console.log('[ERROR] Error #%s: %s\n',streamData.errors[0].code,streamData.errors[0].detail); - return; - } - else{ - for(let u in streamData.items){ - if(streamData.items[u].videoType == 'm3u8'){ - tsDlPath = streamData.items[u].src; - break; + tsDlPath = []; + for (let streamId of streamIds) { + let streamData = await getData({ + baseUrl: api_host, + url: `/source/catalog/video/${streamId.id}/signed`, + token: token, + dinstid: 'uuid', + useToken: true, + useProxy: true, + debug: argv.debug, + }); + if(!streamData.ok){return;} + streamData = JSON.parse(streamData.res.body); + if(streamData.errors){ + console.log('[ERROR] Error #%s: %s\n',streamData.errors[0].code,streamData.errors[0].detail); + return; + } + else{ + for(let u in streamData.items){ + if(streamData.items[u].videoType == 'm3u8'){ + tsDlPath.push({ + path: streamData.items[u].src, + lang: streamId.lang + }); + break; + } } } } - if(!tsDlPath){ + if(tsDlPath.length < 1){ console.log('[ERROR] Unknown error\n'); return; } @@ -443,178 +461,213 @@ function getSubsUrl(m){ async function downloadStreams(){ // req playlist - let plQualityReq = await getData({ - url: tsDlPath, - useProxy: (argv.ssp ? false : true), - debug: argv.debug, - }); - if(!plQualityReq.ok){return;} - - let plQualityLinkList = m3u8(plQualityReq.res.body); - - let mainServersList = [ - 'vmfst-api.prd.funimationsvc.com', - 'd132fumi6di1wa.cloudfront.net', - 'funiprod.akamaized.net', - ]; - - let plServerList = [], - plStreams = {}, - plLayersStr = [], - plLayersRes = {}, - plMaxLayer = 1, - plNewIds = 1, - plAud = { uri: '' }; - - // new uris - let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/; - if(plQualityLinkList.playlists[0].uri.match(vplReg)){ - if(plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128']){ - let audioData = plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128'], - audioEl = Object.keys(audioData); - audioData = audioData[audioEl[0]]; - plAud = { ...audioData, ...{ langStr: audioEl[0] } }; - } - plQualityLinkList.playlists.sort((a, b) => { - let av = parseInt(a.uri.match(vplReg)[3]); - let bv = parseInt(b.uri.match(vplReg)[3]); - if(av > bv){ - return 1; - } - if (av < bv) { - return -1; - } - return 0; - }); - } - - for(let s of plQualityLinkList.playlists){ - if(s.uri.match(/_Layer(\d+)\.m3u8/) || s.uri.match(vplReg)){ - // set layer and max layer - let plLayerId = 0; - if(s.uri.match(/_Layer(\d+)\.m3u8/)){ - plLayerId = parseInt(s.uri.match(/_Layer(\d+)\.m3u8/)[1]); - } - else{ - plLayerId = plNewIds, plNewIds++; - } - plMaxLayer = plMaxLayer < plLayerId ? plLayerId : plMaxLayer; - // set urls and servers - let plUrlDl = s.uri; - let plServer = new URL(plUrlDl).host; - if(!plServerList.includes(plServer)){ - plServerList.push(plServer); - } - if(!Object.keys(plStreams).includes(plServer)){ - plStreams[plServer] = {}; - } - if(plStreams[plServer][plLayerId] && plStreams[plServer][plLayerId] != plUrlDl){ - console.log(`[WARN] Non duplicate url for ${plServer} detected, please report to developer!`); - } - else{ - plStreams[plServer][plLayerId] = plUrlDl; - } - // set plLayersStr - let plResolution = s.attributes.RESOLUTION; - plLayersRes[plLayerId] = plResolution; - let plBandwidth = Math.round(s.attributes.BANDWIDTH/1024); - if(plLayerId<10){ - plLayerId = plLayerId.toString().padStart(2,' '); - } - let qualityStrAdd = `${plLayerId}: ${plResolution.width}x${plResolution.height} (${plBandwidth}KiB/s)`; - let qualityStrRegx = new RegExp(qualityStrAdd.replace(/(:|\(|\)|\/)/g,'\\$1'),'m'); - let qualityStrMatch = !plLayersStr.join('\r\n').match(qualityStrRegx); - if(qualityStrMatch){ - plLayersStr.push(qualityStrAdd); - } - } - else { - console.log(s.uri); - } - } - - for(let s of mainServersList){ - if(plServerList.includes(s)){ - plServerList.splice(plServerList.indexOf(s), 1); - plServerList.unshift(s); - break; - } - } - - if(typeof argv.q == 'object' && argv.q.length > 1){ - argv.q = argv.q[argv.q.length-1]; - } - - argv.q = argv.q < 1 || argv.q > plMaxLayer ? plMaxLayer : argv.q; - - let plSelectedServer = plServerList[argv.x-1]; - let plSelectedList = plStreams[plSelectedServer]; - let videoUrl = argv.x < plServerList.length+1 && plSelectedList[argv.q] ? plSelectedList[argv.q] : ''; - - plLayersStr.sort(); - console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`); - console.log(`[INFO] Available qualities:\n\t${plLayersStr.join('\n\t')}`); - - if(videoUrl != ''){ - console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q].width}x${plLayersRes[argv.q].height}) @ ${plSelectedServer}`); - console.log('[INFO] Stream URL:',videoUrl); - fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height); - console.log(`[INFO] Output filename: ${fnOutput}.ts`); - } - else if(argv.x > plServerList.length){ - console.log('[ERROR] Server not selected!\n'); - return; - } - else{ - console.log('[ERROR] Layer not selected!\n'); - return; - } - - let dlFailed = false; - let dlFailedA = false; - - - video: if (!argv.novids) { - // download video - let reqVideo = await getData({ - url: videoUrl, + let purvideo = [] + let puraudio = [] + let audioAndVideo = [] + let outName; + for (let streamPath of tsDlPath) { + let plQualityReq = await getData({ + url: streamPath.path, useProxy: (argv.ssp ? false : true), debug: argv.debug, }); - if (!reqVideo.ok) { break video; } + if(!plQualityReq.ok){return;} - let chunkList = m3u8(reqVideo.res.body); + let plQualityLinkList = m3u8(plQualityReq.res.body); - let tsFile = path.join(cfg.dir.content, fnOutput); - dlFailed = !await downloadFile(tsFile, chunkList); - } - else{ - console.log('[INFO] Skip video downloading...\n'); + let mainServersList = [ + 'vmfst-api.prd.funimationsvc.com', + 'd132fumi6di1wa.cloudfront.net', + 'funiprod.akamaized.net', + ]; + + let plServerList = [], + plStreams = {}, + plLayersStr = [], + plLayersRes = {}, + plMaxLayer = 1, + plNewIds = 1, + plAud = { uri: '' }; + + // new uris + let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/; + if(plQualityLinkList.playlists[0].uri.match(vplReg)){ + if(plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128']){ + let audioData = plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128'], + audioEl = Object.keys(audioData); + audioData = audioData[audioEl[0]]; + plAud = { ...audioData, ...{ langStr: audioEl[0] } }; + } + plQualityLinkList.playlists.sort((a, b) => { + let av = parseInt(a.uri.match(vplReg)[3]); + let bv = parseInt(b.uri.match(vplReg)[3]); + if(av > bv){ + return 1; + } + if (av < bv) { + return -1; + } + return 0; + }); + } + + for(let s of plQualityLinkList.playlists){ + if(s.uri.match(/_Layer(\d+)\.m3u8/) || s.uri.match(vplReg)){ + // set layer and max layer + let plLayerId = 0; + if(s.uri.match(/_Layer(\d+)\.m3u8/)){ + plLayerId = parseInt(s.uri.match(/_Layer(\d+)\.m3u8/)[1]); + } + else{ + plLayerId = plNewIds, plNewIds++; + } + plMaxLayer = plMaxLayer < plLayerId ? plLayerId : plMaxLayer; + // set urls and servers + let plUrlDl = s.uri; + let plServer = new URL(plUrlDl).host; + if(!plServerList.includes(plServer)){ + plServerList.push(plServer); + } + if(!Object.keys(plStreams).includes(plServer)){ + plStreams[plServer] = {}; + } + if(plStreams[plServer][plLayerId] && plStreams[plServer][plLayerId] != plUrlDl){ + console.log(`[WARN] Non duplicate url for ${plServer} detected, please report to developer!`); + } + else{ + plStreams[plServer][plLayerId] = plUrlDl; + } + // set plLayersStr + let plResolution = s.attributes.RESOLUTION; + plLayersRes[plLayerId] = plResolution; + let plBandwidth = Math.round(s.attributes.BANDWIDTH/1024); + if(plLayerId<10){ + plLayerId = plLayerId.toString().padStart(2,' '); + } + let qualityStrAdd = `${plLayerId}: ${plResolution.width}x${plResolution.height} (${plBandwidth}KiB/s)`; + let qualityStrRegx = new RegExp(qualityStrAdd.replace(/(:|\(|\)|\/)/g,'\\$1'),'m'); + let qualityStrMatch = !plLayersStr.join('\r\n').match(qualityStrRegx); + if(qualityStrMatch){ + plLayersStr.push(qualityStrAdd); + } + } + else { + console.log(s.uri); + } + } + + for(let s of mainServersList){ + if(plServerList.includes(s)){ + plServerList.splice(plServerList.indexOf(s), 1); + plServerList.unshift(s); + break; + } + } + + if(typeof argv.q == 'object' && argv.q.length > 1){ + argv.q = argv.q[argv.q.length-1]; + } + + argv.q = argv.q < 1 || argv.q > plMaxLayer ? plMaxLayer : argv.q; + + let plSelectedServer = plServerList[argv.x-1]; + let plSelectedList = plStreams[plSelectedServer]; + let videoUrl = argv.x < plServerList.length+1 && plSelectedList[argv.q] ? plSelectedList[argv.q] : ''; + + plLayersStr.sort(); + console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`); + console.log(`[INFO] Available qualities:\n\t${plLayersStr.join('\n\t')}`); + + if(videoUrl != ''){ + console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q].width}x${plLayersRes[argv.q].height}) @ ${plSelectedServer}`); + console.log('[INFO] Stream URL:',videoUrl); + + fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height); + outName = fnOutput; + console.log(`[INFO] Output filename: ${fnOutput}.ts`); + } + else if(argv.x > plServerList.length){ + console.log('[ERROR] Server not selected!\n'); + return; + } + else{ + console.log('[ERROR] Layer not selected!\n'); + return; + } + + let dlFailed = false; + let dlFailedA = false; + + video: if (!argv.novids) { + if (plAud.uri && (purvideo.length > 1 || audioAndVideo.length > 1)) { + console.log("break 1") + break video; + } else if (!plAud.uri && (audioAndVideo.some(a => a.lang === streamPath.lang) || puraudio.some(a => a.lang === streamPath.lang))) { + console.log("break 2") + break video; + } + // download video + let reqVideo = await getData({ + url: videoUrl, + useProxy: (argv.ssp ? false : true), + debug: argv.debug, + }); + if (!reqVideo.ok) { break video; } + + let chunkList = m3u8(reqVideo.res.body); + + let tsFile = path.join(cfg.dir.content, `${fnOutput}.video${(plAud.uri ? '' : '.' + streamPath.lang )}`); + dlFailed = !await downloadFile(tsFile, chunkList); + if (!dlFailed) { + if (plAud.uri) { + purvideo.push({ + path: `${tsFile}.ts`, + lang: plAud.language + }) + } else { + audioAndVideo.push({ + path: `${tsFile}.ts`, + lang: streamPath.lang + }) + } + } + } + else{ + console.log('[INFO] Skip video downloading...\n'); + } + if (!argv.noaudio && plAud.uri) { + // download audio + if (audioAndVideo.some(a => a.lang === plAud.language) || puraudio.some(a => a.lang === plAud.language)) + return; + let reqAudio = await getData({ + url: plAud.uri, + useProxy: (argv.ssp ? false : true), + debug: argv.debug, + }); + if (!reqAudio.ok) { return; } + + let chunkListA = m3u8(reqAudio.res.body); + + let tsFileA = path.join(cfg.dir.content, fnOutput + `.audio.${plAud.language}`); + + dlFailedA = !await downloadFile(tsFileA, chunkListA); + if (!dlFailedA) + puraudio.push({ + path: `${tsFileA}.ts`, + lang: plAud.language + }) + + } } - if (!argv.noaudio && plAud.uri) { - // download audio - let reqAudio = await getData({ - url: plAud.uri, - useProxy: (argv.ssp ? false : true), - debug: argv.debug, - }); - if (!reqAudio.ok) { return; } - - let chunkListA = m3u8(reqAudio.res.body); - - let tsFileA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`); - - dlFailedA = !await downloadFile(tsFileA, chunkListA); - } // add subs let subsExt = !argv.mp4 || argv.mp4 && !argv.mks && argv.ass ? '.ass' : '.srt'; let addSubs = argv.mks && tsDlPath ? true : false; // download subtitles - if(stDlPath){ + if(stDlPath.length > 0){ console.log('[INFO] Downloading subtitles...'); for (let subObject of stDlPath) { let subsSrc = await getData({ @@ -625,7 +678,6 @@ async function downloadStreams(){ if(subsSrc.ok){ let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.langName, argv.fontSize); subObject.file = path.join(cfg.dir.content, fnOutput) + subObject.ext + subsExt; - subObject.trashFile = path.join(cfg.dir.trash, fnOutput) + subObject.ext + subsExt; fs.writeFileSync(subObject.file, assData); } else{ @@ -638,37 +690,19 @@ async function downloadStreams(){ console.log('[INFO] Subtitles downloaded!'); } - /* TODO Remove */ - if(false && (dlFailed || dlFailedA)){ - console.log('\n[INFO] TS file not fully downloaded, skip muxing video...\n'); + if((puraudio.length < 1 && audioAndVideo.length < 1) || (purvideo.length < 1 && audioAndVideo.length < 1)){ + console.log('\n[INFO] Unable to locate a video AND audio file\n'); return; } if(argv.skipmux){ + console.log("[INFO] Skipping muxing...") return; } - - let muxTrg = path.join(cfg.dir.content, fnOutput); - let muxTrgA = ''; - let tshTrg = path.join(cfg.dir.trash, fnOutput); - let tshTrgA = ''; - - if(!fs.existsSync(`${muxTrg}.ts`) || !fs.statSync(`${muxTrg}.ts`).isFile()){ - console.log('\n[INFO] TS file not found, skip muxing video...\n'); - return; - } - - if(plAud.uri){ - muxTrgA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`); - tshTrgA = path.join(cfg.dir.trash, fnOutput + `.${plAud.language}`); - if(!fs.existsSync(`${muxTrgA}.ts`) || !fs.statSync(`${muxTrgA}.ts`).isFile()){ - console.log('\n[INFO] TS file not found, skip muxing video...\n'); - return; - } - } // usage - let usableMKVmerge = true; + /* TODO MkvMerge */ + let usableMKVmerge = false; let usableFFmpeg = true; // check exec path @@ -690,44 +724,23 @@ async function downloadStreams(){ if(!argv.mp4 && usableMKVmerge){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; - let command = merger.buildCommandMkvMerge(`${muxTrg}.ts`, plAud, stDlPath, `${muxTrg}.${ffext}`); + let command = merger.buildCommandMkvMerge(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`); console.log(command); shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command); } else if(usableFFmpeg){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; - let command = merger.buildCommandFFmpeg(`${muxTrg}.ts`, plAud, stDlPath, `${muxTrg}.${ffext}`); + let command = merger.buildCommandFFmpeg(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`); shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,command); } else{ console.log('\n[INFO] Done!\n'); return; } - /* TODO Remove */ if (argv.nocleanup) return; - if(argv.notrashfolder && argv.nocleanup){ - // don't move or delete temp files - } - else if(argv.nocleanup){ - fs.renameSync(muxTrg+'.ts', tshTrg + '.ts'); - if (plAud.uri) - fs.renameSync(muxTrgA+'.ts', tshTrgA + '.ts'); - if(addSubs){ - for (let subObj of stDlPath) { - fs.renameSync(subObj.file ? subObj.file : muxTrg+subsExt, subObj.trashFile ? subObj.trashFile : tshTrg + subsExt); - } - } - } - else{ - fs.unlinkSync(muxTrg+'.ts'); - if (plAud.uri) - fs.unlinkSync(muxTrgA+'.ts'); - if(addSubs){ - for (let subObj of stDlPath) - fs.unlinkSync(subObj.file ? subObj.file : muxTrg + subsExt); - } - } + + audioAndVideo.concat(puraudio).concat(purvideo).forEach(a => fs.unlinkSync(a.path)) console.log('\n[INFO] Done!\n'); } diff --git a/modules/build.js b/modules/build.js index 3a1113c..5670c2e 100644 --- a/modules/build.js +++ b/modules/build.js @@ -44,7 +44,6 @@ const nodeVer = 'node14-'; fs.mkdirSync(`${buildDir}/bin`); fs.mkdirSync(`${buildDir}/config`); fs.mkdirSync(`${buildDir}/videos`); - fs.mkdirSync(`${buildDir}/videos/_trash`); fs.copySync('./bin/', `${buildDir}/bin/`); fs.copySync('./config/bin-path.yml', `${buildDir}/config/bin-path.yml`); fs.copySync('./config/cli-defaults.yml', `${buildDir}/config/cli-defaults.yml`); diff --git a/modules/merger.js b/modules/merger.js index 36f5487..6d2b7b7 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -2,30 +2,59 @@ const iso639 = require('iso-639'); const argv = require('../funi').argv; /** - * @param {string} videoFile - * @param {object} audioFile + * @param {Array} videoAndAudio + * @param {Array} onlyVid + * @param {Array} onlyAuido * @param {Array} subtitles + * @param {string} output * @returns {string} */ -const buildCommandFFmpeg = (videoFile, audioSettings, subtitles, output) => { +const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAuido, subtitles, output) => { let args = []; - args.push(`-i "${videoFile}"`); + let metaData = []; + + let index = 0; + let hasVideo = false; + for (let vid of videoAndAudio) { + args.push(`-i "${vid.path}"`) + if (!hasVideo) { + metaData.push(`-map ${index}`) + metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`) + metaData.push(`-metadata:s:v:${index} title="[Funimation]"`) + hasVideo = true + } else { + metaData.push(`-map ${index}:a`) + metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`) + } + index++; + } + + for (let vid of onlyVid) { + if (!hasVideo) { + args.push(`-i "${vid.path}"`) + metaData.push(`-map ${index}`) + metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`) + metaData.push(`-metadata:s:v:${index} title="[Funimation]"`) + hasVideo = true + index++; + } + } + + for (let aud of onlyAuido) { + args.push(`-i "${aud.path}"`) + metaData.push(`-map ${index}`) + metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(aud.lang, aud.lang)}`) + index++; + } - if (audioSettings.uri) - args.push(`-i "${audioSettings.uri}"`); for (let index in subtitles) { let sub = subtitles[index]; args.push(`-i "${sub.file}"`); } - args.push('-map 0'); - if (audioSettings.uri) - args.push( '-map 1'); - - args.push(...subtitles.map((_, index) => `-map ${index + (audioSettings.uri ? 2 : 1)}`)); + args.push(...subtitles.map((_, subIndex) => `-map ${subIndex + index}`)); + args.push(...metaData) args.push( - '-metadata:s:v:0 title="[Funimation]"', - `-metadata:s:a:0 language=${getLanguageCode(audioSettings.language, argv.sub ? 'jpn' : 'eng')}`, '-c:v copy', '-c:a copy', '-c:s mov_text', @@ -58,14 +87,14 @@ const buildCommandMkvMerge = (videoFile, audioSettings, subtitles, output) => { '--no-audio' ); args.push(`"${videoFile}"`); - args.push(`--language 0:${getLanguageCode(audioSettings.language, argv.sub ? 'jpn' : 'eng')}`); + args.push(`--language 0:${getLanguageCode(audioSettings.language, argv.todo ? 'jpn' : 'eng')}`); args.push( '--no-video', '--audio-tracks 0' ); args.push(`"${audioSettings.uri}"`); } else{ - args.push(`--language 1:${argv.sub ? 'jpn' : 'eng'}`); + args.push(`--language 1:${argv.todo ? 'jpn' : 'eng'}`); args.push( '--video-tracks 0', '--audio-tracks 1' diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 8b7f141..796a528 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -13,7 +13,7 @@ const appArgv = (cfg) => { // init const argv = yargs.parserConfiguration({ 'duplicate-arguments-array': true, - 'camel-case-expansion': false + "camel-case-expansion": false }) // main .wrap(Math.min(120)) // yargs.terminalWidth() @@ -74,16 +74,10 @@ const appArgv = (cfg) => { .option('dub', { group: 'Downloading:', describe: 'Download non-Japanese Dub (English Dub mode by default)', - choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN' ], + choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN', 'jpJP' ], default: cfg.dub || 'enUS', type: 'array', }) - .option('sub', { - group: 'Downloading:', - describe: 'Japanese Dub with subtitles mode (English Dub mode by default)', - default: cfg.subsMode || false, - type: 'boolean', - }) .option('subLang', { group: 'Downloading:', describe: 'Set the subtitle language (English is default and fallback)', @@ -187,17 +181,10 @@ const appArgv = (cfg) => { // util .option('nocleanup', { group: 'Utilities:', - describe: 'Move temporary files to trash folder instead of deleting', + describe: 'Dont\'t delete the input files after muxing', default: cfg.noCleanUp || false, type: 'boolean' }) - .option('notrashfolder', { - implies: ['nocleanup'], - group: 'Utilities:', - describe: 'Don\'t move temporary files to trash folder (Used with --nocleanup)', - default: cfg.noTrashFolder || false, - type: 'boolean' - }) // help .option('help', { alias: 'h', @@ -217,8 +204,8 @@ const appArgv = (cfg) => { // Resolve unwanted arrays for (let key in argv) { - if (argv[key] instanceof Array && !(key === 'subLang' || key === 'dub')) { - argv[key] = argv[key].pop(); + if (argv[key] instanceof Array && !(key === "subLang" || key === "dub")) { + argv[key] = argv[key].pop() } } return argv; diff --git a/videos/_trash/.gitkeep b/videos/_trash/.gitkeep deleted file mode 100644 index e69de29..0000000 -- 2.45.2 From ed7f0c2dde8a028477686006a3fbb136df74a893 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Fri, 2 Jul 2021 23:19:37 +0200 Subject: [PATCH 40/51] Mkvmerge --- funi.js | 7 ++--- modules/merger.js | 66 ++++++++++++++++++++++++++++++----------------- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/funi.js b/funi.js index 3dfeebe..3fa9133 100644 --- a/funi.js +++ b/funi.js @@ -601,10 +601,8 @@ async function downloadStreams(){ video: if (!argv.novids) { if (plAud.uri && (purvideo.length > 1 || audioAndVideo.length > 1)) { - console.log("break 1") break video; } else if (!plAud.uri && (audioAndVideo.some(a => a.lang === streamPath.lang) || puraudio.some(a => a.lang === streamPath.lang))) { - console.log("break 2") break video; } // download video @@ -701,8 +699,7 @@ async function downloadStreams(){ } // usage - /* TODO MkvMerge */ - let usableMKVmerge = false; + let usableMKVmerge = true; let usableFFmpeg = true; // check exec path @@ -725,7 +722,6 @@ async function downloadStreams(){ if(!argv.mp4 && usableMKVmerge){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; let command = merger.buildCommandMkvMerge(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`); - console.log(command); shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command); } else if(usableFFmpeg){ @@ -741,6 +737,7 @@ async function downloadStreams(){ return; audioAndVideo.concat(puraudio).concat(purvideo).forEach(a => fs.unlinkSync(a.path)) + stDlPath.forEach(file => fs.unlinkSync(subObject.file)) console.log('\n[INFO] Done!\n'); } diff --git a/modules/merger.js b/modules/merger.js index 6d2b7b7..50cd942 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -4,12 +4,12 @@ const argv = require('../funi').argv; /** * @param {Array} videoAndAudio * @param {Array} onlyVid - * @param {Array} onlyAuido + * @param {Array} onlyAudio * @param {Array} subtitles * @param {string} output * @returns {string} */ -const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAuido, subtitles, output) => { +const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAudio, subtitles, output) => { let args = []; let metaData = []; @@ -40,7 +40,7 @@ const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAuido, subtitles, output } } - for (let aud of onlyAuido) { + for (let aud of onlyAudio) { args.push(`-i "${aud.path}"`) metaData.push(`-map ${index}`) metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(aud.lang, aud.lang)}`) @@ -71,41 +71,61 @@ const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAuido, subtitles, output * @param {Array} subtitles * @returns {string} */ -const buildCommandMkvMerge = (videoFile, audioSettings, subtitles, output) => { +const buildCommandMkvMerge = (videoAndAudio, onlyVid, onlyAudio, subtitles, output) => { let args = []; + + let hasVideo = false; + args.push(`-o "${output}"`); args.push( '--no-date', '--disable-track-statistics-tags', '--engage no_variable_data', - '--track-name 0:[Funimation]' ); - if (audioSettings.uri) { - args.push( - '--video-tracks 0', - '--no-audio' - ); - args.push(`"${videoFile}"`); - args.push(`--language 0:${getLanguageCode(audioSettings.language, argv.todo ? 'jpn' : 'eng')}`); + for (let vid of videoAndAudio) { + if (!hasVideo) { + args.push( + '--video-tracks 0', + '--audio-tracks 1' + ) + args.push(`--track-name 0:[Funimation]`) + args.push(`--language 1:${getLanguageCode(vid.lang, argv.todo ? 'jpn' : 'eng')}`); + hasVideo = true + } else { + args.push( + '--no-video', + '--audio-tracks 1' + ) + args.push(`--language 1:${getLanguageCode(vid.lang, argv.todo ? 'jpn' : 'eng')}`); + } + args.push(`"${vid.path}"`) + } + + for (let vid of onlyVid) { + if (!hasVideo) { + args.push( + '--video-tracks 0', + '--no-audio' + ) + args.push(`--track-name 0:[Funimation]`) + hasVideo = true + args.push(`${vid.path}"`) + } + } + + for (let aud of onlyAudio) { + args.push(`--language 0:${getLanguageCode(aud.lang, argv.todo ? 'jpn' : 'eng')}`); args.push( '--no-video', '--audio-tracks 0' ); - args.push(`"${audioSettings.uri}"`); - } else{ - args.push(`--language 1:${argv.todo ? 'jpn' : 'eng'}`); - args.push( - '--video-tracks 0', - '--audio-tracks 1' - ); - args.push(`"${videoFile}"`); + args.push(`"${aud.path}"`) } if(subtitles.length > 0){ - for (let index in subtitles) { - let subObj = subtitles[index]; - args.push('--language',`${/*parseInt(index) + (audioSettings.uri ? 2 : 1)*/0}:${getLanguageCode(subObj.language)}`); + for (let subObj of subtitles) { + args.push('--language',`0:${getLanguageCode(subObj.language)}`); args.push(`"${subObj.file}"`); } } else { -- 2.45.2 From c54cb79bfbedb78fa559153a402632f109294197 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Fri, 2 Jul 2021 23:26:04 +0200 Subject: [PATCH 41/51] Small bug fix --- funi.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funi.js b/funi.js index 3fa9133..1fa709e 100644 --- a/funi.js +++ b/funi.js @@ -421,7 +421,7 @@ async function getEpisode(fnSlug){ function getSubsUrl(m){ if(argv.nosubs && !argv.sub){ - return false; + return []; } let subLangs = argv.subLang; -- 2.45.2 From d85079c1085ed1920e92fbe0a5577ae181e0c0af Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Fri, 2 Jul 2021 23:58:15 +0200 Subject: [PATCH 42/51] Fixed file not found --- funi.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/funi.js b/funi.js index 1fa709e..bc10af9 100644 --- a/funi.js +++ b/funi.js @@ -737,7 +737,7 @@ async function downloadStreams(){ return; audioAndVideo.concat(puraudio).concat(purvideo).forEach(a => fs.unlinkSync(a.path)) - stDlPath.forEach(file => fs.unlinkSync(subObject.file)) + stDlPath.forEach(subObject => fs.unlinkSync(subObject.file)) console.log('\n[INFO] Done!\n'); } -- 2.45.2 From 74983a0c981aeb4b4dc7fa59ed64048a02f9675c Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Sat, 3 Jul 2021 14:17:13 +0200 Subject: [PATCH 43/51] Fixed broken Audio files --- funi.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/funi.js b/funi.js index bc10af9..9681969 100644 --- a/funi.js +++ b/funi.js @@ -493,8 +493,9 @@ async function downloadStreams(){ // new uris let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/; if(plQualityLinkList.playlists[0].uri.match(vplReg)){ - if(plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128']){ - let audioData = plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128'], + let audioKey = Object.keys(plQualityLinkList.mediaGroups.AUDIO).pop() + if(plQualityLinkList.mediaGroups.AUDIO[audioKey]){ + let audioData = plQualityLinkList.mediaGroups.AUDIO[audioKey], audioEl = Object.keys(audioData); audioData = audioData[audioEl[0]]; plAud = { ...audioData, ...{ langStr: audioEl[0] } }; -- 2.45.2 From 38c2a620321b5cdc76e004210e922f38dcfff9bb Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Sat, 3 Jul 2021 15:11:55 +0200 Subject: [PATCH 44/51] Multiple file download --- funi.js | 7 +++---- modules/merger.js | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/funi.js b/funi.js index 9681969..fb0a7e4 100644 --- a/funi.js +++ b/funi.js @@ -339,7 +339,6 @@ async function getEpisode(fnSlug){ }; // select - media = media.reverse(); for(let m of media){ let selected = false; if(m.id > 0 && m.type == 'Non-Encrypted'){ @@ -352,7 +351,7 @@ async function getEpisode(fnSlug){ if(dub_type == dubType[curDub] && selUncut){ streamIds.push({ id: m.id, - lang: merger.getLanguageCode(dubType[curDub], curDub.slice(0, -2)) + lang: merger.getLanguageCode(curDub, curDub.slice(0, -2)) }); stDlPath.push(...m.subtitles); localSubs = m.subtitles @@ -374,7 +373,6 @@ async function getEpisode(fnSlug){ return true; } }) - if(streamIds.length <1){ console.log('[ERROR] Track not selected\n'); return; @@ -466,6 +464,7 @@ async function downloadStreams(){ let puraudio = [] let audioAndVideo = [] let outName; + console.log(tsDlPath) for (let streamPath of tsDlPath) { let plQualityReq = await getData({ url: streamPath.path, @@ -676,7 +675,7 @@ async function downloadStreams(){ }); if(subsSrc.ok){ let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.langName, argv.fontSize); - subObject.file = path.join(cfg.dir.content, fnOutput) + subObject.ext + subsExt; + subObject.file = path.join(cfg.dir.content, `${fnOutput}.subtitle.${subObject.ext}${subsExt}`) fs.writeFileSync(subObject.file, assData); } else{ diff --git a/modules/merger.js b/modules/merger.js index 50cd942..99a6f13 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -110,7 +110,7 @@ const buildCommandMkvMerge = (videoAndAudio, onlyVid, onlyAudio, subtitles, outp ) args.push(`--track-name 0:[Funimation]`) hasVideo = true - args.push(`${vid.path}"`) + args.push(`"${vid.path}"`) } } -- 2.45.2 From 179535e0d86209afd6f5cb0f03526be494a60085 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Sat, 3 Jul 2021 21:08:15 +0200 Subject: [PATCH 45/51] Final bug fixes --- funi.js | 15 +++++++-------- modules/merger.js | 7 +++---- modules/module.app-args.js | 6 ------ 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/funi.js b/funi.js index fb0a7e4..9ef89d1 100644 --- a/funi.js +++ b/funi.js @@ -464,7 +464,6 @@ async function downloadStreams(){ let puraudio = [] let audioAndVideo = [] let outName; - console.log(tsDlPath) for (let streamPath of tsDlPath) { let plQualityReq = await getData({ url: streamPath.path, @@ -600,7 +599,7 @@ async function downloadStreams(){ let dlFailedA = false; video: if (!argv.novids) { - if (plAud.uri && (purvideo.length > 1 || audioAndVideo.length > 1)) { + if (plAud.uri && (purvideo.length > 0 || audioAndVideo.length > 0)) { break video; } else if (!plAud.uri && (audioAndVideo.some(a => a.lang === streamPath.lang) || puraudio.some(a => a.lang === streamPath.lang))) { break video; @@ -634,10 +633,10 @@ async function downloadStreams(){ else{ console.log('[INFO] Skip video downloading...\n'); } - if (!argv.noaudio && plAud.uri) { + audio: if (!argv.noaudio && plAud.uri) { // download audio if (audioAndVideo.some(a => a.lang === plAud.language) || puraudio.some(a => a.lang === plAud.language)) - return; + break audio; let reqAudio = await getData({ url: plAud.uri, useProxy: (argv.ssp ? false : true), @@ -659,10 +658,9 @@ async function downloadStreams(){ } } - // add subs - let subsExt = !argv.mp4 || argv.mp4 && !argv.mks && argv.ass ? '.ass' : '.srt'; - let addSubs = argv.mks && tsDlPath ? true : false; + let subsExt = !argv.mp4 || argv.mp4 && argv.ass ? '.ass' : '.srt'; + let addSubs = true; // download subtitles if(stDlPath.length > 0){ @@ -675,7 +673,7 @@ async function downloadStreams(){ }); if(subsSrc.ok){ let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.langName, argv.fontSize); - subObject.file = path.join(cfg.dir.content, `${fnOutput}.subtitle.${subObject.ext}${subsExt}`) + subObject.file = path.join(cfg.dir.content, `${fnOutput}.subtitle${subObject.ext}${subsExt}`) fs.writeFileSync(subObject.file, assData); } else{ @@ -722,6 +720,7 @@ async function downloadStreams(){ if(!argv.mp4 && usableMKVmerge){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; let command = merger.buildCommandMkvMerge(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`); + console.log(command, audioAndVideo, puraudio, purvideo) shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command); } else if(usableFFmpeg){ diff --git a/modules/merger.js b/modules/merger.js index 99a6f13..41b2d11 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -52,14 +52,13 @@ const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAudio, subtitles, output args.push(`-i "${sub.file}"`); } - args.push(...subtitles.map((_, subIndex) => `-map ${subIndex + index}`)); args.push(...metaData) + args.push(...subtitles.map((_, subIndex) => `-map ${subIndex + index}`)); args.push( '-c:v copy', - '-c:a copy', - '-c:s mov_text', - '-c:s ass' + '-c:a copy' ); + args.push(output.split('.').pop().toLowerCase() === "mp4" ? '-c:s mov_text' : '*c:s ass') args.push(...subtitles.map((sub, index) => `-metadata:s:${index + 2} language=${getLanguageCode(sub.language)}`)); args.push(`"${output}"`); return args.join(' '); diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 796a528..153ba3d 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -157,12 +157,6 @@ const appArgv = (cfg) => { default: cfg.mp4mux || false, type: 'boolean' }) - .option('mks', { - group: 'Muxing:', - describe: 'Add subtitles to mkv/mp4 (if available)', - default: cfg.muxSubs || false, - type: 'boolean' - }) // filenaming .option('fileName', { group: 'Filename Template:', -- 2.45.2 From ad030cd1f7b126b94e482b41295084ae22bf8fab Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Sat, 3 Jul 2021 21:13:25 +0200 Subject: [PATCH 46/51] Update version and doc files --- docs/CHANGELOG.md | 2 ++ docs/README.md | 17 ++++++++++------- package.json | 2 +- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0cb42ae..4231816 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,5 +1,7 @@ ## Change Log +This changelog is out of date and wont be continued. Please see the releases comments, or if not present the commit comments. + ### 4.7.0 (unreleased) - Change subtitles parser from ttml to vtt - Improve help command diff --git a/docs/README.md b/docs/README.md index a7b1edd..aa739da 100644 --- a/docs/README.md +++ b/docs/README.md @@ -42,15 +42,20 @@ After installing NodeJS with NPM go to directory with `package.json` file and ty * `-s -e ` sets the show id and episode ids (comma-separated, hyphen-sequence) * `-q ` sets the video layer quality [1...10] (optional, 0 is max) +* `--all` download all videos at once * `--alt` alternative episode listing (if available) -* `--sub` switch from English dub to Japanese dub with subtitles +* `--sub` select one or more subtile language +* `--dub` select one or more dub languages * `--simul` force select simulcast version instead of uncut version * `-x` select server -* `--novids` skip download videos (for downloading subtitles only) +* `--novids` skip download videos * `--nosubs` skip download subtitles for Dub (if available) +* `--noaudio` skip downloading audio ### Proxy +The proxy is currently unmainted. Use at your on risk. + * `--proxy ` http(s)/socks proxy WHATWG url (ex. https://myproxyhost:1080) * `--proxy-auth ` Colon-separated username and password for proxy * `--ssp` don't use proxy for stream downloading @@ -63,10 +68,8 @@ After installing NodeJS with NPM go to directory with `package.json` file and ty ### Filenaming (optional) -* `-a ` release group ("Funimation" by default) -* `-t ` show title override -* `--ep ` episode number override (ignored in batch mode) -* `--suffix ` filename suffix override (first "SIZEp" will be replaced with actual video size, "SIZEp" by default) +* `--fileName` Set the filename template. Use ${variable_name} to insert variables. + You may use 'title', 'episode', 'showTitle', 'season', 'width', 'height' as variables. ### Utility @@ -75,7 +78,7 @@ After installing NodeJS with NPM go to directory with `package.json` file and ty ## Filename Template -[`release group`] `title` - `episode` [`suffix`].`extension` +[Funimation] ${showTitle} - ${episode} [${height}p]"] ## CLI Examples diff --git a/package.json b/package.json index 9ab9f31..2c4be24 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "funimation-downloader-nx", "short_name": "funi", - "version": "4.8.3", + "version": "4.9.0", "description": "Download videos from Funimation via cli.", "keywords": [ "download", -- 2.45.2 From 70ad9b5f7224923f52d8dc23ccf6ce8d4f7158f4 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Sat, 3 Jul 2021 21:24:29 +0200 Subject: [PATCH 47/51] Removed debug infomation --- funi.js | 1 - 1 file changed, 1 deletion(-) diff --git a/funi.js b/funi.js index 9ef89d1..6fe2c67 100644 --- a/funi.js +++ b/funi.js @@ -720,7 +720,6 @@ async function downloadStreams(){ if(!argv.mp4 && usableMKVmerge){ let ffext = !argv.mp4 ? 'mkv' : 'mp4'; let command = merger.buildCommandMkvMerge(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`); - console.log(command, audioAndVideo, puraudio, purvideo) shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command); } else if(usableFFmpeg){ -- 2.45.2 From 42bf60db2c25a7fc5688ae17a1b9c128b736fb5f Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Sat, 3 Jul 2021 21:48:31 +0200 Subject: [PATCH 48/51] Fixed typo --- modules/merger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/merger.js b/modules/merger.js index 41b2d11..e8150fa 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -58,7 +58,7 @@ const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAudio, subtitles, output '-c:v copy', '-c:a copy' ); - args.push(output.split('.').pop().toLowerCase() === "mp4" ? '-c:s mov_text' : '*c:s ass') + args.push(output.split('.').pop().toLowerCase() === "mp4" ? '-c:s mov_text' : '-c:s ass') args.push(...subtitles.map((sub, index) => `-metadata:s:${index + 2} language=${getLanguageCode(sub.language)}`)); args.push(`"${output}"`); return args.join(' '); -- 2.45.2 From 79048ff53460c993014f6f640c86094e281a8cf7 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Sun, 4 Jul 2021 20:36:39 +0200 Subject: [PATCH 49/51] Fixed japanese audio being labeled incorrectly --- funi.js | 2 +- modules/merger.js | 41 +++++++++++++++++++------------------- modules/module.app-args.js | 2 +- package.json | 2 +- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/funi.js b/funi.js index 6fe2c67..f16dbd0 100644 --- a/funi.js +++ b/funi.js @@ -335,7 +335,7 @@ async function getEpisode(fnSlug){ 'esLA': 'Spanish (Latin Am)', 'ptBR': 'Portuguese (Brazil)', 'zhMN': 'Chinese (Mandarin, PRC)', - 'jpJP': 'Japanese' + 'jaJP': 'Japanese' }; // select diff --git a/modules/merger.js b/modules/merger.js index e8150fa..7775c98 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -82,25 +82,6 @@ const buildCommandMkvMerge = (videoAndAudio, onlyVid, onlyAudio, subtitles, outp '--engage no_variable_data', ); - for (let vid of videoAndAudio) { - if (!hasVideo) { - args.push( - '--video-tracks 0', - '--audio-tracks 1' - ) - args.push(`--track-name 0:[Funimation]`) - args.push(`--language 1:${getLanguageCode(vid.lang, argv.todo ? 'jpn' : 'eng')}`); - hasVideo = true - } else { - args.push( - '--no-video', - '--audio-tracks 1' - ) - args.push(`--language 1:${getLanguageCode(vid.lang, argv.todo ? 'jpn' : 'eng')}`); - } - args.push(`"${vid.path}"`) - } - for (let vid of onlyVid) { if (!hasVideo) { args.push( @@ -113,8 +94,28 @@ const buildCommandMkvMerge = (videoAndAudio, onlyVid, onlyAudio, subtitles, outp } } + for (let vid of videoAndAudio) { + console.log(vid, vid.lang) + if (!hasVideo) { + args.push( + '--video-tracks 0', + '--audio-tracks 1' + ) + args.push(`--track-name 0:[Funimation]`) + args.push(`--language 1:${getLanguageCode(vid.lang, vid.lang)}`); + hasVideo = true + } else { + args.push( + '--no-video', + '--audio-tracks 1' + ) + args.push(`--language 1:${getLanguageCode(vid.lang, vid.lang)}`); + } + args.push(`"${vid.path}"`) + } + for (let aud of onlyAudio) { - args.push(`--language 0:${getLanguageCode(aud.lang, argv.todo ? 'jpn' : 'eng')}`); + args.push(`--language 0:${getLanguageCode(aud.lang, aud.lang)}`); args.push( '--no-video', '--audio-tracks 0' diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 153ba3d..14cefaf 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -74,7 +74,7 @@ const appArgv = (cfg) => { .option('dub', { group: 'Downloading:', describe: 'Download non-Japanese Dub (English Dub mode by default)', - choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN', 'jpJP' ], + choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN', 'jaJP' ], default: cfg.dub || 'enUS', type: 'array', }) diff --git a/package.json b/package.json index 2c4be24..f2d1d68 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "funimation-downloader-nx", "short_name": "funi", - "version": "4.9.0", + "version": "4.9.1", "description": "Download videos from Funimation via cli.", "keywords": [ "download", -- 2.45.2 From bfec2f4b8932745994924101d3cad037ec185b07 Mon Sep 17 00:00:00 2001 From: Izuco <57068530+izu-co@users.noreply.github.com> Date: Sun, 4 Jul 2021 20:37:20 +0200 Subject: [PATCH 50/51] Eslint --- funi.js | 32 +++++++++++----------- modules/merger.js | 56 +++++++++++++++++++------------------- modules/module.app-args.js | 6 ++-- 3 files changed, 47 insertions(+), 47 deletions(-) diff --git a/funi.js b/funi.js index f16dbd0..7a88626 100644 --- a/funi.js +++ b/funi.js @@ -343,7 +343,7 @@ async function getEpisode(fnSlug){ let selected = false; if(m.id > 0 && m.type == 'Non-Encrypted'){ let dub_type = m.language; - let localSubs = [] + let localSubs = []; let selUncut = !argv.simul && uncut[dub_type] && m.version.match(/uncut/i) ? true : (!uncut[dub_type] || argv.simul && m.version.match(/simulcast/i) ? true : false); @@ -354,7 +354,7 @@ async function getEpisode(fnSlug){ lang: merger.getLanguageCode(curDub, curDub.slice(0, -2)) }); stDlPath.push(...m.subtitles); - localSubs = m.subtitles + localSubs = m.subtitles; selected = true; } } @@ -364,15 +364,15 @@ async function getEpisode(fnSlug){ } } - let already = [] + let already = []; stDlPath = stDlPath.filter(a => { if (already.includes(a.language)) { return false; } else { - already.push(a.language) + already.push(a.language); return true; } - }) + }); if(streamIds.length <1){ console.log('[ERROR] Track not selected\n'); return; @@ -460,9 +460,9 @@ async function downloadStreams(){ // req playlist - let purvideo = [] - let puraudio = [] - let audioAndVideo = [] + let purvideo = []; + let puraudio = []; + let audioAndVideo = []; let outName; for (let streamPath of tsDlPath) { let plQualityReq = await getData({ @@ -491,7 +491,7 @@ async function downloadStreams(){ // new uris let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/; if(plQualityLinkList.playlists[0].uri.match(vplReg)){ - let audioKey = Object.keys(plQualityLinkList.mediaGroups.AUDIO).pop() + let audioKey = Object.keys(plQualityLinkList.mediaGroups.AUDIO).pop(); if(plQualityLinkList.mediaGroups.AUDIO[audioKey]){ let audioData = plQualityLinkList.mediaGroups.AUDIO[audioKey], audioEl = Object.keys(audioData); @@ -621,12 +621,12 @@ async function downloadStreams(){ purvideo.push({ path: `${tsFile}.ts`, lang: plAud.language - }) + }); } else { audioAndVideo.push({ path: `${tsFile}.ts`, lang: streamPath.lang - }) + }); } } } @@ -653,7 +653,7 @@ async function downloadStreams(){ puraudio.push({ path: `${tsFileA}.ts`, lang: plAud.language - }) + }); } } @@ -673,7 +673,7 @@ async function downloadStreams(){ }); if(subsSrc.ok){ let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.langName, argv.fontSize); - subObject.file = path.join(cfg.dir.content, `${fnOutput}.subtitle${subObject.ext}${subsExt}`) + subObject.file = path.join(cfg.dir.content, `${fnOutput}.subtitle${subObject.ext}${subsExt}`); fs.writeFileSync(subObject.file, assData); } else{ @@ -692,7 +692,7 @@ async function downloadStreams(){ } if(argv.skipmux){ - console.log("[INFO] Skipping muxing...") + console.log('[INFO] Skipping muxing...'); return; } @@ -734,8 +734,8 @@ async function downloadStreams(){ if (argv.nocleanup) return; - audioAndVideo.concat(puraudio).concat(purvideo).forEach(a => fs.unlinkSync(a.path)) - stDlPath.forEach(subObject => fs.unlinkSync(subObject.file)) + audioAndVideo.concat(puraudio).concat(purvideo).forEach(a => fs.unlinkSync(a.path)); + stDlPath.forEach(subObject => fs.unlinkSync(subObject.file)); console.log('\n[INFO] Done!\n'); } diff --git a/modules/merger.js b/modules/merger.js index 7775c98..a111545 100644 --- a/modules/merger.js +++ b/modules/merger.js @@ -16,34 +16,34 @@ const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAudio, subtitles, output let index = 0; let hasVideo = false; for (let vid of videoAndAudio) { - args.push(`-i "${vid.path}"`) + args.push(`-i "${vid.path}"`); if (!hasVideo) { - metaData.push(`-map ${index}`) - metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`) - metaData.push(`-metadata:s:v:${index} title="[Funimation]"`) - hasVideo = true + metaData.push(`-map ${index}`); + metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`); + metaData.push(`-metadata:s:v:${index} title="[Funimation]"`); + hasVideo = true; } else { - metaData.push(`-map ${index}:a`) - metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`) + metaData.push(`-map ${index}:a`); + metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`); } index++; } for (let vid of onlyVid) { if (!hasVideo) { - args.push(`-i "${vid.path}"`) - metaData.push(`-map ${index}`) - metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`) - metaData.push(`-metadata:s:v:${index} title="[Funimation]"`) - hasVideo = true + args.push(`-i "${vid.path}"`); + metaData.push(`-map ${index}`); + metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`); + metaData.push(`-metadata:s:v:${index} title="[Funimation]"`); + hasVideo = true; index++; } } for (let aud of onlyAudio) { - args.push(`-i "${aud.path}"`) - metaData.push(`-map ${index}`) - metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(aud.lang, aud.lang)}`) + args.push(`-i "${aud.path}"`); + metaData.push(`-map ${index}`); + metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(aud.lang, aud.lang)}`); index++; } @@ -52,13 +52,13 @@ const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAudio, subtitles, output args.push(`-i "${sub.file}"`); } - args.push(...metaData) + args.push(...metaData); args.push(...subtitles.map((_, subIndex) => `-map ${subIndex + index}`)); args.push( '-c:v copy', '-c:a copy' ); - args.push(output.split('.').pop().toLowerCase() === "mp4" ? '-c:s mov_text' : '-c:s ass') + args.push(output.split('.').pop().toLowerCase() === 'mp4' ? '-c:s mov_text' : '-c:s ass'); args.push(...subtitles.map((sub, index) => `-metadata:s:${index + 2} language=${getLanguageCode(sub.language)}`)); args.push(`"${output}"`); return args.join(' '); @@ -87,31 +87,31 @@ const buildCommandMkvMerge = (videoAndAudio, onlyVid, onlyAudio, subtitles, outp args.push( '--video-tracks 0', '--no-audio' - ) - args.push(`--track-name 0:[Funimation]`) - hasVideo = true - args.push(`"${vid.path}"`) + ); + args.push('--track-name 0:[Funimation]'); + hasVideo = true; + args.push(`"${vid.path}"`); } } for (let vid of videoAndAudio) { - console.log(vid, vid.lang) + console.log(vid, vid.lang); if (!hasVideo) { args.push( '--video-tracks 0', '--audio-tracks 1' - ) - args.push(`--track-name 0:[Funimation]`) + ); + args.push('--track-name 0:[Funimation]'); args.push(`--language 1:${getLanguageCode(vid.lang, vid.lang)}`); - hasVideo = true + hasVideo = true; } else { args.push( '--no-video', '--audio-tracks 1' - ) + ); args.push(`--language 1:${getLanguageCode(vid.lang, vid.lang)}`); } - args.push(`"${vid.path}"`) + args.push(`"${vid.path}"`); } for (let aud of onlyAudio) { @@ -120,7 +120,7 @@ const buildCommandMkvMerge = (videoAndAudio, onlyVid, onlyAudio, subtitles, outp '--no-video', '--audio-tracks 0' ); - args.push(`"${aud.path}"`) + args.push(`"${aud.path}"`); } if(subtitles.length > 0){ diff --git a/modules/module.app-args.js b/modules/module.app-args.js index 14cefaf..8f11149 100644 --- a/modules/module.app-args.js +++ b/modules/module.app-args.js @@ -13,7 +13,7 @@ const appArgv = (cfg) => { // init const argv = yargs.parserConfiguration({ 'duplicate-arguments-array': true, - "camel-case-expansion": false + 'camel-case-expansion': false }) // main .wrap(Math.min(120)) // yargs.terminalWidth() @@ -198,8 +198,8 @@ const appArgv = (cfg) => { // Resolve unwanted arrays for (let key in argv) { - if (argv[key] instanceof Array && !(key === "subLang" || key === "dub")) { - argv[key] = argv[key].pop() + if (argv[key] instanceof Array && !(key === 'subLang' || key === 'dub')) { + argv[key] = argv[key].pop(); } } return argv; -- 2.45.2 From a69f83fea108d6f74339a1d920ba0178f345a1e6 Mon Sep 17 00:00:00 2001 From: Jake K Date: Tue, 6 Jul 2021 11:24:02 -0400 Subject: [PATCH 51/51] fixed eslint errors, removed cli and bin defaults --- funi.js | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/funi.js b/funi.js index 22d0b4e..8ac0d38 100644 --- a/funi.js +++ b/funi.js @@ -37,31 +37,23 @@ let cfg = { cli: getYamlCfg(cliCfgFile), }; -// make sure cfg params aren't empty -// If they are, update them with some defaults +// make sure cfg params aren't null if (cfg.bin === null){ - cfg.bin = {}; - console.log('[WARN] bin-path.yml is empty or does not exist!\n'); + cfg.bin = {}; + console.log('[WARN] bin-path.yml is empty or does not exist!\n'); } -if (!cfg.bin.hasOwnProperty('ffmpeg')) cfg.bin.ffmpeg = './bin/ffmpeg'; -if (!cfg.bin.hasOwnProperty('mkvmerge')) cfg.bin.mkvmerge = './bin/mkvmerge'; if (cfg.dir === null){ - cfg.dir = {}; - console.log('[WARN] dir-path.yml is empty or does not exist!\n'); + cfg.dir = {}; + console.log('[WARN] dir-path.yml is empty or does not exist!\n'); } -if (!cfg.dir.hasOwnProperty('content')) cfg.dir.content = './videos/'; + +if (!Object.prototype.hasOwnProperty.call(cfg.dir, 'content')) cfg.dir.content = './videos/'; if (cfg.cli === null){ - cfg.cli = {}; - console.log('[WARN] cli-defaults.yml is empty or does not exist!\n'); + cfg.cli = {}; + console.log('[WARN] cli-defaults.yml is empty or does not exist!\n'); } -if (!cfg.cli.hasOwnProperty('releaseGroup')) cfg.cli.releaseGroup = "Funimation"; -if (!cfg.cli.hasOwnProperty('videoLayer')) cfg.cli.videoLayer = 7; -if (!cfg.cli.hasOwnProperty('nServer')) cfg.cli.nServer = 1; -if (!cfg.cli.hasOwnProperty('mp4mux')) cfg.cli.mp4mux = false; -if (!cfg.cli.hasOwnProperty('muxSubs')) cfg.cli.muxSubs = false; -if (!cfg.cli.hasOwnProperty('noCleanUp')) cfg.cli.noCleanUp = false; /* Normalise paths for use outside the current directory */ for (let key of Object.keys(cfg.dir)) { @@ -950,4 +942,4 @@ function parseFileName(input, title, episode, showTitle, season, width, height) } } return shlp.cleanupFilename(input); -} \ No newline at end of file +} -- 2.45.2