Update downloader to handle multiple subtitles and audio languages in the same run

This commit is contained in:
Izuco 2021-07-03 21:13:43 +02:00 committed by GitHub
commit 6e4a9371d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 493 additions and 364 deletions

View file

@ -1,2 +1,2 @@
ffmpeg: "./bin/ffmpeg/ffmpeg.exe"
ffmpeg: "C:\\Program Files\\ffmpeg\\bin\\ffmpeg.exe"
mkvmerge: "./bin/mkvtoolnix/mkvmerge.exe"

View file

@ -1,2 +1 @@
content: ./videos/
trash: ./videos/_trash/

View file

@ -1,5 +1,7 @@
## Change Log
This changelog is out of date and wont be continued. Please see the releases comments, or if not present the commit comments.
### 4.7.0 (unreleased)
- Change subtitles parser from ttml to vtt
- Improve help command

View file

@ -42,15 +42,20 @@ After installing NodeJS with NPM go to directory with `package.json` file and ty
* `-s <i> -e <s>` sets the show id and episode ids (comma-separated, hyphen-sequence)
* `-q <i>` sets the video layer quality [1...10] (optional, 0 is max)
* `--all` download all videos at once
* `--alt` alternative episode listing (if available)
* `--sub` switch from English dub to Japanese dub with subtitles
* `--sub` select one or more subtile language
* `--dub` select one or more dub languages
* `--simul` force select simulcast version instead of uncut version
* `-x` select server
* `--novids` skip download videos (for downloading subtitles only)
* `--novids` skip download videos
* `--nosubs` skip download subtitles for Dub (if available)
* `--noaudio` skip downloading audio
### Proxy
The proxy is currently unmainted. Use at your on risk.
* `--proxy <s>` http(s)/socks proxy WHATWG url (ex. https://myproxyhost:1080)
* `--proxy-auth <s>` Colon-separated username and password for proxy
* `--ssp` don't use proxy for stream downloading
@ -63,19 +68,17 @@ After installing NodeJS with NPM go to directory with `package.json` file and ty
### Filenaming (optional)
* `-a <s>` release group ("Funimation" by default)
* `-t <s>` show title override
* `--ep <s>` episode number override (ignored in batch mode)
* `--suffix <s>` filename suffix override (first "SIZEp" will be replaced with actual video size, "SIZEp" by default)
* `--fileName` Set the filename template. Use ${variable_name} to insert variables.
You may use 'title', 'episode', 'showTitle', 'season', 'width', 'height' as variables.
### Utility
* `--nocleanup` move unnecessary files to trash folder after completion instead of deleting
* `--nocleanup` don't delete the input files after the muxing
* `-h`, `--help` show all options
## Filename Template
[`release group`] `title` - `episode` [`suffix`].`extension`
[Funimation] ${showTitle} - ${episode} [${height}p]"]
## CLI Examples

635
funi.js
View file

@ -18,7 +18,6 @@ const { lookpath } = require('lookpath');
const m3u8 = require('m3u8-parsed');
const crypto = require('crypto');
const got = require('got');
const iso639 = require('iso-639');
// extra
const appYargs = require('./modules/module.app-args');
@ -41,13 +40,13 @@ let cfg = {
/* Normalise paths for use outside the current directory */
for (let key of Object.keys(cfg.dir)) {
if (!path.isAbsolute(cfg.dir[key])) {
cfg.dir[key] = path.join(workingDir, cfg.dir[key])
cfg.dir[key] = path.join(workingDir, cfg.dir[key]);
}
}
for (let key of Object.keys(cfg.bin)) {
if (!path.isAbsolute(cfg.bin[key])) {
cfg.bin[key] = path.join(workingDir, cfg.bin[key])
cfg.bin[key] = path.join(workingDir, cfg.bin[key]);
}
}
@ -62,6 +61,14 @@ if(!token){
// cli
const argv = appYargs.appArgv(cfg.cli);
module.exports = {
argv,
cfg
};
// Import merger after argv has been exported
const merger = require('./modules/merger');
// check page
if(!isNaN(parseInt(argv.p, 10)) && parseInt(argv.p, 10) > 0){
@ -77,8 +84,8 @@ let title = '',
fnEpNum = 0,
fnOutput = '',
season = 0,
tsDlPath = false,
stDlPath = undefined;
tsDlPath = [],
stDlPath = [];
// select mode
if(argv.auth){
@ -277,7 +284,7 @@ async function getEpisode(fnSlug){
debug: argv.debug,
});
if(!episodeData.ok){return;}
let ep = JSON.parse(episodeData.res.body).items[0], streamId = 0;
let ep = JSON.parse(episodeData.res.body).items[0], streamIds = [];
// build fn
showTitle = ep.parent.title;
title = ep.title;
@ -327,62 +334,80 @@ async function getEpisode(fnSlug){
'enUS': 'English',
'esLA': 'Spanish (Latin Am)',
'ptBR': 'Portuguese (Brazil)',
'zhMN': 'Chinese (Mandarin, PRC)'
'zhMN': 'Chinese (Mandarin, PRC)',
'jpJP': 'Japanese'
};
// select
media = media.reverse();
for(let m of media){
let selected = false;
if(m.id > 0 && m.type == 'Non-Encrypted'){
let dub_type = m.language;
let localSubs = []
let selUncut = !argv.simul && uncut[dub_type] && m.version.match(/uncut/i)
? true
: (!uncut[dub_type] || argv.simul && m.version.match(/simulcast/i) ? true : false);
if(dub_type == 'Japanese' && argv.sub && selUncut){
streamId = m.id;
stDlPath = m.subtitles;
selected = true;
for (let curDub of argv.dub) {
if(dub_type == dubType[curDub] && selUncut){
streamIds.push({
id: m.id,
lang: merger.getLanguageCode(curDub, curDub.slice(0, -2))
});
stDlPath.push(...m.subtitles);
localSubs = m.subtitles
selected = true;
}
}
else if(dub_type == dubType[argv.dub] && !argv.sub && selUncut){
streamId = m.id;
stDlPath = m.subtitles;
selected = true;
}
console.log(`[#${m.id}] ${dub_type} [${m.version}]`,(selected?'(selected)':''),(m.subtitles.subLangAvailable?'':'(defaulted to English subtitles)'));
console.log(`[#${m.id}] ${dub_type} [${m.version}]${(selected?' (selected)':'')}${
localSubs && localSubs.length > 0 && selected ? ` (using ${localSubs.map(a => `'${a.langName}'`).join(', ')} for subtitles)` : ''
}`);
}
}
if(streamId<1){
let already = []
stDlPath = stDlPath.filter(a => {
if (already.includes(a.language)) {
return false;
} else {
already.push(a.language)
return true;
}
})
if(streamIds.length <1){
console.log('[ERROR] Track not selected\n');
return;
}
else{
let streamData = await getData({
baseUrl: api_host,
url: `/source/catalog/video/${streamId}/signed`,
token: token,
dinstid: 'uuid',
useToken: true,
useProxy: true,
debug: argv.debug,
});
if(!streamData.ok){return;}
streamData = JSON.parse(streamData.res.body);
tsDlPath = false;
if(streamData.errors){
console.log('[ERROR] Error #%s: %s\n',streamData.errors[0].code,streamData.errors[0].detail);
return;
}
else{
for(let u in streamData.items){
if(streamData.items[u].videoType == 'm3u8'){
tsDlPath = streamData.items[u].src;
break;
tsDlPath = [];
for (let streamId of streamIds) {
let streamData = await getData({
baseUrl: api_host,
url: `/source/catalog/video/${streamId.id}/signed`,
token: token,
dinstid: 'uuid',
useToken: true,
useProxy: true,
debug: argv.debug,
});
if(!streamData.ok){return;}
streamData = JSON.parse(streamData.res.body);
if(streamData.errors){
console.log('[ERROR] Error #%s: %s\n',streamData.errors[0].code,streamData.errors[0].detail);
return;
}
else{
for(let u in streamData.items){
if(streamData.items[u].videoType == 'm3u8'){
tsDlPath.push({
path: streamData.items[u].src,
lang: streamId.lang
});
break;
}
}
}
}
if(!tsDlPath){
if(tsDlPath.length < 1){
console.log('[ERROR] Unknown error\n');
return;
}
@ -394,10 +419,10 @@ async function getEpisode(fnSlug){
function getSubsUrl(m){
if(argv.nosubs && !argv.sub){
return false;
return [];
}
let subLang = argv.subLang;
let subLangs = argv.subLang;
const subType = {
'enUS': 'English',
@ -405,264 +430,271 @@ function getSubsUrl(m){
'ptBR': 'Portuguese (Brazil)'
};
let subLangAvailable = m.some(a => {
return a.ext == 'vtt' && a.language === subType[subLang];
});
let subLangAvailable = m.some(a => subLangs.some(subLang => a.ext == 'vtt' && a.language === subType[subLang]));
if (!subLangAvailable) {
subLang = 'enUS';
subLangs = [ 'enUS' ];
}
let found = [];
for(let i in m){
let fpp = m[i].filePath.split('.');
let fpe = fpp[fpp.length-1];
if(fpe == 'vtt' && m[i].language === subType[subLang]) {
return {
path: m[i].filePath,
ext: `.${subLang}`,
langName: subType[subLang],
subLangAvailable: subLangAvailable
};
for (let lang of subLangs) {
if(fpe == 'vtt' && m[i].language === subType[lang]) {
found.push({
path: m[i].filePath,
ext: `.${lang}`,
langName: subType[lang],
language: m[i].languages[0].code ?? lang.slice(0, 2)
});
}
}
}
return false;
return found;
}
async function downloadStreams(){
// req playlist
let plQualityReq = await getData({
url: tsDlPath,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if(!plQualityReq.ok){return;}
let plQualityLinkList = m3u8(plQualityReq.res.body);
let mainServersList = [
'vmfst-api.prd.funimationsvc.com',
'd132fumi6di1wa.cloudfront.net',
'funiprod.akamaized.net',
];
let plServerList = [],
plStreams = {},
plLayersStr = [],
plLayersRes = {},
plMaxLayer = 1,
plNewIds = 1,
plAud = { uri: '' };
// new uris
let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/;
if(plQualityLinkList.playlists[0].uri.match(vplReg)){
if(plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128']){
let audioData = plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128'],
audioEl = Object.keys(audioData);
audioData = audioData[audioEl[0]];
plAud = { ...audioData, ...{ langStr: audioEl[0] } };
}
plQualityLinkList.playlists.sort((a, b) => {
let av = parseInt(a.uri.match(vplReg)[3]);
let bv = parseInt(b.uri.match(vplReg)[3]);
if(av > bv){
return 1;
}
if (av < bv) {
return -1;
}
return 0;
});
}
for(let s of plQualityLinkList.playlists){
if(s.uri.match(/_Layer(\d+)\.m3u8/) || s.uri.match(vplReg)){
// set layer and max layer
let plLayerId = 0;
if(s.uri.match(/_Layer(\d+)\.m3u8/)){
plLayerId = parseInt(s.uri.match(/_Layer(\d+)\.m3u8/)[1]);
}
else{
plLayerId = plNewIds, plNewIds++;
}
plMaxLayer = plMaxLayer < plLayerId ? plLayerId : plMaxLayer;
// set urls and servers
let plUrlDl = s.uri;
let plServer = new URL(plUrlDl).host;
if(!plServerList.includes(plServer)){
plServerList.push(plServer);
}
if(!Object.keys(plStreams).includes(plServer)){
plStreams[plServer] = {};
}
if(plStreams[plServer][plLayerId] && plStreams[plServer][plLayerId] != plUrlDl){
console.log(`[WARN] Non duplicate url for ${plServer} detected, please report to developer!`);
}
else{
plStreams[plServer][plLayerId] = plUrlDl;
}
// set plLayersStr
let plResolution = s.attributes.RESOLUTION;
plLayersRes[plLayerId] = plResolution;
let plBandwidth = Math.round(s.attributes.BANDWIDTH/1024);
if(plLayerId<10){
plLayerId = plLayerId.toString().padStart(2,' ');
}
let qualityStrAdd = `${plLayerId}: ${plResolution.width}x${plResolution.height} (${plBandwidth}KiB/s)`;
let qualityStrRegx = new RegExp(qualityStrAdd.replace(/(:|\(|\)|\/)/g,'\\$1'),'m');
let qualityStrMatch = !plLayersStr.join('\r\n').match(qualityStrRegx);
if(qualityStrMatch){
plLayersStr.push(qualityStrAdd);
}
}
else {
console.log(s.uri);
}
}
for(let s of mainServersList){
if(plServerList.includes(s)){
plServerList.splice(plServerList.indexOf(s), 1);
plServerList.unshift(s);
break;
}
}
if(typeof argv.q == 'object' && argv.q.length > 1){
argv.q = argv.q[argv.q.length-1];
}
argv.q = argv.q < 1 || argv.q > plMaxLayer ? plMaxLayer : argv.q;
let plSelectedServer = plServerList[argv.x-1];
let plSelectedList = plStreams[plSelectedServer];
let videoUrl = argv.x < plServerList.length+1 && plSelectedList[argv.q] ? plSelectedList[argv.q] : '';
plLayersStr.sort();
console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`);
console.log(`[INFO] Available qualities:\n\t${plLayersStr.join('\n\t')}`);
if(videoUrl != ''){
console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q].width}x${plLayersRes[argv.q].height}) @ ${plSelectedServer}`);
console.log('[INFO] Stream URL:',videoUrl);
fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height);
console.log(`[INFO] Output filename: ${fnOutput}.ts`);
}
else if(argv.x > plServerList.length){
console.log('[ERROR] Server not selected!\n');
return;
}
else{
console.log('[ERROR] Layer not selected!\n');
return;
}
let dlFailed = false;
let dlFailedA = false;
video: if (!argv.novids) {
// download video
let reqVideo = await getData({
url: videoUrl,
let purvideo = []
let puraudio = []
let audioAndVideo = []
let outName;
for (let streamPath of tsDlPath) {
let plQualityReq = await getData({
url: streamPath.path,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if (!reqVideo.ok) { break video; }
if(!plQualityReq.ok){return;}
let chunkList = m3u8(reqVideo.res.body);
let plQualityLinkList = m3u8(plQualityReq.res.body);
let tsFile = path.join(cfg.dir.content, fnOutput);
dlFailed = !await downloadFile(tsFile, chunkList);
}
else{
console.log('[INFO] Skip video downloading...\n');
}
let mainServersList = [
'vmfst-api.prd.funimationsvc.com',
'd132fumi6di1wa.cloudfront.net',
'funiprod.akamaized.net',
];
let plServerList = [],
plStreams = {},
plLayersStr = [],
plLayersRes = {},
plMaxLayer = 1,
plNewIds = 1,
plAud = { uri: '' };
// new uris
let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/;
if(plQualityLinkList.playlists[0].uri.match(vplReg)){
let audioKey = Object.keys(plQualityLinkList.mediaGroups.AUDIO).pop()
if(plQualityLinkList.mediaGroups.AUDIO[audioKey]){
let audioData = plQualityLinkList.mediaGroups.AUDIO[audioKey],
audioEl = Object.keys(audioData);
audioData = audioData[audioEl[0]];
plAud = { ...audioData, ...{ langStr: audioEl[0] } };
}
plQualityLinkList.playlists.sort((a, b) => {
let av = parseInt(a.uri.match(vplReg)[3]);
let bv = parseInt(b.uri.match(vplReg)[3]);
if(av > bv){
return 1;
}
if (av < bv) {
return -1;
}
return 0;
});
}
for(let s of plQualityLinkList.playlists){
if(s.uri.match(/_Layer(\d+)\.m3u8/) || s.uri.match(vplReg)){
// set layer and max layer
let plLayerId = 0;
if(s.uri.match(/_Layer(\d+)\.m3u8/)){
plLayerId = parseInt(s.uri.match(/_Layer(\d+)\.m3u8/)[1]);
}
else{
plLayerId = plNewIds, plNewIds++;
}
plMaxLayer = plMaxLayer < plLayerId ? plLayerId : plMaxLayer;
// set urls and servers
let plUrlDl = s.uri;
let plServer = new URL(plUrlDl).host;
if(!plServerList.includes(plServer)){
plServerList.push(plServer);
}
if(!Object.keys(plStreams).includes(plServer)){
plStreams[plServer] = {};
}
if(plStreams[plServer][plLayerId] && plStreams[plServer][plLayerId] != plUrlDl){
console.log(`[WARN] Non duplicate url for ${plServer} detected, please report to developer!`);
}
else{
plStreams[plServer][plLayerId] = plUrlDl;
}
// set plLayersStr
let plResolution = s.attributes.RESOLUTION;
plLayersRes[plLayerId] = plResolution;
let plBandwidth = Math.round(s.attributes.BANDWIDTH/1024);
if(plLayerId<10){
plLayerId = plLayerId.toString().padStart(2,' ');
}
let qualityStrAdd = `${plLayerId}: ${plResolution.width}x${plResolution.height} (${plBandwidth}KiB/s)`;
let qualityStrRegx = new RegExp(qualityStrAdd.replace(/(:|\(|\)|\/)/g,'\\$1'),'m');
let qualityStrMatch = !plLayersStr.join('\r\n').match(qualityStrRegx);
if(qualityStrMatch){
plLayersStr.push(qualityStrAdd);
}
}
else {
console.log(s.uri);
}
}
for(let s of mainServersList){
if(plServerList.includes(s)){
plServerList.splice(plServerList.indexOf(s), 1);
plServerList.unshift(s);
break;
}
}
if(typeof argv.q == 'object' && argv.q.length > 1){
argv.q = argv.q[argv.q.length-1];
}
argv.q = argv.q < 1 || argv.q > plMaxLayer ? plMaxLayer : argv.q;
let plSelectedServer = plServerList[argv.x-1];
let plSelectedList = plStreams[plSelectedServer];
let videoUrl = argv.x < plServerList.length+1 && plSelectedList[argv.q] ? plSelectedList[argv.q] : '';
plLayersStr.sort();
console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`);
console.log(`[INFO] Available qualities:\n\t${plLayersStr.join('\n\t')}`);
if(videoUrl != ''){
console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q].width}x${plLayersRes[argv.q].height}) @ ${plSelectedServer}`);
console.log('[INFO] Stream URL:',videoUrl);
if (!argv.noaudio && plAud.uri) {
// download audio
let reqAudio = await getData({
url: plAud.uri,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if (!reqAudio.ok) { return; }
fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height);
outName = fnOutput;
console.log(`[INFO] Output filename: ${fnOutput}.ts`);
}
else if(argv.x > plServerList.length){
console.log('[ERROR] Server not selected!\n');
return;
}
else{
console.log('[ERROR] Layer not selected!\n');
return;
}
let chunkListA = m3u8(reqAudio.res.body);
let dlFailed = false;
let dlFailedA = false;
video: if (!argv.novids) {
if (plAud.uri && (purvideo.length > 0 || audioAndVideo.length > 0)) {
break video;
} else if (!plAud.uri && (audioAndVideo.some(a => a.lang === streamPath.lang) || puraudio.some(a => a.lang === streamPath.lang))) {
break video;
}
// download video
let reqVideo = await getData({
url: videoUrl,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if (!reqVideo.ok) { break video; }
let chunkList = m3u8(reqVideo.res.body);
let tsFile = path.join(cfg.dir.content, `${fnOutput}.video${(plAud.uri ? '' : '.' + streamPath.lang )}`);
dlFailed = !await downloadFile(tsFile, chunkList);
if (!dlFailed) {
if (plAud.uri) {
purvideo.push({
path: `${tsFile}.ts`,
lang: plAud.language
})
} else {
audioAndVideo.push({
path: `${tsFile}.ts`,
lang: streamPath.lang
})
}
}
}
else{
console.log('[INFO] Skip video downloading...\n');
}
audio: if (!argv.noaudio && plAud.uri) {
// download audio
if (audioAndVideo.some(a => a.lang === plAud.language) || puraudio.some(a => a.lang === plAud.language))
break audio;
let reqAudio = await getData({
url: plAud.uri,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if (!reqAudio.ok) { return; }
let chunkListA = m3u8(reqAudio.res.body);
let tsFileA = path.join(cfg.dir.content, fnOutput + `.audio.${plAud.language}`);
dlFailedA = !await downloadFile(tsFileA, chunkListA);
if (!dlFailedA)
puraudio.push({
path: `${tsFileA}.ts`,
lang: plAud.language
})
let tsFileA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`);
dlFailedA = !await downloadFile(tsFileA, chunkListA);
}
}
// add subs
let subsUrl = stDlPath ? stDlPath.path : false;
let subsExt = !argv.mp4 || argv.mp4 && !argv.mks && argv.ass ? '.ass' : '.srt';
let addSubs = argv.mks && subsUrl ? true : false;
let subFile;
let subTrashFile;
let subsExt = !argv.mp4 || argv.mp4 && argv.ass ? '.ass' : '.srt';
let addSubs = true;
// download subtitles
if(subsUrl){
if(stDlPath.length > 0){
console.log('[INFO] Downloading subtitles...');
console.log(subsUrl);
let subsSrc = await getData({
url: subsUrl,
useProxy: true,
debug: argv.debug,
});
if(subsSrc.ok){
let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), stDlPath.langName, argv.fontSize);
subFile = path.join(cfg.dir.content, fnOutput) + stDlPath.ext + subsExt;
subTrashFile = path.join(cfg.dir.trash, fnOutput) + stDlPath.ext + subsExt;
fs.writeFileSync(subFile, assData);
for (let subObject of stDlPath) {
let subsSrc = await getData({
url: subObject.path,
useProxy: true,
debug: argv.debug,
});
if(subsSrc.ok){
let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.langName, argv.fontSize);
subObject.file = path.join(cfg.dir.content, `${fnOutput}.subtitle${subObject.ext}${subsExt}`)
fs.writeFileSync(subObject.file, assData);
}
else{
console.log('[ERROR] Failed to download subtitles!');
addSubs = false;
break;
}
}
if (addSubs)
console.log('[INFO] Subtitles downloaded!');
}
else{
console.log('[ERROR] Failed to download subtitles!');
addSubs = false;
}
}
if(dlFailed || dlFailedA){
console.log('\n[INFO] TS file not fully downloaded, skip muxing video...\n');
if((puraudio.length < 1 && audioAndVideo.length < 1) || (purvideo.length < 1 && audioAndVideo.length < 1)){
console.log('\n[INFO] Unable to locate a video AND audio file\n');
return;
}
if(argv.skipmux){
console.log("[INFO] Skipping muxing...")
return;
}
let muxTrg = path.join(cfg.dir.content, fnOutput);
let muxTrgA = '';
let tshTrg = path.join(cfg.dir.trash, fnOutput);
let tshTrgA = '';
if(!fs.existsSync(`${muxTrg}.ts`) || !fs.statSync(`${muxTrg}.ts`).isFile()){
console.log('\n[INFO] TS file not found, skip muxing video...\n');
return;
}
if(plAud.uri){
muxTrgA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`);
tshTrgA = path.join(cfg.dir.trash, fnOutput + `.${plAud.language}`);
if(!fs.existsSync(`${muxTrgA}.ts`) || !fs.statSync(`${muxTrgA}.ts`).isFile()){
console.log('\n[INFO] TS file not found, skip muxing video...\n');
return;
}
}
let langCode;
for (let lang in iso639.iso_639_2) {
let langObj = iso639.iso_639_2[lang];
if (langObj.hasOwnProperty('639-1') && langObj['639-1'] === plAud['language']) {
langCode = langObj['639-2'];
}
}
if (!langCode)
langCode = argv.sub ? 'jpn' : 'eng';
// usage
let usableMKVmerge = true;
@ -671,7 +703,7 @@ async function downloadStreams(){
// check exec path
let mkvmergebinfile = await lookpath(path.join(cfg.bin.mkvmerge));
let ffmpegbinfile = await lookpath(path.join(cfg.bin.ffmpeg));
// check exec
if( !argv.mp4 && !mkvmergebinfile ){
console.log('[WARN] MKVMerge not found, skip using this...');
@ -685,77 +717,26 @@ async function downloadStreams(){
console.log('[INFO] Video not downloaded. Skip muxing video.');
}
// select muxer
if(!argv.mp4 && usableMKVmerge){
// mux to mkv
let mkvmux = [];
mkvmux.push('-o',`${muxTrg}.mkv`);
mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data');
mkvmux.push('--track-name','0:[Funimation]');
if(plAud.uri){
mkvmux.push('--video-tracks','0','--no-audio');
mkvmux.push('--no-subtitles','--no-attachments');
mkvmux.push(`${muxTrg}.ts`);
mkvmux.push('--language',`0:${langCode}`);
mkvmux.push('--no-video','--audio-tracks','0');
mkvmux.push('--no-subtitles','--no-attachments');
mkvmux.push(`${muxTrgA}.ts`);
}
else{
mkvmux.push('--language',`1:${langCode}`);
mkvmux.push('--video-tracks','0','--audio-tracks','1');
mkvmux.push('--no-subtitles','--no-attachments');
mkvmux.push(`${muxTrg}.ts`);
}
if(addSubs){
mkvmux.push('--language','0:eng');
mkvmux.push(`${subFile ? subFile : muxTrg + subsExt}`);
}
fs.writeFileSync(`${muxTrg}.json`,JSON.stringify(mkvmux,null,' '));
shlp.exec('mkvmerge',`"${mkvmergebinfile}"`,`@"${muxTrg}.json"`);
fs.unlinkSync(`${muxTrg}.json`);
let ffext = !argv.mp4 ? 'mkv' : 'mp4';
let command = merger.buildCommandMkvMerge(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`);
console.log(command, audioAndVideo, puraudio, purvideo)
shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command);
}
else if(usableFFmpeg){
let ffext = !argv.mp4 ? 'mkv' : 'mp4';
let ffmux = `-i "${muxTrg}.ts" `;
ffmux += plAud.uri ? `-i "${muxTrgA}.ts" ` : '';
ffmux += addSubs ? `-i "${subFile ? subFile : muxTrg + subsExt}" ` : '';
ffmux += plAud.uri ? '-map 1:a ' : '';
ffmux += '-map 0 -c:v copy -c:a copy ';
ffmux += addSubs ? `-map ${plAud.uri ? 2 : 1} ` : '';
ffmux += addSubs && !argv.mp4 ? '-c:s ass ' : '';
ffmux += addSubs && argv.mp4 ? '-c:s mov_text ' : '';
ffmux += '-metadata encoding_tool="no_variable_data" ';
ffmux += `-metadata:s:v:0 title="[${argv.a}]" -metadata:s:a:${plAud.uri?1:0} language=${langCode} `;
ffmux += addSubs ? '-metadata:s:s:0 language=eng ' : '';
ffmux += `"${muxTrg}.${ffext}"`;
// mux to mkv
shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,ffmux);
let command = merger.buildCommandFFmpeg(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`);
shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,command);
}
else{
console.log('\n[INFO] Done!\n');
return;
}
if(argv.notrashfolder && argv.nocleanup){
// don't move or delete temp files
}
else if(argv.nocleanup){
fs.renameSync(muxTrg+'.ts', tshTrg + '.ts');
if (plAud.uri)
fs.renameSync(muxTrgA+'.ts', tshTrgA + '.ts');
if(subsUrl && addSubs){
fs.renameSync(subFile ? subFile : muxTrg+subsExt, subTrashFile ? subTrashFile : tshTrg + subsExt);
}
}
else{
fs.unlinkSync(muxTrg+'.ts');
if (plAud.uri)
fs.unlinkSync(muxTrgA+'.ts');
if(subsUrl && addSubs){
fs.unlinkSync(subFile ? subFile : muxTrg + subsExt);
}
}
if (argv.nocleanup)
return;
audioAndVideo.concat(puraudio).concat(purvideo).forEach(a => fs.unlinkSync(a.path))
stDlPath.forEach(subObject => fs.unlinkSync(subObject.file))
console.log('\n[INFO] Done!\n');
}

View file

@ -44,7 +44,6 @@ const nodeVer = 'node14-';
fs.mkdirSync(`${buildDir}/bin`);
fs.mkdirSync(`${buildDir}/config`);
fs.mkdirSync(`${buildDir}/videos`);
fs.mkdirSync(`${buildDir}/videos/_trash`);
fs.copySync('./bin/', `${buildDir}/bin/`);
fs.copySync('./config/bin-path.yml', `${buildDir}/config/bin-path.yml`);
fs.copySync('./config/cli-defaults.yml', `${buildDir}/config/cli-defaults.yml`);

154
modules/merger.js Normal file
View file

@ -0,0 +1,154 @@
const iso639 = require('iso-639');
const argv = require('../funi').argv;
/**
* @param {Array<object>} videoAndAudio
* @param {Array<object>} onlyVid
* @param {Array<object>} onlyAudio
* @param {Array<object>} subtitles
* @param {string} output
* @returns {string}
*/
const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAudio, subtitles, output) => {
let args = [];
let metaData = [];
let index = 0;
let hasVideo = false;
for (let vid of videoAndAudio) {
args.push(`-i "${vid.path}"`)
if (!hasVideo) {
metaData.push(`-map ${index}`)
metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`)
metaData.push(`-metadata:s:v:${index} title="[Funimation]"`)
hasVideo = true
} else {
metaData.push(`-map ${index}:a`)
metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`)
}
index++;
}
for (let vid of onlyVid) {
if (!hasVideo) {
args.push(`-i "${vid.path}"`)
metaData.push(`-map ${index}`)
metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`)
metaData.push(`-metadata:s:v:${index} title="[Funimation]"`)
hasVideo = true
index++;
}
}
for (let aud of onlyAudio) {
args.push(`-i "${aud.path}"`)
metaData.push(`-map ${index}`)
metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(aud.lang, aud.lang)}`)
index++;
}
for (let index in subtitles) {
let sub = subtitles[index];
args.push(`-i "${sub.file}"`);
}
args.push(...metaData)
args.push(...subtitles.map((_, subIndex) => `-map ${subIndex + index}`));
args.push(
'-c:v copy',
'-c:a copy'
);
args.push(output.split('.').pop().toLowerCase() === "mp4" ? '-c:s mov_text' : '*c:s ass')
args.push(...subtitles.map((sub, index) => `-metadata:s:${index + 2} language=${getLanguageCode(sub.language)}`));
args.push(`"${output}"`);
return args.join(' ');
};
/**
* @param {string} videoFile
* @param {object} audioFile
* @param {Array<object>} subtitles
* @returns {string}
*/
const buildCommandMkvMerge = (videoAndAudio, onlyVid, onlyAudio, subtitles, output) => {
let args = [];
let hasVideo = false;
args.push(`-o "${output}"`);
args.push(
'--no-date',
'--disable-track-statistics-tags',
'--engage no_variable_data',
);
for (let vid of videoAndAudio) {
if (!hasVideo) {
args.push(
'--video-tracks 0',
'--audio-tracks 1'
)
args.push(`--track-name 0:[Funimation]`)
args.push(`--language 1:${getLanguageCode(vid.lang, argv.todo ? 'jpn' : 'eng')}`);
hasVideo = true
} else {
args.push(
'--no-video',
'--audio-tracks 1'
)
args.push(`--language 1:${getLanguageCode(vid.lang, argv.todo ? 'jpn' : 'eng')}`);
}
args.push(`"${vid.path}"`)
}
for (let vid of onlyVid) {
if (!hasVideo) {
args.push(
'--video-tracks 0',
'--no-audio'
)
args.push(`--track-name 0:[Funimation]`)
hasVideo = true
args.push(`"${vid.path}"`)
}
}
for (let aud of onlyAudio) {
args.push(`--language 0:${getLanguageCode(aud.lang, argv.todo ? 'jpn' : 'eng')}`);
args.push(
'--no-video',
'--audio-tracks 0'
);
args.push(`"${aud.path}"`)
}
if(subtitles.length > 0){
for (let subObj of subtitles) {
args.push('--language',`0:${getLanguageCode(subObj.language)}`);
args.push(`"${subObj.file}"`);
}
} else {
args.push(
'--no-subtitles',
'--no-attachments'
);
}
return args.join(' ');
};
const getLanguageCode = (from, _default = 'eng') => {
for (let lang in iso639.iso_639_2) {
let langObj = iso639.iso_639_2[lang];
if (Object.prototype.hasOwnProperty.call(langObj, '639-1') && langObj['639-1'] === from) {
return langObj['639-2'];
}
}
return _default;
};
module.exports = {
buildCommandFFmpeg,
getLanguageCode,
buildCommandMkvMerge
};

View file

@ -11,8 +11,9 @@ const availableFilenameVars = [
const appArgv = (cfg) => {
// init
return yargs.parserConfiguration({
'duplicate-arguments-array': false,
const argv = yargs.parserConfiguration({
'duplicate-arguments-array': true,
"camel-case-expansion": false
})
// main
.wrap(Math.min(120)) // yargs.terminalWidth()
@ -73,22 +74,16 @@ const appArgv = (cfg) => {
.option('dub', {
group: 'Downloading:',
describe: 'Download non-Japanese Dub (English Dub mode by default)',
choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN' ],
choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN', 'jpJP' ],
default: cfg.dub || 'enUS',
type: 'string',
})
.option('sub', {
group: 'Downloading:',
describe: 'Japanese Dub with subtitles mode (English Dub mode by default)',
default: cfg.subsMode || false,
type: 'boolean',
type: 'array',
})
.option('subLang', {
group: 'Downloading:',
describe: 'Set the subtitle language (English is default and fallback)',
default: cfg.subLang || 'enUS',
choices: [ 'enUS', 'esLA', 'ptBR' ],
type: 'string'
type: 'array'
})
.option('fontSize', {
group: 'Downloading:',
@ -128,6 +123,7 @@ const appArgv = (cfg) => {
group: 'Downloading:',
describe: 'Skip downloading subtitles for English Dub (if available)',
type: 'boolean',
default: false
})
// proxy
.option('proxy', {
@ -161,12 +157,6 @@ const appArgv = (cfg) => {
default: cfg.mp4mux || false,
type: 'boolean'
})
.option('mks', {
group: 'Muxing:',
describe: 'Add subtitles to mkv/mp4 (if available)',
default: cfg.muxSubs || false,
type: 'boolean'
})
// filenaming
.option('fileName', {
group: 'Filename Template:',
@ -185,17 +175,10 @@ const appArgv = (cfg) => {
// util
.option('nocleanup', {
group: 'Utilities:',
describe: 'Move temporary files to trash folder instead of deleting',
describe: 'Dont\'t delete the input files after muxing',
default: cfg.noCleanUp || false,
type: 'boolean'
})
.option('notrashfolder', {
implies: ['nocleanup'],
group: 'Utilities:',
describe: 'Don\'t move temporary files to trash folder (Used with --nocleanup)',
default: cfg.noTrashFolder || false,
type: 'boolean'
})
// help
.option('help', {
alias: 'h',
@ -212,6 +195,14 @@ const appArgv = (cfg) => {
// --
.argv;
// Resolve unwanted arrays
for (let key in argv) {
if (argv[key] instanceof Array && !(key === "subLang" || key === "dub")) {
argv[key] = argv[key].pop()
}
}
return argv;
};
const showHelp = yargs.showHelp;

View file

@ -1,7 +1,7 @@
{
"name": "funimation-downloader-nx",
"short_name": "funi",
"version": "4.8.3",
"version": "4.9.0",
"description": "Download videos from Funimation via cli.",
"keywords": [
"download",

View file