Creepso/multi-downloader-nx
1
0
Fork 0
Code Issues 38 Pull requests 10 Projects Releases 70 Packages Wiki Activity Actions

Update downloader to handle multiple subtitles and audio languages in the same run

Browse source
This commit is contained in:
Izuco 2021-07-03 21:13:43 +02:00 committed by GitHub
commit 6e4a9371d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 493 additions and 364 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
config/bin-path.yml
View file

@ -1,2 +1,2 @@
ffmpeg: "./bin/ffmpeg/ffmpeg.exe"
ffmpeg: "C:\\Program Files\\ffmpeg\\bin\\ffmpeg.exe"
mkvmerge: "./bin/mkvtoolnix/mkvmerge.exe"

1
config/dir-path.yml
View file

@ -1,2 +1 @@
content: ./videos/
trash: ./videos/_trash/

2
docs/CHANGELOG.md
View file

@ -1,5 +1,7 @@
## Change Log
This changelog is out of date and wont be continued. Please see the releases comments, or if not present the commit comments.
### 4.7.0 (unreleased)
- Change subtitles parser from ttml to vtt
- Improve help command

19
docs/README.md
View file

@ -42,15 +42,20 @@ After installing NodeJS with NPM go to directory with `package.json` file and ty
* `-s <i> -e <s>` sets the show id and episode ids (comma-separated, hyphen-sequence)
* `-q <i>` sets the video layer quality [1...10] (optional, 0 is max)
* `--all` download all videos at once
* `--alt` alternative episode listing (if available)
* `--sub` switch from English dub to Japanese dub with subtitles
* `--sub` select one or more subtile language
* `--dub` select one or more dub languages
* `--simul` force select simulcast version instead of uncut version
* `-x` select server
* `--novids` skip download videos (for downloading subtitles only)
* `--novids` skip download videos
* `--nosubs` skip download subtitles for Dub (if available)
* `--noaudio` skip downloading audio
### Proxy
The proxy is currently unmainted. Use at your on risk.
* `--proxy <s>` http(s)/socks proxy WHATWG url (ex. https://myproxyhost:1080)
* `--proxy-auth <s>` Colon-separated username and password for proxy
* `--ssp` don't use proxy for stream downloading
@ -63,19 +68,17 @@ After installing NodeJS with NPM go to directory with `package.json` file and ty
### Filenaming (optional)
* `-a <s>` release group ("Funimation" by default)
* `-t <s>` show title override
* `--ep <s>` episode number override (ignored in batch mode)
* `--suffix <s>` filename suffix override (first "SIZEp" will be replaced with actual video size, "SIZEp" by default)
* `--fileName` Set the filename template. Use ${variable_name} to insert variables.
You may use 'title', 'episode', 'showTitle', 'season', 'width', 'height' as variables.
### Utility
* `--nocleanup` move unnecessary files to trash folder after completion instead of deleting
* `--nocleanup` don't delete the input files after the muxing
* `-h`, `--help` show all options
## Filename Template
[`release group`] `title` - `episode` [`suffix`].`extension`
[Funimation] ${showTitle} - ${episode} [${height}p]"]
## CLI Examples

635
funi.js
View file

@ -18,7 +18,6 @@ const { lookpath } = require('lookpath');
const m3u8 = require('m3u8-parsed');
const crypto = require('crypto');
const got = require('got');
const iso639 = require('iso-639');
// extra
const appYargs = require('./modules/module.app-args');
@ -41,13 +40,13 @@ let cfg = {
/* Normalise paths for use outside the current directory */
for (let key of Object.keys(cfg.dir)) {
if (!path.isAbsolute(cfg.dir[key])) {
cfg.dir[key] = path.join(workingDir, cfg.dir[key])
cfg.dir[key] = path.join(workingDir, cfg.dir[key]);
}
}
for (let key of Object.keys(cfg.bin)) {
if (!path.isAbsolute(cfg.bin[key])) {
cfg.bin[key] = path.join(workingDir, cfg.bin[key])
cfg.bin[key] = path.join(workingDir, cfg.bin[key]);
}
}
@ -62,6 +61,14 @@ if(!token){
// cli
const argv = appYargs.appArgv(cfg.cli);
module.exports = {
argv,
cfg
};
// Import merger after argv has been exported
const merger = require('./modules/merger');
// check page
if(!isNaN(parseInt(argv.p, 10)) && parseInt(argv.p, 10) > 0){
@ -77,8 +84,8 @@ let title = '',
fnEpNum = 0,
fnOutput = '',
season = 0,
tsDlPath = false,
stDlPath = undefined;
tsDlPath = [],
stDlPath = [];
// select mode
if(argv.auth){
@ -277,7 +284,7 @@ async function getEpisode(fnSlug){
debug: argv.debug,
});
if(!episodeData.ok){return;}
let ep = JSON.parse(episodeData.res.body).items[0], streamId = 0;
let ep = JSON.parse(episodeData.res.body).items[0], streamIds = [];
// build fn
showTitle = ep.parent.title;
title = ep.title;
@ -327,62 +334,80 @@ async function getEpisode(fnSlug){
'enUS': 'English',
'esLA': 'Spanish (Latin Am)',
'ptBR': 'Portuguese (Brazil)',
'zhMN': 'Chinese (Mandarin, PRC)'
'zhMN': 'Chinese (Mandarin, PRC)',
'jpJP': 'Japanese'
};
// select
media = media.reverse();
for(let m of media){
let selected = false;
if(m.id > 0 && m.type == 'Non-Encrypted'){
let dub_type = m.language;
let localSubs = []
let selUncut = !argv.simul && uncut[dub_type] && m.version.match(/uncut/i)
? true
: (!uncut[dub_type] || argv.simul && m.version.match(/simulcast/i) ? true : false);
if(dub_type == 'Japanese' && argv.sub && selUncut){
streamId = m.id;
stDlPath = m.subtitles;
selected = true;
for (let curDub of argv.dub) {
if(dub_type == dubType[curDub] && selUncut){
streamIds.push({
id: m.id,
lang: merger.getLanguageCode(curDub, curDub.slice(0, -2))
});
stDlPath.push(...m.subtitles);
localSubs = m.subtitles
selected = true;
}
}
else if(dub_type == dubType[argv.dub] && !argv.sub && selUncut){
streamId = m.id;
stDlPath = m.subtitles;
selected = true;
}
console.log(`[#${m.id}] ${dub_type} [${m.version}]`,(selected?'(selected)':''),(m.subtitles.subLangAvailable?'':'(defaulted to English subtitles)'));
console.log(`[#${m.id}] ${dub_type} [${m.version}]${(selected?' (selected)':'')}${
localSubs && localSubs.length > 0 && selected ? ` (using ${localSubs.map(a => `'${a.langName}'`).join(', ')} for subtitles)` : ''
}`);
}
}
if(streamId<1){
let already = []
stDlPath = stDlPath.filter(a => {
if (already.includes(a.language)) {
return false;
} else {
already.push(a.language)
return true;
}
})
if(streamIds.length <1){
console.log('[ERROR] Track not selected\n');
return;
}
else{
let streamData = await getData({
baseUrl: api_host,
url: `/source/catalog/video/${streamId}/signed`,
token: token,
dinstid: 'uuid',
useToken: true,
useProxy: true,
debug: argv.debug,
});
if(!streamData.ok){return;}
streamData = JSON.parse(streamData.res.body);
tsDlPath = false;
if(streamData.errors){
console.log('[ERROR] Error #%s: %s\n',streamData.errors[0].code,streamData.errors[0].detail);
return;
}
else{
for(let u in streamData.items){
if(streamData.items[u].videoType == 'm3u8'){
tsDlPath = streamData.items[u].src;
break;
tsDlPath = [];
for (let streamId of streamIds) {
let streamData = await getData({
baseUrl: api_host,
url: `/source/catalog/video/${streamId.id}/signed`,
token: token,
dinstid: 'uuid',
useToken: true,
useProxy: true,
debug: argv.debug,
});
if(!streamData.ok){return;}
streamData = JSON.parse(streamData.res.body);
if(streamData.errors){
console.log('[ERROR] Error #%s: %s\n',streamData.errors[0].code,streamData.errors[0].detail);
return;
}
else{
for(let u in streamData.items){
if(streamData.items[u].videoType == 'm3u8'){
tsDlPath.push({
path: streamData.items[u].src,
lang: streamId.lang
});
break;
}
}
}
}
if(!tsDlPath){
if(tsDlPath.length < 1){
console.log('[ERROR] Unknown error\n');
return;
}
@ -394,10 +419,10 @@ async function getEpisode(fnSlug){
function getSubsUrl(m){
if(argv.nosubs && !argv.sub){
return false;
return [];
}
let subLang = argv.subLang;
let subLangs = argv.subLang;
const subType = {
'enUS': 'English',
@ -405,264 +430,271 @@ function getSubsUrl(m){
'ptBR': 'Portuguese (Brazil)'
};
let subLangAvailable = m.some(a => {
return a.ext == 'vtt' && a.language === subType[subLang];
});
let subLangAvailable = m.some(a => subLangs.some(subLang => a.ext == 'vtt' && a.language === subType[subLang]));
if (!subLangAvailable) {
subLang = 'enUS';
subLangs = [ 'enUS' ];
}
let found = [];
for(let i in m){
let fpp = m[i].filePath.split('.');
let fpe = fpp[fpp.length-1];
if(fpe == 'vtt' && m[i].language === subType[subLang]) {
return {
path: m[i].filePath,
ext: `.${subLang}`,
langName: subType[subLang],
subLangAvailable: subLangAvailable
};
for (let lang of subLangs) {
if(fpe == 'vtt' && m[i].language === subType[lang]) {
found.push({
path: m[i].filePath,
ext: `.${lang}`,
langName: subType[lang],
language: m[i].languages[0].code ?? lang.slice(0, 2)
});
}
}
}
return false;
return found;
}
async function downloadStreams(){
// req playlist
let plQualityReq = await getData({
url: tsDlPath,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if(!plQualityReq.ok){return;}
let plQualityLinkList = m3u8(plQualityReq.res.body);
let mainServersList = [
'vmfst-api.prd.funimationsvc.com',
'd132fumi6di1wa.cloudfront.net',
'funiprod.akamaized.net',
];
let plServerList = [],
plStreams = {},
plLayersStr = [],
plLayersRes = {},
plMaxLayer = 1,
plNewIds = 1,
plAud = { uri: '' };
// new uris
let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/;
if(plQualityLinkList.playlists[0].uri.match(vplReg)){
if(plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128']){
let audioData = plQualityLinkList.mediaGroups.AUDIO['audio-aacl-128'],
audioEl = Object.keys(audioData);
audioData = audioData[audioEl[0]];
plAud = { ...audioData, ...{ langStr: audioEl[0] } };
}
plQualityLinkList.playlists.sort((a, b) => {
let av = parseInt(a.uri.match(vplReg)[3]);
let bv = parseInt(b.uri.match(vplReg)[3]);
if(av > bv){
return 1;
}
if (av < bv) {
return -1;
}
return 0;
});
}
for(let s of plQualityLinkList.playlists){
if(s.uri.match(/_Layer(\d+)\.m3u8/) || s.uri.match(vplReg)){
// set layer and max layer
let plLayerId = 0;
if(s.uri.match(/_Layer(\d+)\.m3u8/)){
plLayerId = parseInt(s.uri.match(/_Layer(\d+)\.m3u8/)[1]);
}
else{
plLayerId = plNewIds, plNewIds++;
}
plMaxLayer = plMaxLayer < plLayerId ? plLayerId : plMaxLayer;
// set urls and servers
let plUrlDl = s.uri;
let plServer = new URL(plUrlDl).host;
if(!plServerList.includes(plServer)){
plServerList.push(plServer);
}
if(!Object.keys(plStreams).includes(plServer)){
plStreams[plServer] = {};
}
if(plStreams[plServer][plLayerId] && plStreams[plServer][plLayerId] != plUrlDl){
console.log(`[WARN] Non duplicate url for ${plServer} detected, please report to developer!`);
}
else{
plStreams[plServer][plLayerId] = plUrlDl;
}
// set plLayersStr
let plResolution = s.attributes.RESOLUTION;
plLayersRes[plLayerId] = plResolution;
let plBandwidth = Math.round(s.attributes.BANDWIDTH/1024);
if(plLayerId<10){
plLayerId = plLayerId.toString().padStart(2,' ');
}
let qualityStrAdd = `${plLayerId}: ${plResolution.width}x${plResolution.height} (${plBandwidth}KiB/s)`;
let qualityStrRegx = new RegExp(qualityStrAdd.replace(/(:|\(|\)|\/)/g,'\\$1'),'m');
let qualityStrMatch = !plLayersStr.join('\r\n').match(qualityStrRegx);
if(qualityStrMatch){
plLayersStr.push(qualityStrAdd);
}
}
else {
console.log(s.uri);
}
}
for(let s of mainServersList){
if(plServerList.includes(s)){
plServerList.splice(plServerList.indexOf(s), 1);
plServerList.unshift(s);
break;
}
}
if(typeof argv.q == 'object' && argv.q.length > 1){
argv.q = argv.q[argv.q.length-1];
}
argv.q = argv.q < 1 || argv.q > plMaxLayer ? plMaxLayer : argv.q;
let plSelectedServer = plServerList[argv.x-1];
let plSelectedList = plStreams[plSelectedServer];
let videoUrl = argv.x < plServerList.length+1 && plSelectedList[argv.q] ? plSelectedList[argv.q] : '';
plLayersStr.sort();
console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`);
console.log(`[INFO] Available qualities:\n\t${plLayersStr.join('\n\t')}`);
if(videoUrl != ''){
console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q].width}x${plLayersRes[argv.q].height}) @ ${plSelectedServer}`);
console.log('[INFO] Stream URL:',videoUrl);
fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height);
console.log(`[INFO] Output filename: ${fnOutput}.ts`);
}
else if(argv.x > plServerList.length){
console.log('[ERROR] Server not selected!\n');
return;
}
else{
console.log('[ERROR] Layer not selected!\n');
return;
}
let dlFailed = false;
let dlFailedA = false;
video: if (!argv.novids) {
// download video
let reqVideo = await getData({
url: videoUrl,
let purvideo = []
let puraudio = []
let audioAndVideo = []
let outName;
for (let streamPath of tsDlPath) {
let plQualityReq = await getData({
url: streamPath.path,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if (!reqVideo.ok) { break video; }
if(!plQualityReq.ok){return;}
let chunkList = m3u8(reqVideo.res.body);
let plQualityLinkList = m3u8(plQualityReq.res.body);
let tsFile = path.join(cfg.dir.content, fnOutput);
dlFailed = !await downloadFile(tsFile, chunkList);
}
else{
console.log('[INFO] Skip video downloading...\n');
}
let mainServersList = [
'vmfst-api.prd.funimationsvc.com',
'd132fumi6di1wa.cloudfront.net',
'funiprod.akamaized.net',
];
let plServerList = [],
plStreams = {},
plLayersStr = [],
plLayersRes = {},
plMaxLayer = 1,
plNewIds = 1,
plAud = { uri: '' };
// new uris
let vplReg = /streaming_video_(\d+)_(\d+)_(\d+)_index\.m3u8/;
if(plQualityLinkList.playlists[0].uri.match(vplReg)){
let audioKey = Object.keys(plQualityLinkList.mediaGroups.AUDIO).pop()
if(plQualityLinkList.mediaGroups.AUDIO[audioKey]){
let audioData = plQualityLinkList.mediaGroups.AUDIO[audioKey],
audioEl = Object.keys(audioData);
audioData = audioData[audioEl[0]];
plAud = { ...audioData, ...{ langStr: audioEl[0] } };
}
plQualityLinkList.playlists.sort((a, b) => {
let av = parseInt(a.uri.match(vplReg)[3]);
let bv = parseInt(b.uri.match(vplReg)[3]);
if(av > bv){
return 1;
}
if (av < bv) {
return -1;
}
return 0;
});
}
for(let s of plQualityLinkList.playlists){
if(s.uri.match(/_Layer(\d+)\.m3u8/) || s.uri.match(vplReg)){
// set layer and max layer
let plLayerId = 0;
if(s.uri.match(/_Layer(\d+)\.m3u8/)){
plLayerId = parseInt(s.uri.match(/_Layer(\d+)\.m3u8/)[1]);
}
else{
plLayerId = plNewIds, plNewIds++;
}
plMaxLayer = plMaxLayer < plLayerId ? plLayerId : plMaxLayer;
// set urls and servers
let plUrlDl = s.uri;
let plServer = new URL(plUrlDl).host;
if(!plServerList.includes(plServer)){
plServerList.push(plServer);
}
if(!Object.keys(plStreams).includes(plServer)){
plStreams[plServer] = {};
}
if(plStreams[plServer][plLayerId] && plStreams[plServer][plLayerId] != plUrlDl){
console.log(`[WARN] Non duplicate url for ${plServer} detected, please report to developer!`);
}
else{
plStreams[plServer][plLayerId] = plUrlDl;
}
// set plLayersStr
let plResolution = s.attributes.RESOLUTION;
plLayersRes[plLayerId] = plResolution;
let plBandwidth = Math.round(s.attributes.BANDWIDTH/1024);
if(plLayerId<10){
plLayerId = plLayerId.toString().padStart(2,' ');
}
let qualityStrAdd = `${plLayerId}: ${plResolution.width}x${plResolution.height} (${plBandwidth}KiB/s)`;
let qualityStrRegx = new RegExp(qualityStrAdd.replace(/(:|\(|\)|\/)/g,'\\$1'),'m');
let qualityStrMatch = !plLayersStr.join('\r\n').match(qualityStrRegx);
if(qualityStrMatch){
plLayersStr.push(qualityStrAdd);
}
}
else {
console.log(s.uri);
}
}
for(let s of mainServersList){
if(plServerList.includes(s)){
plServerList.splice(plServerList.indexOf(s), 1);
plServerList.unshift(s);
break;
}
}
if(typeof argv.q == 'object' && argv.q.length > 1){
argv.q = argv.q[argv.q.length-1];
}
argv.q = argv.q < 1 || argv.q > plMaxLayer ? plMaxLayer : argv.q;
let plSelectedServer = plServerList[argv.x-1];
let plSelectedList = plStreams[plSelectedServer];
let videoUrl = argv.x < plServerList.length+1 && plSelectedList[argv.q] ? plSelectedList[argv.q] : '';
plLayersStr.sort();
console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`);
console.log(`[INFO] Available qualities:\n\t${plLayersStr.join('\n\t')}`);
if(videoUrl != ''){
console.log(`[INFO] Selected layer: ${argv.q} (${plLayersRes[argv.q].width}x${plLayersRes[argv.q].height}) @ ${plSelectedServer}`);
console.log('[INFO] Stream URL:',videoUrl);
if (!argv.noaudio && plAud.uri) {
// download audio
let reqAudio = await getData({
url: plAud.uri,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if (!reqAudio.ok) { return; }
fnOutput = parseFileName(argv.fileName, title, fnEpNum, showTitle, season, plLayersRes[argv.q].width, plLayersRes[argv.q].height);
outName = fnOutput;
console.log(`[INFO] Output filename: ${fnOutput}.ts`);
}
else if(argv.x > plServerList.length){
console.log('[ERROR] Server not selected!\n');
return;
}
else{
console.log('[ERROR] Layer not selected!\n');
return;
}
let chunkListA = m3u8(reqAudio.res.body);
let dlFailed = false;
let dlFailedA = false;
video: if (!argv.novids) {
if (plAud.uri && (purvideo.length > 0 || audioAndVideo.length > 0)) {
break video;
} else if (!plAud.uri && (audioAndVideo.some(a => a.lang === streamPath.lang) || puraudio.some(a => a.lang === streamPath.lang))) {
break video;
}
// download video
let reqVideo = await getData({
url: videoUrl,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if (!reqVideo.ok) { break video; }
let chunkList = m3u8(reqVideo.res.body);
let tsFile = path.join(cfg.dir.content, `${fnOutput}.video${(plAud.uri ? '' : '.' + streamPath.lang )}`);
dlFailed = !await downloadFile(tsFile, chunkList);
if (!dlFailed) {
if (plAud.uri) {
purvideo.push({
path: `${tsFile}.ts`,
lang: plAud.language
})
} else {
audioAndVideo.push({
path: `${tsFile}.ts`,
lang: streamPath.lang
})
}
}
}
else{
console.log('[INFO] Skip video downloading...\n');
}
audio: if (!argv.noaudio && plAud.uri) {
// download audio
if (audioAndVideo.some(a => a.lang === plAud.language) || puraudio.some(a => a.lang === plAud.language))
break audio;
let reqAudio = await getData({
url: plAud.uri,
useProxy: (argv.ssp ? false : true),
debug: argv.debug,
});
if (!reqAudio.ok) { return; }
let chunkListA = m3u8(reqAudio.res.body);
let tsFileA = path.join(cfg.dir.content, fnOutput + `.audio.${plAud.language}`);
dlFailedA = !await downloadFile(tsFileA, chunkListA);
if (!dlFailedA)
puraudio.push({
path: `${tsFileA}.ts`,
lang: plAud.language
})
let tsFileA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`);
dlFailedA = !await downloadFile(tsFileA, chunkListA);
}
}
// add subs
let subsUrl = stDlPath ? stDlPath.path : false;
let subsExt = !argv.mp4 || argv.mp4 && !argv.mks && argv.ass ? '.ass' : '.srt';
let addSubs = argv.mks && subsUrl ? true : false;
let subFile;
let subTrashFile;
let subsExt = !argv.mp4 || argv.mp4 && argv.ass ? '.ass' : '.srt';
let addSubs = true;
// download subtitles
if(subsUrl){
if(stDlPath.length > 0){
console.log('[INFO] Downloading subtitles...');
console.log(subsUrl);
let subsSrc = await getData({
url: subsUrl,
useProxy: true,
debug: argv.debug,
});
if(subsSrc.ok){
let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), stDlPath.langName, argv.fontSize);
subFile = path.join(cfg.dir.content, fnOutput) + stDlPath.ext + subsExt;
subTrashFile = path.join(cfg.dir.trash, fnOutput) + stDlPath.ext + subsExt;
fs.writeFileSync(subFile, assData);
for (let subObject of stDlPath) {
let subsSrc = await getData({
url: subObject.path,
useProxy: true,
debug: argv.debug,
});
if(subsSrc.ok){
let assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.langName, argv.fontSize);
subObject.file = path.join(cfg.dir.content, `${fnOutput}.subtitle${subObject.ext}${subsExt}`)
fs.writeFileSync(subObject.file, assData);
}
else{
console.log('[ERROR] Failed to download subtitles!');
addSubs = false;
break;
}
}
if (addSubs)
console.log('[INFO] Subtitles downloaded!');
}
else{
console.log('[ERROR] Failed to download subtitles!');
addSubs = false;
}
}
if(dlFailed || dlFailedA){
console.log('\n[INFO] TS file not fully downloaded, skip muxing video...\n');
if((puraudio.length < 1 && audioAndVideo.length < 1) || (purvideo.length < 1 && audioAndVideo.length < 1)){
console.log('\n[INFO] Unable to locate a video AND audio file\n');
return;
}
if(argv.skipmux){
console.log("[INFO] Skipping muxing...")
return;
}
let muxTrg = path.join(cfg.dir.content, fnOutput);
let muxTrgA = '';
let tshTrg = path.join(cfg.dir.trash, fnOutput);
let tshTrgA = '';
if(!fs.existsSync(`${muxTrg}.ts`) || !fs.statSync(`${muxTrg}.ts`).isFile()){
console.log('\n[INFO] TS file not found, skip muxing video...\n');
return;
}
if(plAud.uri){
muxTrgA = path.join(cfg.dir.content, fnOutput + `.${plAud.language}`);
tshTrgA = path.join(cfg.dir.trash, fnOutput + `.${plAud.language}`);
if(!fs.existsSync(`${muxTrgA}.ts`) || !fs.statSync(`${muxTrgA}.ts`).isFile()){
console.log('\n[INFO] TS file not found, skip muxing video...\n');
return;
}
}
let langCode;
for (let lang in iso639.iso_639_2) {
let langObj = iso639.iso_639_2[lang];
if (langObj.hasOwnProperty('639-1') && langObj['639-1'] === plAud['language']) {
langCode = langObj['639-2'];
}
}
if (!langCode)
langCode = argv.sub ? 'jpn' : 'eng';
// usage
let usableMKVmerge = true;
@ -671,7 +703,7 @@ async function downloadStreams(){
// check exec path
let mkvmergebinfile = await lookpath(path.join(cfg.bin.mkvmerge));
let ffmpegbinfile = await lookpath(path.join(cfg.bin.ffmpeg));
// check exec
if( !argv.mp4 && !mkvmergebinfile ){
console.log('[WARN] MKVMerge not found, skip using this...');
@ -685,77 +717,26 @@ async function downloadStreams(){
console.log('[INFO] Video not downloaded. Skip muxing video.');
}
// select muxer
if(!argv.mp4 && usableMKVmerge){
// mux to mkv
let mkvmux = [];
mkvmux.push('-o',`${muxTrg}.mkv`);
mkvmux.push('--no-date','--disable-track-statistics-tags','--engage','no_variable_data');
mkvmux.push('--track-name','0:[Funimation]');
if(plAud.uri){
mkvmux.push('--video-tracks','0','--no-audio');
mkvmux.push('--no-subtitles','--no-attachments');
mkvmux.push(`${muxTrg}.ts`);
mkvmux.push('--language',`0:${langCode}`);
mkvmux.push('--no-video','--audio-tracks','0');
mkvmux.push('--no-subtitles','--no-attachments');
mkvmux.push(`${muxTrgA}.ts`);
}
else{
mkvmux.push('--language',`1:${langCode}`);
mkvmux.push('--video-tracks','0','--audio-tracks','1');
mkvmux.push('--no-subtitles','--no-attachments');
mkvmux.push(`${muxTrg}.ts`);
}
if(addSubs){
mkvmux.push('--language','0:eng');
mkvmux.push(`${subFile ? subFile : muxTrg + subsExt}`);
}
fs.writeFileSync(`${muxTrg}.json`,JSON.stringify(mkvmux,null,' '));
shlp.exec('mkvmerge',`"${mkvmergebinfile}"`,`@"${muxTrg}.json"`);
fs.unlinkSync(`${muxTrg}.json`);
let ffext = !argv.mp4 ? 'mkv' : 'mp4';
let command = merger.buildCommandMkvMerge(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`);
console.log(command, audioAndVideo, puraudio, purvideo)
shlp.exec('mkvmerge', `"${mkvmergebinfile}"`, command);
}
else if(usableFFmpeg){
let ffext = !argv.mp4 ? 'mkv' : 'mp4';
let ffmux = `-i "${muxTrg}.ts" `;
ffmux += plAud.uri ? `-i "${muxTrgA}.ts" ` : '';
ffmux += addSubs ? `-i "${subFile ? subFile : muxTrg + subsExt}" ` : '';
ffmux += plAud.uri ? '-map 1:a ' : '';
ffmux += '-map 0 -c:v copy -c:a copy ';
ffmux += addSubs ? `-map ${plAud.uri ? 2 : 1} ` : '';
ffmux += addSubs && !argv.mp4 ? '-c:s ass ' : '';
ffmux += addSubs && argv.mp4 ? '-c:s mov_text ' : '';
ffmux += '-metadata encoding_tool="no_variable_data" ';
ffmux += `-metadata:s:v:0 title="[${argv.a}]" -metadata:s:a:${plAud.uri?1:0} language=${langCode} `;
ffmux += addSubs ? '-metadata:s:s:0 language=eng ' : '';
ffmux += `"${muxTrg}.${ffext}"`;
// mux to mkv
shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,ffmux);
let command = merger.buildCommandFFmpeg(audioAndVideo, purvideo, puraudio, stDlPath, `${path.join(cfg.dir.content, outName)}.${ffext}`);
shlp.exec('ffmpeg',`"${ffmpegbinfile}"`,command);
}
else{
console.log('\n[INFO] Done!\n');
return;
}
if(argv.notrashfolder && argv.nocleanup){
// don't move or delete temp files
}
else if(argv.nocleanup){
fs.renameSync(muxTrg+'.ts', tshTrg + '.ts');
if (plAud.uri)
fs.renameSync(muxTrgA+'.ts', tshTrgA + '.ts');
if(subsUrl && addSubs){
fs.renameSync(subFile ? subFile : muxTrg+subsExt, subTrashFile ? subTrashFile : tshTrg + subsExt);
}
}
else{
fs.unlinkSync(muxTrg+'.ts');
if (plAud.uri)
fs.unlinkSync(muxTrgA+'.ts');
if(subsUrl && addSubs){
fs.unlinkSync(subFile ? subFile : muxTrg + subsExt);
}
}
if (argv.nocleanup)
return;
audioAndVideo.concat(puraudio).concat(purvideo).forEach(a => fs.unlinkSync(a.path))
stDlPath.forEach(subObject => fs.unlinkSync(subObject.file))
console.log('\n[INFO] Done!\n');
}

1
modules/build.js
View file

@ -44,7 +44,6 @@ const nodeVer = 'node14-';
fs.mkdirSync(`${buildDir}/bin`);
fs.mkdirSync(`${buildDir}/config`);
fs.mkdirSync(`${buildDir}/videos`);
fs.mkdirSync(`${buildDir}/videos/_trash`);
fs.copySync('./bin/', `${buildDir}/bin/`);
fs.copySync('./config/bin-path.yml', `${buildDir}/config/bin-path.yml`);
fs.copySync('./config/cli-defaults.yml', `${buildDir}/config/cli-defaults.yml`);

154
modules/merger.js Normal file
View file

@ -0,0 +1,154 @@
const iso639 = require('iso-639');
const argv = require('../funi').argv;
/**
* @param {Array<object>} videoAndAudio
* @param {Array<object>} onlyVid
* @param {Array<object>} onlyAudio
* @param {Array<object>} subtitles
* @param {string} output
* @returns {string}
*/
const buildCommandFFmpeg = (videoAndAudio, onlyVid, onlyAudio, subtitles, output) => {
let args = [];
let metaData = [];
let index = 0;
let hasVideo = false;
for (let vid of videoAndAudio) {
args.push(`-i "${vid.path}"`)
if (!hasVideo) {
metaData.push(`-map ${index}`)
metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`)
metaData.push(`-metadata:s:v:${index} title="[Funimation]"`)
hasVideo = true
} else {
metaData.push(`-map ${index}:a`)
metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`)
}
index++;
}
for (let vid of onlyVid) {
if (!hasVideo) {
args.push(`-i "${vid.path}"`)
metaData.push(`-map ${index}`)
metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(vid.lang, vid.lang)}`)
metaData.push(`-metadata:s:v:${index} title="[Funimation]"`)
hasVideo = true
index++;
}
}
for (let aud of onlyAudio) {
args.push(`-i "${aud.path}"`)
metaData.push(`-map ${index}`)
metaData.push(`-metadata:s:a:${index} language=${getLanguageCode(aud.lang, aud.lang)}`)
index++;
}
for (let index in subtitles) {
let sub = subtitles[index];
args.push(`-i "${sub.file}"`);
}
args.push(...metaData)
args.push(...subtitles.map((_, subIndex) => `-map ${subIndex + index}`));
args.push(
'-c:v copy',
'-c:a copy'
);
args.push(output.split('.').pop().toLowerCase() === "mp4" ? '-c:s mov_text' : '*c:s ass')
args.push(...subtitles.map((sub, index) => `-metadata:s:${index + 2} language=${getLanguageCode(sub.language)}`));
args.push(`"${output}"`);
return args.join(' ');
};
/**
* @param {string} videoFile
* @param {object} audioFile
* @param {Array<object>} subtitles
* @returns {string}
*/
const buildCommandMkvMerge = (videoAndAudio, onlyVid, onlyAudio, subtitles, output) => {
let args = [];
let hasVideo = false;
args.push(`-o "${output}"`);
args.push(
'--no-date',
'--disable-track-statistics-tags',
'--engage no_variable_data',
);
for (let vid of videoAndAudio) {
if (!hasVideo) {
args.push(
'--video-tracks 0',
'--audio-tracks 1'
)
args.push(`--track-name 0:[Funimation]`)
args.push(`--language 1:${getLanguageCode(vid.lang, argv.todo ? 'jpn' : 'eng')}`);
hasVideo = true
} else {
args.push(
'--no-video',
'--audio-tracks 1'
)
args.push(`--language 1:${getLanguageCode(vid.lang, argv.todo ? 'jpn' : 'eng')}`);
}
args.push(`"${vid.path}"`)
}
for (let vid of onlyVid) {
if (!hasVideo) {
args.push(
'--video-tracks 0',
'--no-audio'
)
args.push(`--track-name 0:[Funimation]`)
hasVideo = true
args.push(`"${vid.path}"`)
}
}
for (let aud of onlyAudio) {
args.push(`--language 0:${getLanguageCode(aud.lang, argv.todo ? 'jpn' : 'eng')}`);
args.push(
'--no-video',
'--audio-tracks 0'
);
args.push(`"${aud.path}"`)
}
if(subtitles.length > 0){
for (let subObj of subtitles) {
args.push('--language',`0:${getLanguageCode(subObj.language)}`);
args.push(`"${subObj.file}"`);
}
} else {
args.push(
'--no-subtitles',
'--no-attachments'
);
}
return args.join(' ');
};
const getLanguageCode = (from, _default = 'eng') => {
for (let lang in iso639.iso_639_2) {
let langObj = iso639.iso_639_2[lang];
if (Object.prototype.hasOwnProperty.call(langObj, '639-1') && langObj['639-1'] === from) {
return langObj['639-2'];
}
}
return _default;
};
module.exports = {
buildCommandFFmpeg,
getLanguageCode,
buildCommandMkvMerge
};

41
modules/module.app-args.js
View file

@ -11,8 +11,9 @@ const availableFilenameVars = [
const appArgv = (cfg) => {
// init
return yargs.parserConfiguration({
'duplicate-arguments-array': false,
const argv = yargs.parserConfiguration({
'duplicate-arguments-array': true,
"camel-case-expansion": false
})
// main
.wrap(Math.min(120)) // yargs.terminalWidth()
@ -73,22 +74,16 @@ const appArgv = (cfg) => {
.option('dub', {
group: 'Downloading:',
describe: 'Download non-Japanese Dub (English Dub mode by default)',
choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN' ],
choices: [ 'enUS', 'esLA', 'ptBR', 'zhMN', 'jpJP' ],
default: cfg.dub || 'enUS',
type: 'string',
})
.option('sub', {
group: 'Downloading:',
describe: 'Japanese Dub with subtitles mode (English Dub mode by default)',
default: cfg.subsMode || false,
type: 'boolean',
type: 'array',
})
.option('subLang', {
group: 'Downloading:',
describe: 'Set the subtitle language (English is default and fallback)',
default: cfg.subLang || 'enUS',
choices: [ 'enUS', 'esLA', 'ptBR' ],
type: 'string'
type: 'array'
})
.option('fontSize', {
group: 'Downloading:',
@ -128,6 +123,7 @@ const appArgv = (cfg) => {
group: 'Downloading:',
describe: 'Skip downloading subtitles for English Dub (if available)',
type: 'boolean',
default: false
})
// proxy
.option('proxy', {
@ -161,12 +157,6 @@ const appArgv = (cfg) => {
default: cfg.mp4mux || false,
type: 'boolean'
})
.option('mks', {
group: 'Muxing:',
describe: 'Add subtitles to mkv/mp4 (if available)',
default: cfg.muxSubs || false,
type: 'boolean'
})
// filenaming
.option('fileName', {
group: 'Filename Template:',
@ -185,17 +175,10 @@ const appArgv = (cfg) => {
// util
.option('nocleanup', {
group: 'Utilities:',
describe: 'Move temporary files to trash folder instead of deleting',
describe: 'Dont\'t delete the input files after muxing',
default: cfg.noCleanUp || false,
type: 'boolean'
})
.option('notrashfolder', {
implies: ['nocleanup'],
group: 'Utilities:',
describe: 'Don\'t move temporary files to trash folder (Used with --nocleanup)',
default: cfg.noTrashFolder || false,
type: 'boolean'
})
// help
.option('help', {
alias: 'h',
@ -212,6 +195,14 @@ const appArgv = (cfg) => {
// --
.argv;
// Resolve unwanted arrays
for (let key in argv) {
if (argv[key] instanceof Array && !(key === "subLang" || key === "dub")) {
argv[key] = argv[key].pop()
}
}
return argv;
};
const showHelp = yargs.showHelp;

2
package.json
View file

@ -1,7 +1,7 @@
{
"name": "funimation-downloader-nx",
"short_name": "funi",
"version": "4.8.3",
"version": "4.9.0",
"description": "Download videos from Funimation via cli.",
"keywords": [
"download",

0
videos/_trash/.gitkeep
View file