Use Langs data in Funi

This commit is contained in:
Izuco 2021-11-30 19:44:23 +01:00
parent 5be607ff2f
commit e642e73a52
No known key found for this signature in database
GPG key ID: 318460063D70949F
10 changed files with 92 additions and 135 deletions

14
@types/funiTypes.d.ts vendored Normal file
View file

@ -0,0 +1,14 @@
import { LanguageItem } from '../modules/module.langsData';
export type FunimationMediaDownload = {
id: string,
title: string,
showTitle: string
}
export type Subtitle = {
url: string,
lang: LanguageItem,
ext: string,
out?: string
}

View file

@ -1,7 +0,0 @@
export type Subtitle = {
path: string,
ext: string,
langName: string,
language: string,
file?: string
}

View file

@ -85,7 +85,7 @@ export default (async () => {
if (argv.dubLang.length > 1) {
console.log('[INFO] One show can only be downloaded with one dub. Use --srz instead.');
}
argv.dubLang = argv.dubLang[0];
argv.dubLang = [argv.dubLang[0]];
return await getSeasonById();
}
else if(argv.e){
@ -876,9 +876,6 @@ const downloadFromSeriesID = async () => {
if (!parsed)
return;
const result = parseSeriesResult(parsed);
/*if(selectedMedia.length > 1){
appstore.isBatch = true;
}*/
const episodes : Record<string, {
items: Item[],
langs: langsData.LanguageItem[]
@ -1178,7 +1175,7 @@ async function downloadMediaList(medias: CrunchyEpMeta) : Promise<{
return undefined;
}
const audDub = langsData.findLang(langsData.fixLanguageTag(pbData.audio_locale)).code;
const audDub = langsData.findLang(langsData.fixLanguageTag(pbData.audio_locale) || '').code;
hsLangs = langsData.sortTags(hsLangs);
streams = streams.map((s) => {
@ -1237,10 +1234,6 @@ async function downloadMediaList(medias: CrunchyEpMeta) : Promise<{
console.log('[INFO] Downloading video...');
curStream = streams[argv.kstream-1];
if(argv.dubLang != curStream.audio_lang){
argv.dubLang = curStream.audio_lang as string;
console.log(`[INFO] audio language code detected, setted to ${curStream.audio_lang} for this episode`);
}
console.log('[INFO] Playlists URL: %s (%s)', curStream.url, curStream.type);
}

129
funi.ts
View file

@ -28,22 +28,21 @@ const token = yamlCfg.loadFuniToken();
const argv = appYargs.appArgv(cfg.cli);
// Import modules after argv has been exported
import getData from './modules/module.getdata.js';
import merger, { SubtitleInput } from './modules/module.merger';
import merger from './modules/module.merger';
import parseSelect from './modules/module.parseSelect';
import { EpisodeData, MediaChild } from './@types/episode';
import { Subtitle } from './@types/subtitleObject';
import { Subtitle } from './@types/funiTypes';
import { StreamData } from './@types/streamData';
import { DownloadedFile } from './@types/downloadedFile';
import parseFileName, { Variable } from './modules/module.filename';
import { downloaded } from './modules/module.downloadArchive';
import { FunimationMediaDownload } from './@types/funiTypes';
import * as langsData from './modules/module.langsData';
// check page
argv.p = 1;
// fn variables
let title = '',
showTitle = '',
fnEpNum: string|number = 0,
let fnEpNum: string|number = 0,
fnOutput: string[] = [],
season = 0,
tsDlPath: {
@ -58,10 +57,10 @@ export default (async () => {
console.log(`\n=== Multi Downloader NX ${packageJson.version} ===\n`);
cfg.bin = await yamlCfg.loadBinCfg();
if (argv.allDubs) {
argv.dub = appYargs.dubLang;
argv.dub = langsData.dubLanguageCodes;
}
if (argv.allSubs) {
argv.subLang = appYargs.subLang;
argv.subLang = langsData.languages.map(a => a.code);
}
// select mode
if(argv.auth){
@ -288,8 +287,6 @@ async function getEpisode(fnSlug: {
if(!episodeData.ok || !episodeData.res){return;}
const ep = JSON.parse(episodeData.res.body).items[0] as EpisodeData, streamIds = [];
// build fn
showTitle = ep.parent.title;
title = ep.title;
season = parseInt(ep.parent.seasonNumber);
if(ep.mediaCategory != 'Episode'){
ep.number = ep.number !== '' ? ep.mediaCategory+ep.number : ep.mediaCategory+'#'+ep.id;
@ -332,14 +329,6 @@ async function getEpisode(fnSlug: {
}
});
const dubType = {
'enUS': 'English',
'esLA': 'Spanish (Latin Am)',
'ptBR': 'Portuguese (Brazil)',
'zhMN': 'Chinese (Mandarin, PRC)',
'jaJP': 'Japanese'
};
// select
stDlPath = [];
for(const m of media){
@ -352,10 +341,10 @@ async function getEpisode(fnSlug: {
const selUncut = !argv.simul && uncut[dub_type] && m.version?.match(/uncut/i)
? true
: (!uncut[dub_type] || argv.simul && m.version?.match(/simulcast/i) ? true : false);
for (const curDub of (argv.dub as appYargs.possibleDubs)) {
if(dub_type == dubType[curDub] && selUncut){
for (const curDub of argv.dubLang) {
const item = langsData.languages.find(a => a.code === curDub);
if(item && dub_type == (item.funi_name || item.name) && selUncut){
streamIds.push({
id: m.id,
lang: merger.getLanguageCode(curDub, curDub.slice(0, -2))
});
@ -365,17 +354,17 @@ async function getEpisode(fnSlug: {
}
}
console.log(`[#${m.id}] ${dub_type} [${m.version}]${(selected?' (selected)':'')}${
localSubs && localSubs.length > 0 && selected ? ` (using ${localSubs.map(a => `'${a.langName}'`).join(', ')} for subtitles)` : ''
localSubs && localSubs.length > 0 && selected ? ` (using ${localSubs.map(a => `'${a.lang.name}'`).join(', ')} for subtitles)` : ''
}`);
}
}
const already: string[] = [];
stDlPath = stDlPath.filter(a => {
if (already.includes(a.language)) {
if (already.includes(a.lang.code)) {
return false;
} else {
already.push(a.language);
already.push(a.lang.code);
return true;
}
});
@ -417,51 +406,48 @@ async function getEpisode(fnSlug: {
return;
}
else{
return await downloadStreams(fnSlug.episodeID);
const res = await downloadStreams({
id: fnSlug.episodeID,
title: ep.title,
showTitle: ep.parent.title
});
if (res === true) {
downloaded({
service: 'funi',
type: 's'
}, argv.s as string, [fnSlug.episodeID]);
}
return res;
}
}
}
function getSubsUrl(m: MediaChild[]) : Subtitle[] {
if(argv.nosubs && !argv.sub){
if((argv.nosubs && !argv.sub) || argv.dlsubs.includes('none')){
return [];
}
let subLangs = argv.subLang as appYargs.possibleSubs;
const subType = {
'enUS': 'English',
'esLA': 'Spanish (Latin Am)',
'ptBR': 'Portuguese (Brazil)'
};
const subLangAvailable = m.some(a => subLangs.some(subLang => a.ext == 'vtt' && a.language === subType[subLang]));
if (!subLangAvailable) {
subLangs = [ 'enUS' ];
}
const found: Subtitle[] = [];
for(const i in m){
const fpp = m[i].filePath.split('.');
const fpe = fpp[fpp.length-1];
for (const lang of subLangs) {
if(fpe == 'vtt' && m[i].language === subType[lang]) {
found.push({
path: m[i].filePath,
ext: `.${lang}`,
langName: subType[lang],
language: m[i].languages[0].code || lang.slice(0, 2)
});
}
let media = m.filter(a => a.filePath.split('.').pop() === 'vtt');
for (let me of media) {
const lang = langsData.languages.find(a => me.language === (a.funi_name || a.name))
if (!lang) {
continue;
}
if (argv.dlsubs.includes('all') || argv.dlsubs.some(a => a === lang.locale)) {
found.push({
url: me.filePath,
ext: `.${lang.code}`,
lang
})
}
}
return found;
}
async function downloadStreams(epsiodeID: string){
async function downloadStreams(epsiode: FunimationMediaDownload){
// req playlist
@ -602,8 +588,8 @@ async function downloadStreams(epsiodeID: string){
fnOutput = parseFileName(argv.fileName, ([
['episode', fnEpNum],
['title', title],
['showTitle', showTitle],
['title', epsiode.title],
['showTitle', epsiode.showTitle],
['season', season],
['width', plLayersRes[argv.q].width],
['height', plLayersRes[argv.q].height],
@ -700,13 +686,13 @@ async function downloadStreams(epsiodeID: string){
console.log('[INFO] Downloading subtitles...');
for (const subObject of stDlPath) {
const subsSrc = await getData({
url: subObject.path,
url: subObject.url,
debug: argv.debug,
});
if(subsSrc.ok && subsSrc.res){
const assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.langName, argv.fontSize, argv.fontName);
subObject.file = path.join(cfg.dir.content, ...fnOutput.slice(0, -1), `${fnOutput.slice(-1)}.subtitle${subObject.ext}${subsExt}`);
fs.writeFileSync(subObject.file, assData);
const assData = vttConvert(subsSrc.res.body, (subsExt == '.srt' ? true : false), subObject.lang.name, argv.fontSize, argv.fontName);
subObject.out = path.join(cfg.dir.content, ...fnOutput.slice(0, -1), `${fnOutput.slice(-1)}.subtitle${subObject.ext}${subsExt}`);
fs.writeFileSync(subObject.out, assData);
}
else{
console.log('[ERROR] Failed to download subtitles!');
@ -725,10 +711,6 @@ async function downloadStreams(epsiodeID: string){
if(argv.skipmux){
console.log('[INFO] Skipping muxing...');
downloaded({
service: 'funi',
type: 's'
}, argv.s as string, [epsiodeID]);
return;
}
@ -752,7 +734,14 @@ async function downloadStreams(epsiodeID: string){
onlyAudio: puraudio,
onlyVid: purvideo,
output: `${path.join(cfg.dir.content, ...fnOutput)}.${ffext}`,
subtitels: stDlPath as SubtitleInput[],
subtitels: stDlPath.map(a => {
return {
file: a.out as string,
language: a.lang.code,
lookup: false,
title: a.lang.name
};
}),
videoAndAudio: audioAndVideo,
simul: argv.simul,
skipSubMux: argv.skipSubMux
@ -768,26 +757,14 @@ async function downloadStreams(epsiodeID: string){
}
else{
console.log('\n[INFO] Done!\n');
downloaded({
service: 'funi',
type: 's'
}, argv.s as string, [epsiodeID]);
return true;
}
if (argv.nocleanup) {
downloaded({
service: 'funi',
type: 's'
}, argv.s as string, [epsiodeID]);
return true;
}
mergeInstance.cleanUp();
console.log('\n[INFO] Done!\n');
downloaded({
service: 'funi',
type: 's'
}, argv.s as string, [epsiodeID]);
return true;
}

View file

@ -26,15 +26,7 @@ const availableFilenameVars: AvailableFilenameVars[] = [
'service'
];
export type possibleDubs = (
'enUS' | 'esLA' | 'ptBR' | 'zhMN' | 'jaJP'
)[];
export type possibleSubs = (
'enUS' | 'esLA' | 'ptBR'
)[];
const subLang: possibleSubs = ['enUS', 'esLA', 'ptBR'];
const dubLang: possibleDubs = ['enUS', 'esLA', 'ptBR', 'zhMN', 'jaJP'];
let argvC: { [x: string]: unknown; skipSubMux: boolean, downloadArchive: boolean, addArchive: boolean, but: boolean, auth: boolean | undefined; dlFonts: boolean | undefined; search: string | undefined; 'search-type': string; page: number | undefined; 'search-locale': string; new: boolean | undefined; 'movie-listing': string | undefined; series: string | undefined; s: string | undefined; e: string | undefined; q: number; x: number; kstream: number; partsize: number; hslang: string; subLang: string[]; dlsubs: string | string[]; novids: boolean | undefined; noaudio: boolean | undefined; nosubs: boolean | undefined; dub: possibleDubs; dubLang: string; all: boolean; fontSize: number; allSubs: boolean; allDubs: boolean; timeout: number; simul: boolean; mp4: boolean; skipmux: boolean | undefined; fileName: string; numbers: number; nosess: string; debug: boolean | undefined; nocleanup: boolean; help: boolean | undefined; service: 'funi' | 'crunchy'; update: boolean; fontName: string | undefined; _: (string | number)[]; $0: string; };
let argvC: { [x: string]: unknown; skipSubMux: boolean, downloadArchive: boolean, addArchive: boolean, but: boolean, auth: boolean | undefined; dlFonts: boolean | undefined; search: string | undefined; 'search-type': string; page: number | undefined; 'search-locale': string; new: boolean | undefined; 'movie-listing': string | undefined; series: string | undefined; s: string | undefined; e: string | undefined; q: number; x: number; kstream: number; partsize: number; hslang: string; dlsubs: string[]; novids: boolean | undefined; noaudio: boolean | undefined; nosubs: boolean | undefined; dubLang: string[]; all: boolean; fontSize: number; allSubs: boolean; allDubs: boolean; timeout: number; simul: boolean; mp4: boolean; skipmux: boolean | undefined; fileName: string; numbers: number; nosess: string; debug: boolean | undefined; nocleanup: boolean; help: boolean | undefined; service: 'funi' | 'crunchy'; update: boolean; fontName: string | undefined; _: (string | number)[]; $0: string; };
export type ArgvType = typeof argvC;
@ -66,8 +58,6 @@ export {
appArgv,
showHelp,
availableFilenameVars,
subLang,
dubLang,
overrideArguments
};
@ -185,18 +175,13 @@ const getArgv = (cfg: { [key:string]: unknown }) => {
default: parseDefault<string>('hsLang', 'none'),
type: 'string',
})
.option('subLang', {
group: groups.dl,
describe: 'Set the subtitles to download (Funi only)',
choices: subLang,
default: parseDefault<string[]>('subLang', []),
type: 'array'
})
.option('dlsubs', {
group: groups.dl,
describe: 'Download subtitles by language tag (space-separated) (crunchy only)',
describe: 'Download subtitles by language tag (space-separated)'
+ `\nFuni Only: ${langsData.languages.filter(a => a.funi_locale && !a.cr_locale).map(a => a.locale).join(', ')}`
+ `\nCrunchy Only: ${langsData.languages.filter(a => a.cr_locale && !a.funi_locale).map(a => a.locale).join(', ')}`,
choices: langsData.subtitleLanguagesFilter,
default: parseDefault<string|string[]>('dlSubs', 'all'),
default: parseDefault<string[]>('dlSubs', ['all']),
type: 'array',
})
.option('novids', {
@ -214,18 +199,13 @@ const getArgv = (cfg: { [key:string]: unknown }) => {
describe: 'Skip downloading subtitles',
type: 'boolean'
})
.option('dub', {
group: groups.dl,
describe: 'Set languages to download (funi only)',
choices: dubLang,
default: parseDefault<possibleDubs>('dub', ['enUS']),
type: 'array'
})
.option('dubLang', {
group: groups.dl,
describe: 'Set the language to download (Crunchy only)',
describe: 'Set the language to download: '
+ `\nFuni Only: ${langsData.languages.filter(a => a.funi_locale && !a.cr_locale).map(a => a.code).join(', ')}`
+ `\nCrunchy Only: ${langsData.languages.filter(a => a.cr_locale && !a.funi_locale).map(a => a.code).join(', ')}`,
choices: langsData.dubLanguageCodes,
default: parseDefault('dubLanguage', langsData.dubLanguageCodes.slice(-1)[0]),
default: parseDefault<string[]>('dubLanguage', [langsData.dubLanguageCodes.slice(-1)[0]]),
array: true,
})
.option('all', {

View file

@ -1,19 +1,20 @@
// available langs
export type LanguageItem = {
cr_locale: string,
cr_locale?: string,
locale: string,
code: string,
name: string,
language?: string
language?: string,
funi_locale?: string,
funi_name?: string
}
const languages: LanguageItem[] = [
{ cr_locale: 'en-US', locale: 'en', code: 'eng', name: 'English' },
{ cr_locale: 'es-LA', locale: 'es-419', code: 'spa', name: 'Spanish', language: 'Latin American Spanish' },
{ cr_locale: 'es-LA', funi_name: 'Spanish (Latin Am)', funi_locale: 'esLA', locale: 'es-419', code: 'spa', name: 'Spanish', language: 'Latin American Spanish' },
{ cr_locale: 'es-419', locale: 'es-419', code: 'spa', name: 'Spanish', language: 'Latin American Spanish' },
{ cr_locale: 'es-ES', locale: 'es', code: 'spa', name: 'Spanish' },
{ cr_locale: 'pt-BR', locale: 'pt-BR', code: 'por', name: 'Portuguese', language: 'Brazilian Portuguese' },
{ cr_locale: 'pt-BR', funi_name: 'Portuguese (Brazil)', funi_locale: 'ptBR', locale: 'pt-BR', code: 'por', name: 'Portuguese', language: 'Brazilian Portuguese' },
{ cr_locale: 'fr-FR', locale: 'fr', code: 'fra', name: 'French' },
{ cr_locale: 'de-DE', locale: 'de', code: 'deu', name: 'German' },
{ cr_locale: 'ar-ME', locale: 'ar', code: 'ara', name: 'Arabic' },
@ -21,7 +22,9 @@ const languages: LanguageItem[] = [
{ cr_locale: 'it-IT', locale: 'it', code: 'ita', name: 'Italian' },
{ cr_locale: 'ru-RU', locale: 'ru', code: 'rus', name: 'Russian' },
{ cr_locale: 'tr-TR', locale: 'tr', code: 'tur', name: 'Turkish' },
{ cr_locale: 'ja-JP', locale: 'ja', code: 'jpn', name: 'Japanese' },
{ cr_locale: 'ja-JP', funi_locale: 'jaJP', locale: 'ja', code: 'jpn', name: 'Japanese' },
{ funi_locale: 'zhMN', locale: 'zh', code: 'cmn', name: 'Chinese (Mandarin, PRC)' },
{ cr_locale: 'en-US', funi_locale: 'enUS', locale: 'en', code: 'eng', name: 'English' },
];
// add en language names
@ -77,7 +80,8 @@ const findLang = (cr_locale: string) => {
};
const fixAndFindCrLC = (cr_locale: string) => {
return findLang(fixLanguageTag(cr_locale));
const str = fixLanguageTag(cr_locale);
return findLang(str || '');
};
// rss subs lang parser

View file

@ -82,11 +82,6 @@ class Req {
// if auth
const loc = new URL(durl);
// avoid cloudflare protection
if(loc.origin == this.domain.www){
options.minVersion = 'TLSv1.3';
options.maxVersion = 'TLSv1.3';
options.http2 = true;
}
// debug
options.hooks = {
beforeRequest: [

2
package-lock.json generated
View file

@ -1,6 +1,6 @@
{
"name": "multi-downloader-nx",
"version": "2.0.6",
"version": "2.0.7",
"lockfileVersion": 1,
"requires": true,
"dependencies": {

View file

@ -1,7 +1,7 @@
{
"name": "multi-downloader-nx",
"short_name": "aniDL",
"version": "2.0.6",
"version": "2.0.7",
"description": "Download videos from Funimation or Crunchyroll via cli",
"keywords": [
"download",

1
tsc.ts
View file

@ -12,6 +12,7 @@ const ignore = [
path.join('config', 'token.yml'),
path.join('config', 'updates.json'),
path.join('config', 'cr_token.yml'),
path.join('config', 'token.yml'),
'.eslint',
].map(a => path.join(__dirname, a));