Everything but merging

This commit is contained in:
Izuco 2021-10-31 20:55:42 +01:00
parent 22b67b45d5
commit 8b557abdf1
No known key found for this signature in database
GPG key ID: 318460063D70949F
7 changed files with 138 additions and 130 deletions

View file

@ -164,12 +164,4 @@ export interface SeriesMetadata {
maturity_ratings: string[];
season_count: number;
tenant_categories: TenantCategory[];
}
export enum ItemType {
Episode = "episode",
Series = "series",
Season = 'season',
MovieListing = 'movie_listing',
Movie = 'Movie'
}
}

View file

@ -8,6 +8,7 @@ export type CrunchyEpMeta = {
}
export type ParseItem = {
__class__?: string;
isSelected?: boolean,
type?: string,
id: string,

View file

@ -1,5 +1,5 @@
declare module 'm3u8-parsed' {
export default function (data: string): {
export type M3U8 = {
allowCache: boolean,
discontinuityStarts: [],
segments: {
@ -44,5 +44,6 @@ declare module 'm3u8-parsed' {
'BANDWIDTH': number
}
}[],
};
}
export default function (data: string): M3U8;
}

View file

@ -18,7 +18,7 @@ import * as fontsData from './modules/module.fontsData';
import * as langsData from './modules/module.langsData';
import * as yamlCfg from './modules/module.cfg-loader';
import * as yargs from './modules/module.app-args';
import epsFilter from './modules/module.eps-filter';
import * as epsFilter from './modules/module.eps-filter';
import Merger from './modules/module.merger';
// new-cfg paths
@ -32,7 +32,8 @@ let cmsToken: {
const argv = yargs.appArgv(cfg.cli)
const appstore: {
fn: Variable[],
isBatch: boolean
isBatch: boolean,
out?: string
} = {
fn: [],
isBatch: false
@ -41,11 +42,11 @@ const appstore: {
// load req
import { domain, api } from './modules/module.api-urls';
import * as reqModule from './modules/module.req';
import { CrunchySearch, ItemItem, ItemType } from './@types/crunchySearch';
import { CrunchySearch } from './@types/crunchySearch';
import { CrunchyEpisodeList } from './@types/crunchyEpisodeList';
import { CrunchyEpMeta, ParseItem } from './@types/crunchyTypes';
import { ObjectInfo } from './@types/objectInfo';
import { Variable } from './modules/module.filename';
import parseFileName, { Variable } from './modules/module.filename';
import { PlaybackData } from './@types/playbackData';
const req = new reqModule.Req(domain, argv);
@ -80,16 +81,16 @@ const req = new reqModule.Req(domain, argv);
await refreshToken();
await getMovieListingById();
}
else if(argv.season && argv.season.match(/^[0-9A-Z]{9}$/)){
else if(argv.s && argv.s.match(/^[0-9A-Z]{9}$/)){
await refreshToken();
await getSeasonById();
}
else if(argv.episode){
else if(argv.e){
await refreshToken();
await getObjectById();
}
else{
appYargs.showHelp();
yargs.showHelp();
}
})();
@ -340,9 +341,8 @@ async function parseObject(item: ParseItem, pad?: number, getSeries?: boolean, g
getSeries = getSeries === undefined ? true : getSeries;
getMovieListing = getMovieListing === undefined ? true : getMovieListing;
item.isSelected = item.isSelected === undefined ? false : item.isSelected;
if(!item.type){
console.log('[INFO] Unable to parse type for %s. Defaulted to %s', item.id, ItemType.Episode)
item.type = ItemType.Episode;
if(!item.type) {
item.type = item.__class__;
}
const oTypes = {
'series': 'Z', // SRZ
@ -357,7 +357,7 @@ async function parseObject(item: ParseItem, pad?: number, getSeries?: boolean, g
const oMetadata = [],
oBooleans = [],
tMetadata = item.type + '_metadata',
iMetadata = item[tMetadata] ? item[tMetadata] : item,
iMetadata = (Object.prototype.hasOwnProperty.call(item, tMetadata) ? item[tMetadata as keyof ParseItem] : item) as Record<string, any>,
iTitle = [ item.title ];
// set object booleans
if(iMetadata.duration_ms){
@ -599,7 +599,7 @@ async function getSeasonById(){
api.beta_cms,
cmsToken.cms.bucket,
'/seasons/',
argv.season,
argv.s,
'?',
new URLSearchParams({
'Policy': cmsToken.cms.policy,
@ -619,7 +619,7 @@ async function getSeasonById(){
cmsToken.cms.bucket,
'/episodes?',
new URLSearchParams({
'season_id': argv.season as string,
'season_id': argv.s as string,
'Policy': cmsToken.cms.policy,
'Signature': cmsToken.cms.signature,
'Key-Pair-Id': cmsToken.cms.key_pair_id,
@ -644,7 +644,7 @@ async function getSeasonById(){
}
const doEpsFilter = new epsFilter.doFilter();
const selEps = doEpsFilter.checkFilter(argv.episode);
const selEps = doEpsFilter.checkFilter(argv.e);
const selectedMedia: CrunchyEpMeta[] = [];
episodeList.items.forEach((item) => {
@ -717,7 +717,7 @@ async function getObjectById(returnData?: boolean){
}
const doEpsFilter = new epsFilter.doFilter();
const inpMedia = doEpsFilter.checkBetaFilter(argv.e);
const inpMedia = doEpsFilter.checkBetaFilter(argv.e as string);
if(inpMedia.length < 1){
console.log('\n[INFO] Objects not selected!\n');
@ -922,9 +922,8 @@ async function getMedia(mMeta: CrunchyEpMeta){
console.log('[INFO] Downloading video...');
curStream = streams[argv.kstream-1];
if(argv.dub != curStream.audio_lang){
argv.dub = curStream.audio_lang;
if(argv.dubLang != curStream.audio_lang){
argv.dubLang = curStream.audio_lang as string;
console.log(`[INFO] audio language code detected, setted to ${curStream.audio_lang} for this episode`);
}
@ -932,24 +931,28 @@ async function getMedia(mMeta: CrunchyEpMeta){
console.log('[INFO] Playlists URL: %s (%s)', streamUrlTxt, curStream.type);
}
if(!argv.skipdl && !dlFailed){
const streamPlaylistsReq = await req.getData(curStream.url, {useProxy: argv['use-proxy-streaming']});
if(!streamPlaylistsReq.ok){
if(!argv.skipdl && !dlFailed && curStream){
const streamPlaylistsReq = await req.getData(curStream.url);
if(!streamPlaylistsReq.ok || !streamPlaylistsReq.res){
console.log('[ERROR] CAN\'T FETCH VIDEO PLAYLISTS!');
dlFailed = true;
}
else{
const streamPlaylists = m3u8(streamPlaylistsReq.res.body);
let plServerList = [],
plStreams = {},
plQualityStr = [],
plMaxQuality = 240;
let plServerList: string[] = [],
plStreams: Record<string, Record<string, string>> = {},
plQuality: {
str: string,
dim: string,
RESOLUTION: {
width: number,
height: number
}
}[] = []
for(const pl of streamPlaylists.playlists){
// set quality
let plResolution = pl.attributes.RESOLUTION.height;
let plResolutionText = `${plResolution}p`;
// set max quality
plMaxQuality = plMaxQuality < plResolution ? plResolution : plMaxQuality;
let plResolution = pl.attributes.RESOLUTION;
let plResolutionText = `${plResolution.width}x${plResolution.height}`;
// parse uri
let plUri = new URL(pl.uri);
let plServer = plUri.hostname;
@ -976,76 +979,84 @@ async function getMedia(mMeta: CrunchyEpMeta){
}
// set plQualityStr
let plBandwidth = Math.round(pl.attributes.BANDWIDTH/1024);
if(plResolution < 1000){
plResolution = plResolution.toString().padStart(4, ' ');
}
let qualityStrAdd = `${plResolution}p (${plBandwidth}KiB/s)`;
let qualityStrAdd = `${plResolutionText} (${plBandwidth}KiB/s)`;
let qualityStrRegx = new RegExp(qualityStrAdd.replace(/(:|\(|\)|\/)/g, '\\$1'), 'm');
let qualityStrMatch = !plQualityStr.join('\r\n').match(qualityStrRegx);
let qualityStrMatch = !plQuality.map(a => a.str).join('\r\n').match(qualityStrRegx);
if(qualityStrMatch){
plQualityStr.push(qualityStrAdd);
plQuality.push({
str: qualityStrAdd,
dim: plResolutionText,
RESOLUTION: plResolution
});
}
}
argv.server = argv.server > plServerList.length ? 1 : argv.server;
argv.quality = argv.quality == 'max' ? `${plMaxQuality}p` : argv.quality;
argv.appstore.fn.out = fnOutputGen();
argv.server = argv.x > plServerList.length ? 1 : argv.x;
let plSelectedServer = plServerList[argv.server - 1];
let plSelectedServer = plServerList[argv.x - 1];
let plSelectedList = plStreams[plSelectedServer];
let selPlUrl = plSelectedList[argv.quality] ? plSelectedList[argv.quality] : '';
plQualityStr.sort();
console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`);
console.log(`[INFO] Available qualities:\n\t${plQualityStr.join('\n\t')}`);
plQuality.sort((a, b) => {
const aMatch = a.dim.match(/[0-9]+/) || []
const bMatch = b.dim.match(/[0-9]+/) || []
return parseInt(aMatch[0]) - parseInt(bMatch[0])
});
let quality = argv.q;
if (quality > plQuality.length) {
console.log(`[WARN] The requested quality of ${argv.q} is greater than the maximun ${plQuality.length}.\nTherefor the maximum will be capped at ${plQuality.length}.`)
quality = plQuality.length;
}
let selPlUrl = quality === 0 ? plSelectedList[plQuality.pop()?.dim as string] :
plSelectedList[plQuality.map(a => a.dim)[quality - 1]] ? plSelectedList[plQuality.map(a => a.dim)[quality - 1]] : '';
console.log(`[INFO] Servers available:\n\t${plServerList.join('\n\t')}`);
console.log(`[INFO] Available qualities:\n\t${plQuality.map((a, ind) => `[${ind+1}] ${a.str}`).join('\n\t')}`);
if(selPlUrl != ''){
console.log(`[INFO] Selected quality: ${argv.quality} @ ${plSelectedServer}`);
appstore.fn.push({
name: 'height',
type: 'number',
replaceWith: quality === 0 ? plQuality.pop()?.RESOLUTION.height as number : plQuality[quality - 1].RESOLUTION.height
}, {
name: 'width',
type: 'number',
replaceWith: quality === 0 ? plQuality.pop()?.RESOLUTION.width as number : plQuality[quality - 1].RESOLUTION.width
})
console.log(`[INFO] Selected quality: ${Object.keys(plSelectedList).find(a => plSelectedList[a] === selPlUrl)} @ ${plSelectedServer}`);
if(argv['show-stream-url']){
console.log('[INFO] Stream URL:', selPlUrl);
}
console.log(`[INFO] Output filename: ${argv.appstore.fn.out}`);
const chunkPage = await req.getData(selPlUrl, {useProxy: argv['use-proxy-streaming']});
if(!chunkPage.ok){
// TODO check filename
appstore.out = parseFileName(argv.fileName, appstore.fn, argv.numbers).join(path.sep)
console.log(`[INFO] Output filename: ${appstore.out}`);
const chunkPage = await req.getData(selPlUrl);
if(!chunkPage.ok || !chunkPage.res){
console.log('[ERROR] CAN\'T FETCH VIDEO PLAYLIST!');
dlFailed = true;
}
else{
const chunkPlaylist = m3u8(chunkPage.res.body);
let proxyHLS;
if(argv.proxy && argv['use-proxy-streaming']){
try{
proxyHLS = {};
proxyHLS.url = reqModule.buildProxy(argv.proxy, argv['proxy-auth']);
proxyHLS.url = proxyHLS.url.toString();
}
catch(e){
console.log(`\n[WARN] Not valid proxy URL${e.input?' ('+e.input+')':''}!`);
console.log('[WARN] Skiping...');
}
}
let totalParts = chunkPlaylist.segments.length;
let mathParts = Math.ceil(totalParts / argv.tsparts);
let mathMsg = `(${mathParts}*${argv.tsparts})`;
let mathParts = Math.ceil(totalParts / argv.partsize);
let mathMsg = `(${mathParts}*${argv.partsize})`;
console.log('[INFO] Total parts in stream:', totalParts, mathMsg);
let tsFile = path.join(cfg.dir.content, argv.appstore.fn.out);
let tsFile = path.join(cfg.dir.content, appstore.out);
const split = appstore.out.split(path.sep).slice(0, -1);
split.forEach((val, ind, arr) => {
let isAbsolut = path.isAbsolute(appstore.out as string);
if (!fs.existsSync(path.join(isAbsolut ? '' : cfg.dir.content, ...arr.slice(0, ind), val)))
fs.mkdirSync(path.join(isAbsolut ? '' : cfg.dir.content, ...arr.slice(0, ind), val));
})
let streamdlParams = {
fn: `${tsFile}.ts`,
m3u8json: chunkPlaylist,
// baseurl: chunkPlaylist.baseUrl,
pcount: argv.tsparts,
partsOffset: 0,
proxy: proxyHLS || false,
};
let dlStreamByPl = await new streamdl(streamdlParams).download();
if(!dlStreamByPl.ok){
fs.writeFileSync(`${tsFile}.ts.resume`, JSON.stringify(dlStreamByPl.parts));
console.log(`[ERROR] DL Stats: ${JSON.stringify(dlStreamByPl.parts)}\n`);
dlFailed = true;
}
else if(fs.existsSync(`${tsFile}.ts.resume`) && dlStreamByPl.ok){
fs.unlinkSync(`${tsFile}.ts.resume`);
}
}
}
else{
@ -1059,6 +1070,7 @@ async function getMedia(mMeta: CrunchyEpMeta){
}
// fix max quality for non streams
/*
if(argv.quality == 'max'){
argv.quality = '1080p';
argv.appstore.fn.out = fnOutputGen();
@ -1125,10 +1137,12 @@ async function getMedia(mMeta: CrunchyEpMeta){
else{
console.log();
}
*/
}
async function muxStreams(){
/*
const merger = await appMux.checkMerger(cfg.bin, argv.mp4);
const muxFile = path.join(cfg.dir.content, argv.appstore.fn.out);
const sxList = argv.appstore.sxList;
@ -1196,12 +1210,15 @@ async function muxStreams(){
}
doCleanUp(isMuxed, muxFile, addSubs, sxList);
*/
}
function doCleanUp(isMuxed, muxFile, addSubs, sxList){
function doCleanUp(isMuxed: boolean, muxFile: string, addSubs: boolean, sxList: {
file: string
}[]){
// set output filename
const fnOut = argv.appstore.fn.out;
const fnOut = appstore.out;
// check paths if same
if(path.join(cfg.dir.trash) == path.join(cfg.dir.content)){
argv.notrashfolder = true;
@ -1241,18 +1258,6 @@ function doCleanUp(isMuxed, muxFile, addSubs, sxList){
}
}
}
// move to subfolder
if(argv.folder && isMuxed){
const dubName = argv.dub.toUpperCase().slice(0, -1);
const dubSuffix = argv.dub != 'jpn' ? ` [${dubName}DUB]` : '';
const titleFolder = shlp.cleanupFilename(argv.appstore.fn.title + dubSuffix);
const subFolder = path.join(cfg.dir.content, '/', titleFolder, '/');
const vExt = '.' + ( !argv.mp4 ? 'mkv' : 'mp4' );
if(!fs.existsSync(subFolder)){
fs.mkdirSync(subFolder);
}
fs.renameSync(muxFile + vExt, path.join(subFolder, fnOut + vExt));
}
// done
console.log('\n[INFO] Done!\n');
}

View file

@ -144,7 +144,7 @@ const appArgv = (cfg: {
type: 'number',
default: parseDefault<number>('partsize', 10)
})
.option('hsland', {
.option('hslang', {
group: groups.dl,
describe: 'Download video with specific hardsubs',
choices: langsData.subtitleLanguagesFilter.slice(1),
@ -180,6 +180,13 @@ const appArgv = (cfg: {
default: parseDefault<possibleDubs>('dub', ['enUS']),
type: 'array'
})
.option('dubLang', {
group: groups.dl,
describe: 'Set the language to download (Crunchy only)',
choices: langsData.dubLanguageCodes,
default: parseDefault('dubLanguage', langsData.dubLanguageCodes.slice(-1)[0]),
type: 'string',
})
.option('all', {
group: groups.dl,
describe: 'Used to download all episodes from the show (Funi only)',
@ -266,8 +273,9 @@ const appArgv = (cfg: {
.option('service', {
group: groups.util,
describe: 'Set the service to use',
choices: ['funi', 'chrunchy'],
demandOption: true
choices: ['funi', 'crunchy'],
demandOption: true,
default: parseDefault<'crunchy'|'funi'|undefined>('service', undefined)
})
.parseSync();
return argv;

View file

@ -8,38 +8,39 @@ const epLtReg = new RegExp (/(?:E|S|M)/);
class doFilter {
constructor(){}
ifMaxEp(type, num){
ifMaxEp(type: keyof typeof epNumLen, num: number){
const maxEp = Math.pow(10, epNumLen[type]) - 1;
return num > maxEp ? true : false;
}
powNum(type){
powNum(type: keyof typeof epNumLen){
return Math.pow(10, epNumLen[type]);
}
checkFilter(inputEps){
checkFilter(inputEps?: string){
// check
inputEps = typeof inputEps != 'undefined'
const inputEpsArr = inputEps !== undefined
? inputEps.toString().split(',') : [];
// input range
const inputEpsRange = [];
const inputEpsRange: string | string[] = [];
// filter wrong numbers
inputEps = inputEps.map((e) => {
const filteredArr = inputEpsArr.map((e) => {
// convert to uppercase
e = e.toUpperCase();
// if range
if(e.match('-') && e.split('-').length == 2){
const eRange = e.split('-');
const eRange: (string|number)[] = e.split('-');
// check range
if (!eRange[0].match(epRegex)) return '';
if (!eRange[0].toString().match(epRegex)) return '';
// set ep latter and pad
const epLetter = eRange[0].match(epLtReg) ? eRange[0].match(epLtReg)[0] : 'E';
const padLen = epNumLen[epLetter];
const epLetter = (eRange[0].toString().match(epLtReg) ? (eRange[0].toString().match(epLtReg) || [])[0] : 'E') as keyof typeof epNumLen;
const padLen = epNumLen[epLetter as keyof typeof epNumLen];
// parse range
eRange[0] = eRange[0].replace(epLtReg, '');
eRange[0] = eRange[0].toString().replace(epLtReg, '');
eRange[0] = parseInt(eRange[0]);
eRange[0] = this.ifMaxEp(epLetter, eRange[0]) ? this.powNum(epLetter) - 1 : eRange[0];
eRange[1] = eRange[1].match(/^\d+$/) ? parseInt(eRange[1]) : 0;
eRange[1] = eRange[1].toString().match(/^\d+$/) ? parseInt(eRange[1] as string) : 0;
eRange[1] = this.ifMaxEp(epLetter, eRange[1]) ? this.powNum(epLetter) - 1 : eRange[1];
console.log(eRange)
// check if correct range
if (eRange[0] > eRange[1]){
const parsedEl = [
@ -63,11 +64,11 @@ class doFilter {
return '';
}
else if(e.match(epRegex)){
const epLetter = e.match(epLtReg) ? e.match(epLtReg)[0] : 'E';
const epLetter = (e.match(epLtReg) ? (e.match(epLtReg) || [])[0] : 'E') as keyof typeof epNumLen;
const padLen = epNumLen[epLetter];
e = parseInt(e.replace(epLtReg, ''));
e = this.ifMaxEp(epLetter, e) ? this.powNum(epLetter) - 1 : e;
return (epLetter != 'E' ? epLetter : '') + e.toString().padStart(padLen, '0');
const e1 = parseInt(e.replace(epLtReg, ''));
const e2 = this.ifMaxEp(epLetter, e1) ? this.powNum(epLetter) - 1 : e1;
return (epLetter != 'E' ? epLetter : '') + e2.toString().padStart(padLen, '0');
}
else if(e.match(betaEpRegex)){
return e;
@ -75,22 +76,21 @@ class doFilter {
return '';
});
// end
inputEps = [...new Set(inputEps.concat(inputEpsRange))];
inputEps = inputEps.indexOf('') > -1 ? inputEps.slice(1) : inputEps;
return inputEps;
const filteredArr1 = [...new Set(filteredArr.concat(inputEpsRange))]
const filteredArr2 = filteredArr1.indexOf('') > -1 ? filteredArr1.slice(1) : filteredArr1;
return filteredArr2;
}
checkMediaFilter(e){
e = e.split(',');
checkMediaFilter(e: string){
const split = e.split(',');
const epLetter = 'M';
const inpMedia = [''];
// map select
e.map((e) => {
split.map((e) => {
if(e.match('-')){
const eRange = e.split('-');
if(eRange[0].match(/^m?\d+$/i)){
eRange[0] = eRange[0].replace(/^m/i,'');
eRange[0] = parseInt(eRange[0]);
eRange[0] = this.ifMaxEp(epLetter, eRange[0]) ? this.powNum(epLetter) - 1 : eRange[0];
eRange[0] = (this.ifMaxEp(epLetter, parseInt(eRange[0])) ? this.powNum(epLetter) - 1 : parseInt(eRange[0])).toString();
inpMedia.push(eRange[0].toString());
}
}
@ -102,21 +102,21 @@ class doFilter {
});
return [...new Set(inpMedia)].splice(1);
}
checkBetaFilter(e){
e = ['', ...e.split(',')];
e = e.map((e) => {
checkBetaFilter(e: string){
const e1 = ['', ...e.split(',')];
const e2 = e1.map((e) => {
if(e.match(betaEpRegex)){
return e;
}
return '';
});
e = [...new Set(e)].splice(1);
e = e.length > 100 ? e.slice(0, 100) : e;
return e;
const e3 = [...new Set(e2)].splice(1);
const e4 = e3.length > 100 ? e3.slice(0, 100) : e3;
return e4;
}
}
module.exports = {
export {
epNumLen,
doFilter,
};

View file

@ -7,7 +7,8 @@ export type Variable = ({
type: 'number',
replaceWith: number
} | {
type: 'string',
replaceWith: string
}) & {
name: AvailableFilenameVars
}
@ -19,7 +20,7 @@ const parseFileName = (input: string, variables: Variable[], numbers: number): s
return [input];
for (let i = 0; i < vars.length; i++) {
const type = vars[i];
const varName = type.slice(2, -1).toLowerCase();
const varName = type.slice(2, -1);
const use = variables.find(a => a.name === varName);
if (use === undefined) {
console.log(`[ERROR] Found variable '${type}' in fileName but no values was internally found!`)