Working downloads with queue system

This commit is contained in:
Izuco 2022-02-05 17:08:57 +01:00
parent 7dc72a1507
commit 40d07bb40d
No known key found for this signature in database
GPG key ID: E9CBE9E4EF3A1BFA
19 changed files with 267 additions and 96 deletions

View file

@ -1,6 +1,7 @@
import { HLSCallback } from 'hls-download';
import { sxItem } from '../crunchy';
import { LanguageItem } from '../modules/module.langsData';
import { DownloadInfo } from './messageHandler';
export type CrunchyDownloadOptions = {
hslang: string,
@ -11,7 +12,7 @@ export type CrunchyDownloadOptions = {
fileName: string,
numbers: number,
partsize: number,
callback?: HLSCallback,
callbackMaker?: (data: DownloadInfo) => HLSCallback,
timeout: number,
fsRetryTime: number,
dlsubs: string[],
@ -47,7 +48,8 @@ export type CrunchyEpMeta = {
seasonID: string,
season: number,
showID: string,
e: string
e: string,
image: string
}
export type DownloadedMedia = {

View file

@ -3,7 +3,8 @@ import { LanguageItem } from '../modules/module.langsData';
export type FunimationMediaDownload = {
id: string,
title: string,
showTitle: string
showTitle: string,
image: string
}
export type Subtitle = {

View file

@ -10,7 +10,8 @@ export interface MessageHandler {
handleDefault: (name: string) => Promise<any>,
resolveItems: (data: ResolveItemsData) => Promise<ResponseBase<QueueItem[]>>,
listEpisodes: (id: string) => Promise<EpisodeListResponse>,
downloadItem: (data) => void
downloadItem: (data) => void,
isDownloading: () => boolean
}
export type QueueItem = {
@ -68,8 +69,8 @@ export type FuniEpisodeData = {
export type AuthData = { username: string, password: string };
export type SearchData = { search: string, page?: number, 'search-type'?: string, 'search-locale'?: string };
export type FuniGetShowData = { id: number, e?: string, but: boolean, all: boolean };
export type FuniGetEpisodeData = { subs: FuniSubsData, fnSlug: FuniEpisodeData, callback?: HLSCallback, simul?: boolean; dubLang: string[], s: string }
export type FuniStreamData = { q: number, callback?: HLSCallback, x: number, fileName: string, numbers: number, novids?: boolean,
export type FuniGetEpisodeData = { subs: FuniSubsData, fnSlug: FuniEpisodeData, simul?: boolean; dubLang: string[], s: string }
export type FuniStreamData = { callbackMaker?: (data: DownloadInfo) => HLSCallback, q: number, x: number, fileName: string, numbers: number, novids?: boolean,
timeout: number, partsize: number, fsRetryTime: number, noaudio?: boolean, mp4: boolean, ass: boolean, fontSize: number, fontName?: string, skipmux?: boolean,
forceMuxer: AvailableMuxer | undefined, simul: boolean, skipSubMux: boolean, nocleanup: boolean }
export type FuniSubsData = { nosubs?: boolean, sub: boolean, dlsubs: string[] }
@ -98,4 +99,18 @@ export type ProgressData = {
downloadSpeed: number
};
export type PossibleMessanges = keyof ServiceHandler;
export type PossibleMessanges = keyof ServiceHandler;
export type DownloadInfo = {
image: string,
parent: {
title: string
},
title: string,
fileName: string
}
export type ExtendedProgress = {
progress: ProgressData,
downloadInfo: DownloadInfo
}

View file

@ -1,6 +1,13 @@
import { ProgressData } from "hls-download";
import { ExtendedProgress } from "./messageHandler";
export type RandomEvents = {
progress: ProgressData,
progress: ExtendedProgress,
finish: undefined
}
}
export interface RandomEvent<T extends keyof RandomEvents> {
name: T,
data: RandomEvents[T]
}
export type Handler<T extends keyof RandomEvents> = (data: RandomEvent<T>) => unknown;

View file

@ -1,3 +1,4 @@
- [ ] Hls-Download force yes or no on rewrite promt
- [ ] Pick up if a download is currently in progress
- [ ] Send more information with the progress event like the title and image to display more information
- [ ] Hls-Download force yes or no on rewrite promt as well as for mkvmerge/ffmpeg
- [x] Pick up if a download is currently in progress
- [x] Send more information with the progress event like the title and image to display more information
- [ ] Use Click away listener for the search popup

View file

@ -738,6 +738,7 @@ export default class Crunchy implements ServiceClass {
: '' + parseInt(epNum, 10).toString().padStart(epNumLen, '0')
);
// set data
const images = (item.images.thumbnail ?? [[ { source: '/notFound.png' } ]])[0];
const epMeta: CrunchyEpMeta = {
data: [
{
@ -750,7 +751,8 @@ export default class Crunchy implements ServiceClass {
seasonID: item.season_id,
season: item.season_number,
showID: id,
e: selEpId
e: selEpId,
image: images[Math.floor(images.length / 2)].source
};
if(item.playback){
epMeta.data[0].playback = item.playback;
@ -779,7 +781,7 @@ export default class Crunchy implements ServiceClass {
if (res === undefined) {
return false;
} else {
this.muxStreams(res.data, { ...options, output: res.fileName });
await this.muxStreams(res.data, { ...options, output: res.fileName });
downloaded({
service: 'crunchy',
type: 's'
@ -1138,7 +1140,14 @@ export default class Crunchy implements ServiceClass {
// baseurl: chunkPlaylist.baseUrl,
threads: options.partsize,
fsRetryTime: options.fsRetryTime * 1000,
callback: options.callback
callback: options.callbackMaker ? options.callbackMaker({
fileName: `${path.isAbsolute(outFile) ? outFile.slice(this.cfg.dir.content.length) : outFile}`,
image: medias.image,
parent: {
title: medias.seasonTitle
},
title: medias.episodeTitle
}) : undefined
}).download();
if(!dlStreamByPl.ok){
console.log(`[ERROR] DL Stats: ${JSON.stringify(dlStreamByPl.parts)}\n`);
@ -1390,6 +1399,7 @@ export default class Crunchy implements ServiceClass {
}
const epNum = key.startsWith('E') ? key.slice(1) : key;
// set data
const images = (item.images.thumbnail ?? [[ { source: '/notFound.png' } ]])[0];
const epMeta: CrunchyEpMeta = {
data: [
{
@ -1402,7 +1412,8 @@ export default class Crunchy implements ServiceClass {
seasonID: item.season_id,
season: item.season_number,
showID: item.series_id,
e: epNum
e: epNum,
image: images[Math.floor(images.length / 2)].source
};
if(item.playback){
epMeta.data[0].playback = item.playback;
@ -1528,12 +1539,4 @@ export default class Crunchy implements ServiceClass {
return episodeList;
}
}
/*
// MULTI DOWNLOADING
const
const
*/
}

21
funi.ts
View file

@ -449,7 +449,8 @@ export default class Funi implements ServiceClass {
const res = await this.downloadStreams(true, {
id: data.fnSlug.episodeID,
title: ep.title,
showTitle: ep.parent.title
showTitle: ep.parent.title,
image: ep.thumb
}, downloadData);
if (res === true) {
downloaded({
@ -671,7 +672,14 @@ export default class Funi implements ServiceClass {
const chunkList = m3u8(reqVideo.res.body);
const tsFile = path.join(this.cfg.dir.content, ...fnOutput.slice(0, -1), `${fnOutput.slice(-1)}.video${(plAud?.uri ? '' : '.' + streamPath.lang.code )}`);
dlFailed = !await this.downloadFile(tsFile, chunkList, data.timeout, data.partsize, data.fsRetryTime, data.callback);
dlFailed = !await this.downloadFile(tsFile, chunkList, data.timeout, data.partsize, data.fsRetryTime, data.callbackMaker ? data.callbackMaker({
fileName: `${fnOutput.slice(-1)}.video${(plAud?.uri ? '' : '.' + streamPath.lang.code )}.ts`,
parent: {
title: epsiode.showTitle
},
title: epsiode.title,
image: epsiode.image
}) : undefined);
if (!dlFailed) {
if (plAud) {
purvideo.push({
@ -704,7 +712,14 @@ export default class Funi implements ServiceClass {
const tsFileA = path.join(this.cfg.dir.content, ...fnOutput.slice(0, -1), `${fnOutput.slice(-1)}.audio.${plAud.language.code}`);
dlFailedA = !await this.downloadFile(tsFileA, chunkListA, data.timeout, data.partsize, data.fsRetryTime, data.callback);
dlFailedA = !await this.downloadFile(tsFileA, chunkListA, data.timeout, data.partsize, data.fsRetryTime, data.callbackMaker ? data.callbackMaker({
fileName: `${fnOutput.slice(-1)}.audio.${plAud.language.code}.ts`,
parent: {
title: epsiode.showTitle
},
title: epsiode.title,
image: epsiode.image
}) : undefined);
if (!dlFailedA)
puraudio.push({
path: `${tsFileA}.ts`,

View file

@ -22,9 +22,5 @@ export default (window: BrowserWindow) => {
ipcMain.handle('resolveItems', async (_, data) => handler?.resolveItems(data));
ipcMain.handle('listEpisodes', async (_, data) => handler?.listEpisodes(data));
ipcMain.handle('downloadItem', async (_, data) => handler?.downloadItem(data));
/*
setInterval(() => {
mainWindow?.webContents.send('progress', { hello: 'World' });
}, 1000);
*/
ipcMain.on('isDownloading', (ev) => ev.returnValue = handler?.isDownloading());
};

View file

@ -1,5 +1,6 @@
import { BrowserWindow, dialog } from "electron";
import { ProgressData } from "../../../../@types/messageHandler";
import { DownloadInfo, ExtendedProgress, ProgressData } from "../../../../@types/messageHandler";
import { RandomEvent, RandomEvents } from "../../../../@types/randomEvents";
export default class Base {
@ -24,12 +25,28 @@ export default class Base {
})
}
handleProgress(data: ProgressData) {
this.window.webContents.send('progress', data);
makeProgressHandler(videoInfo: DownloadInfo) {
return ((data: ProgressData) => {
this.sendMessage({
name: 'progress',
data: {
downloadInfo: videoInfo,
progress: data
}
})
}).bind(this);
}
getWindow() {
return this.window;
}
sendMessage<T extends keyof RandomEvents>(data: RandomEvent<T>) {
this.window.webContents.send('randomEvent', data);
}
isDownloading() {
return this.downloading;
}
}

View file

@ -75,7 +75,7 @@ class CrunchyHandler extends Base implements MessageHandler {
});
if (res.isOk) {
for (const select of res.value) {
if (!(await this.crunchy.downloadEpisode(select, {..._default, skipsubs: false, callback: this.handleProgress.bind(this), q: data.q, fileName: data.fileName }))) {
if (!(await this.crunchy.downloadEpisode(select, {..._default, skipsubs: false, callbackMaker: this.makeProgressHandler.bind(this), q: data.q, fileName: data.fileName }))) {
const er = new Error(`Unable to download episode ${data.e} from ${data.id}`);
er.name = 'Download error';
this.alertError(er);
@ -84,7 +84,7 @@ class CrunchyHandler extends Base implements MessageHandler {
} else {
this.alertError(res.reason);
}
this.getWindow().webContents.send('finish');
this.sendMessage({ name: 'finish', data: undefined })
this.setDownloading(false);
}
}

View file

@ -87,9 +87,9 @@ class FunimationHandler extends Base implements MessageHandler {
return this.alertError(res.reason);
for (const ep of res.value) {
await this.funi.getEpisode(false, { dubLang: data.dubLang, fnSlug: ep, s: data.id, subs: { dlsubs: ['all'], sub: false }, callback: this.handleProgress.bind(this) }, { ..._default, ass: true, callback: this.handleProgress.bind(this), fileName: data.fileName, q: data.q })
await this.funi.getEpisode(false, { dubLang: data.dubLang, fnSlug: ep, s: data.id, subs: { dlsubs: ['all'], sub: false } }, { ..._default, callbackMaker: this.makeProgressHandler.bind(this), ass: true, fileName: data.fileName, q: data.q })
}
this.getWindow().webContents.send('finish');
this.sendMessage({ name: 'finish', data: undefined })
this.setDownloading(false);
};
}

View file

@ -1,5 +1,6 @@
import React from "react";
import { ProgressData } from "../../../../../../@types/messageHandler";
import { ExtendedProgress, ProgressData } from "../../../../../../@types/messageHandler";
import { RandomEvent } from "../../../../../../@types/randomEvents";
import useStore from "../../../hooks/useStore";
import { messageChannelContext } from "../../../provider/MessageChannel";
@ -7,17 +8,21 @@ const useDownloadManager = () => {
const [ { currentDownload }, dispatch ] = useStore();
const messageHandler = React.useContext(messageChannelContext);
const [progressData, setProgressData] = React.useState<ProgressData|undefined>();
const [progressData, setProgressData] = React.useState<ExtendedProgress|undefined>();
React.useEffect(() => {
const handler = (data: ProgressData) => {
console.log(data);
setProgressData(data);
const handler = (ev: RandomEvent<'progress'>) => {
console.log(ev.data);
setProgressData(ev.data);
}
messageHandler?.randomEvents.on('progress', handler);
const finishHandler = () => {
console.log('DONE');
setProgressData(undefined);
dispatch({
type: 'finish',
payload: undefined
})
}
messageHandler?.randomEvents.on('finish', finishHandler);
@ -30,6 +35,8 @@ const useDownloadManager = () => {
React.useEffect(() => {
if (!currentDownload)
return;
if (messageHandler?.isDownloading())
return;
messageHandler?.downloadItem(currentDownload);
}, [currentDownload]);

View file

@ -1,7 +1,7 @@
import React from "react";
import { Box, Button, Checkbox, Chip, FormControl, FormControlLabel, IconButton, InputLabel, MenuItem, OutlinedInput, Select, TextField } from "@mui/material";
import useStore from "../../../hooks/useStore";
import MultiSelect from "../../MultiSelect";
import MultiSelect from "../../reusable/MultiSelect";
import { messageChannelContext } from "../../../provider/MessageChannel";
import LoadingButton from '@mui/lab/LoadingButton';
import { useSnackbar } from "notistack";
@ -69,7 +69,7 @@ const DownloadSelector: React.FC = () => {
setLoading(false);
}
console.log(store.queue);
console.log(store.queue, store.currentDownload);
return <Box sx={{ display: 'flex', flexDirection: 'column' }}>
<Box sx={{ m: 2, gap: 1, display: 'flex', justifyContent: 'center', alignItems: 'center', flexWrap: 'wrap' }}>

View file

@ -7,13 +7,17 @@ import Progress from "./Progress/Progress";
import SearchBox from "./SearchBox/SearchBox";
const MainFrame: React.FC = () => {
return <Box sx={{ border: '2px solid white', width: '75%' }}>
<SearchBox />
<Divider variant='middle' className="divider-width" light sx={{ color: 'text.primary', fontSize: '1.2rem' }}>Options</Divider>
<DownloadSelector />
<Divider variant='middle' className="divider-width" light />
<Bottom />
<Progress />
return <Box sx={{ display: 'grid', gridTemplateColumns: '3fr 1fr', borderCollapse: 'collapse' }}>
<Box sx={{ border: '2px solid white' }}>
<SearchBox />
<Divider variant='middle' className="divider-width" light sx={{ color: 'text.primary', fontSize: '1.2rem' }}>Options</Divider>
<DownloadSelector />
<Divider variant='middle' className="divider-width" light />
<Bottom />
</Box>
<Box sx={{ marginLeft: 1 }}>
<Progress />
</Box>
</Box>
}

View file

@ -1,12 +1,93 @@
import { Close } from "@mui/icons-material";
import { Box, Typography } from "@mui/material";
import React from "react";
import { messageChannelContext } from "../../../provider/MessageChannel";
import LinearProgressWithLabel from "../../reusable/LinearProgressWithLabel";
import useDownloadManager from "../DownloadManager/DownloadManager";
const Progress: React.FC = () => {
useDownloadManager();
const data = useDownloadManager();
return <></>
return data ? <Box sx={{ display: 'grid', gridTemplateRows: '1fr 2fr', height: '100%' }}>
<img style={{ maxWidth: '100%', maxHeight: '100%', border: '2px solid white', padding: 8 }} src={data.downloadInfo.image}></img>
<Box sx={{ display: 'grid', gridTemplateRows: '1ft fit-content', gap: 1 }}>
<table>
<tbody style={{ verticalAlign: 'text-top' }}>
<tr>
<td>
<Typography color='text.primary'>
Title:
</Typography>
</td>
<td>
<Typography color='text.primary'>
{data.downloadInfo.title}
</Typography>
</td>
</tr>
<tr>
<td>
<Typography color='text.primary'>
Season:
</Typography>
</td>
<td>
<Typography color='text.primary'>
{data.downloadInfo.parent.title}
</Typography>
</td>
</tr>
<tr>
<td>
<Typography color='text.primary' sx={{ verticalAlign: 'text-top' }}>
Filename:
</Typography>
</td>
<td>
<Typography color='text.primary'>
{data.downloadInfo.fileName}
</Typography>
</td>
</tr>
<tr>
<td>
<Typography color='text.primary' sx={{ verticalAlign: 'text-top' }}>
Progress:
</Typography>
</td>
<td>
<Typography color='text.primary'>
{`${data.progress.cur} / ${data.progress.total} - ${data.progress.percent}%`}
</Typography>
</td>
</tr>
</tbody>
</table>
<Box>
<Typography color='text.primary'>
{`ETA ${formatTime(data.progress.time)}`}
</Typography>
<Box sx={{ width: '100%' }}>
<LinearProgressWithLabel sx={{ height: '10px' }} value={typeof data.progress.percent === 'string' ? parseInt(data.progress.percent) : data.progress.percent} />
</Box>
</Box>
</Box>
</Box> : <Box sx={{ width: '100%', height: '100%', display: 'flex', justifyContent: 'center', alignItems: 'center' }}>
<Close color='primary' fontSize="large"/>
</Box>
}
const formatTime = (time: number) => {
let seconds = Math.ceil(time / 1000);
let minutes = 0;
if (seconds >= 60) {
minutes = Math.floor(seconds / 60);
seconds = seconds % 60;
}
return `${minutes != 0 ? `${minutes} Minutes ` : ''}${seconds !== 0 ? `${seconds} Seconds` : ''}`
}
export default Progress;

View file

@ -0,0 +1,24 @@
import { LinearProgressProps, Box, LinearProgress, Typography } from '@mui/material';
import React from 'react';
// The following code has been taken from the mui example
// Thanks for that mui
export type LinearProgressWithLabelProps = LinearProgressProps & { value: number };
const LinearProgressWithLabel: React.FC<LinearProgressWithLabelProps> = (props) => {
return (
<Box sx={{ display: 'flex', alignItems: 'center' }}>
<Box sx={{ width: '100%', mr: 1 }}>
<LinearProgress variant="determinate" {...props} />
</Box>
<Box sx={{ minWidth: 35 }}>
<Typography variant="body2" color="text.secondary">{`${Math.round(
props.value,
)}%`}</Typography>
</Box>
</Box>
);
}
export default LinearProgressWithLabel;

View file

@ -3,44 +3,33 @@ import type { MessageHandler } from '../../../../@types/messageHandler';
import type { IpcRenderer, IpcRendererEvent } from "electron";
import useStore from '../hooks/useStore';
import type { RandomEvents } from '../../../../@types/randomEvents';
import type { Handler, RandomEvent, RandomEvents } from '../../../../@types/randomEvents';
export type Handler<T> = (data: T) => unknown;
export type FrontEndMessanges = (MessageHandler & { randomEvents: RandomEventHandler });
export class RandomEventHandler {
private handler: {
[eventName in keyof RandomEvents]: Handler<RandomEvents[eventName]>[]
[eventName in keyof RandomEvents]: Handler<eventName>[]
} = {
progress: [],
finish: []
};
private allHandler: Handler<unknown>[] = [];
public on<T extends keyof RandomEvents>(name: T, listener: Handler<RandomEvents[T]>) {
public on<T extends keyof RandomEvents>(name: T, listener: Handler<T>) {
if (Object.prototype.hasOwnProperty.call(this.handler, name)) {
this.handler[name].push(listener);
this.handler[name].push(listener as any);
} else {
this.handler[name] = [ listener ];
this.handler[name] = [ listener as any ];
}
}
public emit<T extends keyof RandomEvents>(name: T, data: RandomEvents[T]) {
(this.handler[name] ?? []).forEach(handler => handler(data));
this.allHandler.forEach(handler => handler(data));
public emit<T extends keyof RandomEvents>(name: keyof RandomEvents, data: RandomEvent<T>) {
(this.handler[name] ?? []).forEach(handler => handler(data as any));
}
public removeListener<T extends keyof RandomEvents>(name: T, listener: Handler<RandomEvents[T]>) {
this.handler[name] = this.handler[name].filter(a => a !== listener);
}
public onAll(listener: Handler<unknown>) {
this.allHandler.push(listener);
}
public removeAllListener(listener: Handler<unknown>) {
this.allHandler = this.allHandler.filter(a => a !== listener);
public removeListener<T extends keyof RandomEvents>(name: T, listener: Handler<T>) {
this.handler[name] = (this.handler[name] as Handler<T>[]).filter(a => a !== listener) as any;
}
}
@ -53,10 +42,6 @@ const MessageChannelProvider: React.FC = ({ children }) => {
const { ipcRenderer } = (window as any).Electron as { ipcRenderer: IpcRenderer };
const [ randomEventHandler ] = React.useState(new RandomEventHandler());
const buildListener = (event: keyof RandomEvents) => {
return (_: IpcRendererEvent, ...data: any[]) => randomEventHandler.emit(event, data.length === 0 ? undefined : data[0]);
}
React.useEffect(() => {
(async () => {
const currentService = await ipcRenderer.invoke('type');
@ -68,13 +53,16 @@ const MessageChannelProvider: React.FC = ({ children }) => {
}, [store.service])
React.useEffect(() => {
const progressListener = buildListener('progress');
const finishListener = buildListener('finish');
ipcRenderer.on('progress', progressListener);
ipcRenderer.on('finish', finishListener);
/* finish is a placeholder */
const listener = (_: IpcRendererEvent, initalData: RandomEvent<'finish'>) => {
const eventName = initalData.name as keyof RandomEvents;
const data = initalData as unknown as RandomEvent<typeof eventName>;
randomEventHandler.emit(data.name, data);
}
ipcRenderer.on('randomEvent', listener);
return () => {
ipcRenderer.removeListener('progress', progressListener);
ipcRenderer.removeListener('finish', finishListener);
ipcRenderer.removeListener('randomEvent', listener);
};
}, [ ipcRenderer ]);
@ -88,7 +76,8 @@ const MessageChannelProvider: React.FC = ({ children }) => {
resolveItems: async (data) => await ipcRenderer.invoke('resolveItems', data),
listEpisodes: async (data) => await ipcRenderer.invoke('listEpisodes', data),
randomEvents: randomEventHandler,
downloadItem: (data) => ipcRenderer.invoke('downloadItem', data)
downloadItem: (data) => ipcRenderer.invoke('downloadItem', data),
isDownloading: () => ipcRenderer.sendSync('isDownloading')
}
return <messageChannelContext.Provider value={messageHandler}>

View file

@ -17,7 +17,8 @@ export type StoreState = {
episodeListing: Episode[];
downloadOptions: DownloadOptions,
service: 'crunchy'|'funi'|undefined,
currentDownload?: QueueItem
currentDownload?: QueueItem,
finish?: undefined
}
export type StoreAction<T extends keyof StoreState> = {
@ -28,12 +29,20 @@ export type StoreAction<T extends keyof StoreState> = {
const Reducer = <T extends keyof StoreState,>(state: StoreState, action: StoreAction<T>): StoreState => {
switch(action.type) {
case "queue":
let queue = state.queue.concat(action.payload as QueueItem[]);
if (!state.currentDownload && queue.length > 0) {
state.currentDownload = queue[0];
queue = queue.slice(1);
state.queue = state.queue.concat(action.payload as QueueItem[]);
if (state.currentDownload === undefined && state.queue.length > 0) {
state.currentDownload = state.queue[0];
state.queue = state.queue.slice(1);
}
return { ...state, queue };
return { ...state };
case "finish":
if (state.queue.length > 0) {
state.currentDownload = state.queue[0];
state.queue = state.queue.slice(1);
} else {
state.currentDownload = undefined;
}
return { ...state }
default:
return { ...state, [action.type]: action.payload }
}
@ -51,7 +60,7 @@ const initialState: StoreState = {
but: false
},
service: undefined,
episodeListing: []
episodeListing: [],
};
const Store: React.FC = ({children}) => {