From 5ae7a8d58108f47bcefad1f9439270d5ab2376f0 Mon Sep 17 00:00:00 2001 From: Schnitzel5 Date: Tue, 7 Jan 2025 23:37:29 +0100 Subject: [PATCH] added option to download novels --- lib/eval/dart/bridge/m_provider.dart | 11 +++ lib/eval/dart/service.dart | 5 ++ lib/eval/interface.dart | 2 + lib/eval/javascript/service.dart | 12 +++ lib/eval/model/m_provider.dart | 2 + lib/modules/browse/extension/edit_code.dart | 13 ++- .../extension/widgets/create_extension.dart | 10 +++ .../update_manga_detail_providers.g.dart | 2 +- .../manga/download/download_page_widget.dart | 10 +++ .../download/providers/download_provider.dart | 87 ++++++++++++++++--- .../providers/download_provider.g.dart | 2 +- lib/services/fetch_novel_sources.g.dart | 2 +- lib/services/get_html_content.dart | 25 ++++-- lib/services/get_html_content.g.dart | 2 +- 14 files changed, 161 insertions(+), 24 deletions(-) diff --git a/lib/eval/dart/bridge/m_provider.dart b/lib/eval/dart/bridge/m_provider.dart index 8713b86..03a2b41 100644 --- a/lib/eval/dart/bridge/m_provider.dart +++ b/lib/eval/dart/bridge/m_provider.dart @@ -106,6 +106,13 @@ class $MProvider extends MProvider with $Bridge { BridgeParameter('url', BridgeTypeAnnotation(BridgeTypeRef(CoreTypes.string)), false), ])), + 'cleanHtmlContent': BridgeMethodDef(BridgeFunctionDef( + returns: BridgeTypeAnnotation(BridgeTypeRef( + CoreTypes.future, [BridgeTypeRef(CoreTypes.string)])), + params: [ + BridgeParameter('html', + BridgeTypeAnnotation(BridgeTypeRef(CoreTypes.string)), false), + ])), 'getFilterList': BridgeMethodDef(BridgeFunctionDef( returns: BridgeTypeAnnotation(BridgeTypeRef( CoreTypes.list, [BridgeTypeRef(CoreTypes.dynamic)])), @@ -1188,6 +1195,10 @@ class $MProvider extends MProvider with $Bridge { Future getHtmlContent(String url) async => await $_invoke('getHtmlContent', [$String(url)]); + @override + Future cleanHtmlContent(String html) async => + await $_invoke('cleanHtmlContent', [$String(html)]); + @override Map get headers { try { diff --git a/lib/eval/dart/service.dart b/lib/eval/dart/service.dart index ad8f6b6..b278601 100644 --- a/lib/eval/dart/service.dart +++ b/lib/eval/dart/service.dart @@ -120,6 +120,11 @@ class DartExtensionService implements ExtensionService { return await _executeLib().getHtmlContent(url); } + @override + Future cleanHtmlContent(String html) async { + return await _executeLib().cleanHtmlContent(html); + } + @override FilterList getFilterList() { List list; diff --git a/lib/eval/interface.dart b/lib/eval/interface.dart index f196164..0f14770 100644 --- a/lib/eval/interface.dart +++ b/lib/eval/interface.dart @@ -31,6 +31,8 @@ abstract interface class ExtensionService { Future getHtmlContent(String url); + Future cleanHtmlContent(String html); + FilterList getFilterList(); List getSourcePreferences(); diff --git a/lib/eval/javascript/service.dart b/lib/eval/javascript/service.dart index 9f10b87..5d8f9cb 100644 --- a/lib/eval/javascript/service.dart +++ b/lib/eval/javascript/service.dart @@ -63,6 +63,9 @@ class MProvider { async getHtmlContent(url) { throw new Error("getHtmlContent not implemented"); } + async cleanHtmlContent(html) { + throw new Error("cleanHtmlContent not implemented"); + } getFilterList() { throw new Error("getFilterList not implemented"); } @@ -147,6 +150,15 @@ var extention = new DefaultExtension(); return res; } + @override + Future cleanHtmlContent(String html) async { + _init(); + final res = (await runtime.handlePromise(await runtime.evaluateAsync( + 'jsonStringify(() => extention.cleanHtmlContent(`$html`))'))) + .stringResult; + return res; + } + @override FilterList getFilterList() { List list; diff --git a/lib/eval/model/m_provider.dart b/lib/eval/model/m_provider.dart index b7a0378..61e29c4 100644 --- a/lib/eval/model/m_provider.dart +++ b/lib/eval/model/m_provider.dart @@ -26,6 +26,8 @@ abstract class MProvider { Future getHtmlContent(String url); + Future cleanHtmlContent(String html); + List getFilterList(); List getSourcePreferences(); diff --git a/lib/modules/browse/extension/edit_code.dart b/lib/modules/browse/extension/edit_code.dart index 60ee6f0..66fcf3d 100644 --- a/lib/modules/browse/extension/edit_code.dart +++ b/lib/modules/browse/extension/edit_code.dart @@ -51,13 +51,15 @@ class _CodeEditorState extends ConsumerState { ("getDetail", 3), ("getPageList", 4), ("getVideoList", 5), - ("getHtmlContent", 6) + ("getHtmlContent", 6), + ("cleanHtmlContent", 7) ]; int _serviceIndex = 0; int _page = 1; String _query = ""; String _url = ""; + String _html = ""; bool _isLoading = false; String _errorText = ""; bool _error = false; @@ -226,6 +228,10 @@ class _CodeEditorState extends ConsumerState { (v) { _url = v; }), + if (_serviceIndex == 7) + _textEditing("Html", context, "ex.

Text

", (v) { + _html = v; + }), Padding( padding: const EdgeInsets.all(8.0), child: Wrap( @@ -293,9 +299,12 @@ class _CodeEditorState extends ConsumerState { (await service.getVideoList(_url)) .map((e) => e.toJson()) .toList(); - } else { + } else if (_serviceIndex == 6) { result = (await service .getHtmlContent(_url)); + } else { + result = (await service + .cleanHtmlContent(_html)); } if (mounted) { setState(() { diff --git a/lib/modules/browse/extension/widgets/create_extension.dart b/lib/modules/browse/extension/widgets/create_extension.dart index ec203e1..4d5913c 100644 --- a/lib/modules/browse/extension/widgets/create_extension.dart +++ b/lib/modules/browse/extension/widgets/create_extension.dart @@ -284,6 +284,12 @@ class TestSource extends MProvider { // TODO: implement } + // Clean html up for reader + @override + Future cleanHtmlContent(String html) async { + // TODO: implement + } + // For anime episode video list @override Future> getVideoList(String url) async { @@ -349,6 +355,10 @@ class DefaultExtension extends MProvider { async getHtmlContent(url) { throw new Error("getHtmlContent not implemented"); } + // Clean html up for reader + async cleanHtmlContent(html) { + throw new Error("cleanHtmlContent not implemented"); + } // For anime episode video list async getVideoList(url) { throw new Error("getVideoList not implemented"); diff --git a/lib/modules/manga/detail/providers/update_manga_detail_providers.g.dart b/lib/modules/manga/detail/providers/update_manga_detail_providers.g.dart index 15f19ab..2ffd011 100644 --- a/lib/modules/manga/detail/providers/update_manga_detail_providers.g.dart +++ b/lib/modules/manga/detail/providers/update_manga_detail_providers.g.dart @@ -6,7 +6,7 @@ part of 'update_manga_detail_providers.dart'; // RiverpodGenerator // ************************************************************************** -String _$updateMangaDetailHash() => r'dcc5fd8f666959f62ee9ad6540eb0493ac9759f1'; +String _$updateMangaDetailHash() => r'a86fe8fea46e411203182287c970cd80cc9a1a0c'; /// Copied from Dart SDK class _SystemHash { diff --git a/lib/modules/manga/download/download_page_widget.dart b/lib/modules/manga/download/download_page_widget.dart index fa63ecc..d2fe76c 100644 --- a/lib/modules/manga/download/download_page_widget.dart +++ b/lib/modules/manga/download/download_page_widget.dart @@ -53,10 +53,13 @@ class _ChapterPageDownloadState extends ConsumerState final cbzFile = File(p.join(mangaDir!.path, "${widget.chapter.name}.cbz")); final mp4File = File(p.join(mangaDir.path, "${widget.chapter.name!.replaceForbiddenCharacters(' ')}.mp4")); + final htmlFile = File(p.join(mangaDir.path, "${widget.chapter.name}.html")); if (cbzFile.existsSync()) { files = [XFile(cbzFile.path)]; } else if (mp4File.existsSync()) { files = [XFile(mp4File.path)]; + } else if (htmlFile.existsSync()) { + files = [XFile(htmlFile.path)]; } else { files = path!.listSync().map((e) => XFile(e.path)).toList(); } @@ -86,6 +89,13 @@ class _ChapterPageDownloadState extends ConsumerState mp4File.deleteSync(); } } catch (_) {} + try { + final htmlFile = + File(p.join(mangaDir!.path, "${widget.chapter.name}.html")); + if (htmlFile.existsSync()) { + htmlFile.deleteSync(); + } + } catch (_) {} path!.deleteSync(recursive: true); } catch (_) {} isar.writeTxnSync(() { diff --git a/lib/modules/manga/download/providers/download_provider.dart b/lib/modules/manga/download/providers/download_provider.dart index dfeecec..ff756f3 100644 --- a/lib/modules/manga/download/providers/download_provider.dart +++ b/lib/modules/manga/download/providers/download_provider.dart @@ -31,6 +31,8 @@ Future> downloadChapter( required Chapter chapter, bool? useWifi, }) async { + final http = MClient.init( + reqcopyWith: {'useDartHttpClient': true, 'followRedirects': false}); List pageUrls = []; List tasks = []; final StorageProvider storageProvider = StorageProvider(); @@ -48,7 +50,6 @@ Future> downloadChapter( final chapterName = chapter.name!.replaceForbiddenCharacters(' '); final itemType = chapter.manga.value!.itemType; - final isManga = itemType == ItemType.manga; final itemTypePath = itemType == ItemType.manga ? "Manga" : itemType == ItemType.anime @@ -60,13 +61,18 @@ Future> downloadChapter( "${manga.source} (${manga.lang!.toUpperCase()})", manga.name!.replaceForbiddenCharacters('_'), ]; - if (isManga) { + if (itemType == ItemType.manga) { pathSegments.add(scanlator); pathSegments.add(chapter.name!.replaceForbiddenCharacters('_')); } final finalPath = p.joinAll(pathSegments); path = Directory(p.join(path1!.path, finalPath)); Map videoHeader = {}; + Map htmlHeader = { + "Priority": "u=0, i", + "User-Agent": + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36", + }; bool hasM3U8File = false; bool nonM3U8File = false; M3u8Downloader? m3u8Downloader; @@ -75,6 +81,7 @@ Future> downloadChapter( int? m3u8MediaSequence; Future processConvert() async { + if (itemType == ItemType.novel) return; if (hasM3U8File) { await m3u8Downloader?.mergeTsToMp4(p.join(path!.path, "$chapterName.mp4"), p.join(path.path, chapterName)); @@ -109,7 +116,7 @@ Future> downloadChapter( isar.settings.putSync(settings..chapterPageUrlsList = chapterPageUrls)); } - if (isManga) { + if (itemType == ItemType.manga) { ref .read(getChapterPagesProvider( chapter: chapter, @@ -120,7 +127,7 @@ Future> downloadChapter( isOk = true; } }); - } else { + } else if (itemType == ItemType.anime) { ref.read(getVideoListProvider(episode: chapter).future).then((value) async { final m3u8Urls = value.$1 .where((element) => @@ -150,6 +157,25 @@ Future> downloadChapter( isOk = true; } }); + } else if (itemType == ItemType.novel && chapter.url != null) { + final cookie = MClient.getCookiesPref(chapter.url!); + final headers = itemType == ItemType.manga + ? ref.watch(headersProvider(source: manga.source!, lang: manga.lang!)) + : itemType == ItemType.anime + ? videoHeader + : htmlHeader; + if (cookie.isNotEmpty) { + final userAgent = isar.settings.getSync(227)!.userAgent!; + headers.addAll(cookie); + headers[HttpHeaders.userAgentHeader] = userAgent; + } + final res = await http.get(Uri.parse(chapter.url!), headers: headers); + if (res.headers.containsKey("Location")) { + pageUrls = [PageUrl(res.headers["Location"]!)]; + } else { + pageUrls = [PageUrl(chapter.url!)]; + } + isOk = true; } await Future.doWhile(() async { @@ -166,12 +192,20 @@ Future> downloadChapter( ref.watch(saveAsCBZArchiveStateProvider); bool mp4FileExist = await File(p.join(mangaDir.path, "$chapterName.mp4")).exists(); - if (!cbzFileExist && isManga || !mp4FileExist && !isManga) { + bool htmlFileExist = + await File(p.join(mangaDir.path, "$chapterName.html")).exists(); + if (!cbzFileExist && itemType == ItemType.manga || + !mp4FileExist && itemType == ItemType.anime || + !htmlFileExist && itemType == ItemType.novel) { for (var index = 0; index < pageUrls.length; index++) { final path2 = Directory(p.join( path1.path, "downloads", - isManga ? "Manga" : "Anime", + itemType == ItemType.manga + ? "Manga" + : itemType == ItemType.anime + ? "Anime" + : "Novel", "${manga.source} (${manga.lang!.toUpperCase()})", manga.name!.replaceForbiddenCharacters('_'))); if (!(await path2.exists())) { @@ -184,10 +218,12 @@ Future> downloadChapter( } final page = pageUrls[index]; final cookie = MClient.getCookiesPref(page.url); - final headers = isManga + final headers = itemType == ItemType.manga ? ref.watch( headersProvider(source: manga.source!, lang: manga.lang!)) - : videoHeader; + : itemType == ItemType.anime + ? videoHeader + : htmlHeader; if (cookie.isNotEmpty) { final userAgent = isar.settings.getSync(227)!.userAgent!; headers.addAll(cookie); @@ -196,7 +232,7 @@ Future> downloadChapter( Map pageHeaders = headers; pageHeaders.addAll(page.headers ?? {}); - if (isManga) { + if (itemType == ItemType.manga) { final file = File(p.join(tempDir.path, "Mangayomi", finalPath, "${padIndex(index + 1)}.jpg")); if (file.existsSync()) { @@ -222,7 +258,7 @@ Future> downloadChapter( requiresWiFi: onlyOnWifi)); } } - } else { + } else if (itemType == ItemType.anime) { final file = File( p.join(tempDir.path, "Mangayomi", finalPath, "$chapterName.mp4")); if (file.existsSync()) { @@ -270,6 +306,31 @@ Future> downloadChapter( requiresWiFi: onlyOnWifi)); } } + } else { + final file = File(p.join( + tempDir.path, "Mangayomi", finalPath, "$chapterName.html")); + if (file.existsSync()) { + await file.copy(p.join(path.path, "$chapterName.html")); + await file.delete(); + } else { + if (!(await path.exists())) { + await path.create(); + } + if (!(await File(p.join(path.path, "$chapterName.html")) + .exists())) { + tasks.add(DownloadTask( + taskId: page.url, + headers: pageHeaders, + url: page.url.trim().trimLeft().trimRight(), + filename: "$chapterName.html", + baseDirectory: BaseDirectory.temporary, + directory: p.join("Mangayomi", finalPath), + updates: Updates.statusAndProgress, + allowPause: true, + retries: 3, + requiresWiFi: onlyOnWifi)); + } + } } } } @@ -298,7 +359,9 @@ Future> downloadChapter( await FileDownloader().downloadBatch( tasks, batchProgressCallback: (succeeded, failed) async { - if (isManga || hasM3U8File) { + if (itemType == ItemType.manga || + itemType == ItemType.novel || + hasM3U8File) { if (succeeded == tasks.length) { await processConvert(); } @@ -335,7 +398,7 @@ Future> downloadChapter( }, taskProgressCallback: (taskProgress) async { final progress = taskProgress.progress; - if (!isManga && !hasM3U8File) { + if (itemType == ItemType.anime && !hasM3U8File) { bool isEmpty = isar.downloads .filter() .chapterIdEqualTo(chapter.id!) diff --git a/lib/modules/manga/download/providers/download_provider.g.dart b/lib/modules/manga/download/providers/download_provider.g.dart index 4ae49a4..938860a 100644 --- a/lib/modules/manga/download/providers/download_provider.g.dart +++ b/lib/modules/manga/download/providers/download_provider.g.dart @@ -6,7 +6,7 @@ part of 'download_provider.dart'; // RiverpodGenerator // ************************************************************************** -String _$downloadChapterHash() => r'de8e2d5b952071bc0d014fc3aa5c9b0714fbcee0'; +String _$downloadChapterHash() => r'2873b00f9f4d0fd91bc90a28e2700a6c0d187a46'; /// Copied from Dart SDK class _SystemHash { diff --git a/lib/services/fetch_novel_sources.g.dart b/lib/services/fetch_novel_sources.g.dart index c212a85..e9c72a2 100644 --- a/lib/services/fetch_novel_sources.g.dart +++ b/lib/services/fetch_novel_sources.g.dart @@ -7,7 +7,7 @@ part of 'fetch_novel_sources.dart'; // ************************************************************************** String _$fetchNovelSourcesListHash() => - r'cc4b989c0248c3b16155444c0c429d1ed0025ecb'; + r'1444d9ca12204ccf5389efe085c8a20e3498a808'; /// Copied from Dart SDK class _SystemHash { diff --git a/lib/services/get_html_content.dart b/lib/services/get_html_content.dart index 5501250..111fabb 100644 --- a/lib/services/get_html_content.dart +++ b/lib/services/get_html_content.dart @@ -1,7 +1,9 @@ -import 'package:mangayomi/eval/dart/service.dart'; -import 'package:mangayomi/eval/javascript/service.dart'; +import 'dart:io'; + +import 'package:html/parser.dart'; +import 'package:mangayomi/eval/lib.dart'; import 'package:mangayomi/models/chapter.dart'; -import 'package:mangayomi/models/source.dart'; +import 'package:mangayomi/providers/storage_provider.dart'; import 'package:mangayomi/utils/utils.dart'; import 'package:riverpod_annotation/riverpod_annotation.dart'; import 'package:flutter_riverpod/flutter_riverpod.dart'; @@ -12,13 +14,24 @@ Future getHtmlContent(Ref ref, {required Chapter chapter}) async { if (!chapter.manga.isLoaded) { chapter.manga.loadSync(); } + final storageProvider = StorageProvider(); + final mangaDirectory = await storageProvider.getMangaMainDirectory(chapter); + final htmlPath = "${mangaDirectory!.path}${chapter.name}.html"; + final htmlFile = File(htmlPath); + String? htmlContent; + if (await htmlFile.exists()) { + htmlContent = await htmlFile.readAsString(); + final temp = parse(htmlContent); + temp.getElementsByTagName("script").forEach((el) => el.remove()); + htmlContent = temp.outerHtml; + } final source = getSource(chapter.manga.value!.lang!, chapter.manga.value!.source!); String? html; - if (source!.sourceCodeLanguage == SourceCodeLanguage.dart) { - html = await DartExtensionService(source).getHtmlContent(chapter.url!); + if (htmlContent != null) { + html = await getExtensionService(source!).cleanHtmlContent(htmlContent); } else { - html = await JsExtensionService(source).getHtmlContent(chapter.url!); + html = await getExtensionService(source!).getHtmlContent(chapter.url!); } return '''
${html.substring(1, html.length - 1)}
''' .replaceAll("\\n", "") diff --git a/lib/services/get_html_content.g.dart b/lib/services/get_html_content.g.dart index 5444182..968a412 100644 --- a/lib/services/get_html_content.g.dart +++ b/lib/services/get_html_content.g.dart @@ -6,7 +6,7 @@ part of 'get_html_content.dart'; // RiverpodGenerator // ************************************************************************** -String _$getHtmlContentHash() => r'0c964239912b7f93bfb4c80a47f7266ff1ae3f5e'; +String _$getHtmlContentHash() => r'c88d61e16b50cef52da04efc5c53de7390f4910d'; /// Copied from Dart SDK class _SystemHash {