Creepso/mangayomi
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

added option to download novels

Browse source
This commit is contained in:
Schnitzel5 2025-01-07 23:37:29 +01:00
parent a5ff175c06
commit 5ae7a8d581
14 changed files with 161 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
lib/eval/dart/bridge/m_provider.dart
View file

@ -106,6 +106,13 @@ class $MProvider extends MProvider with $Bridge<MProvider> {
BridgeParameter('url',
BridgeTypeAnnotation(BridgeTypeRef(CoreTypes.string)), false),
])),
'cleanHtmlContent': BridgeMethodDef(BridgeFunctionDef(
returns: BridgeTypeAnnotation(BridgeTypeRef(
CoreTypes.future, [BridgeTypeRef(CoreTypes.string)])),
params: [
BridgeParameter('html',
BridgeTypeAnnotation(BridgeTypeRef(CoreTypes.string)), false),
])),
'getFilterList': BridgeMethodDef(BridgeFunctionDef(
returns: BridgeTypeAnnotation(BridgeTypeRef(
CoreTypes.list, [BridgeTypeRef(CoreTypes.dynamic)])),
@ -1188,6 +1195,10 @@ class $MProvider extends MProvider with $Bridge<MProvider> {
Future<String> getHtmlContent(String url) async =>
await $_invoke('getHtmlContent', [$String(url)]);
@override
Future<String> cleanHtmlContent(String html) async =>
await $_invoke('cleanHtmlContent', [$String(html)]);
@override
Map<String, String> get headers {
try {

5
lib/eval/dart/service.dart
View file

@ -120,6 +120,11 @@ class DartExtensionService implements ExtensionService {
return await _executeLib().getHtmlContent(url);
}
@override
Future<String> cleanHtmlContent(String html) async {
return await _executeLib().cleanHtmlContent(html);
}
@override
FilterList getFilterList() {
List<dynamic> list;

2
lib/eval/interface.dart
View file

@ -31,6 +31,8 @@ abstract interface class ExtensionService {
Future<String> getHtmlContent(String url);
Future<String> cleanHtmlContent(String html);
FilterList getFilterList();
List<SourcePreference> getSourcePreferences();

12
lib/eval/javascript/service.dart
View file

@ -63,6 +63,9 @@ class MProvider {
async getHtmlContent(url) {
throw new Error("getHtmlContent not implemented");
}
async cleanHtmlContent(html) {
throw new Error("cleanHtmlContent not implemented");
}
getFilterList() {
throw new Error("getFilterList not implemented");
}
@ -147,6 +150,15 @@ var extention = new DefaultExtension();
return res;
}
@override
Future<String> cleanHtmlContent(String html) async {
_init();
final res = (await runtime.handlePromise(await runtime.evaluateAsync(
'jsonStringify(() => extention.cleanHtmlContent(`$html`))')))
.stringResult;
return res;
}
@override
FilterList getFilterList() {
List<dynamic> list;

2
lib/eval/model/m_provider.dart
View file

@ -26,6 +26,8 @@ abstract class MProvider {
Future<String> getHtmlContent(String url);
Future<String> cleanHtmlContent(String html);
List<dynamic> getFilterList();
List<dynamic> getSourcePreferences();

13
lib/modules/browse/extension/edit_code.dart
View file

@ -51,13 +51,15 @@ class _CodeEditorState extends ConsumerState<CodeEditor> {
("getDetail", 3),
("getPageList", 4),
("getVideoList", 5),
("getHtmlContent", 6)
("getHtmlContent", 6),
("cleanHtmlContent", 7)
];
int _serviceIndex = 0;
int _page = 1;
String _query = "";
String _url = "";
String _html = "";
bool _isLoading = false;
String _errorText = "";
bool _error = false;
@ -226,6 +228,10 @@ class _CodeEditorState extends ConsumerState<CodeEditor> {
(v) {
_url = v;
}),
if (_serviceIndex == 7)
_textEditing("Html", context, "ex. <p>Text</p>", (v) {
_html = v;
}),
Padding(
padding: const EdgeInsets.all(8.0),
child: Wrap(
@ -293,9 +299,12 @@ class _CodeEditorState extends ConsumerState<CodeEditor> {
(await service.getVideoList(_url))
.map((e) => e.toJson())
.toList();
} else {
} else if (_serviceIndex == 6) {
result = (await service
.getHtmlContent(_url));
} else {
result = (await service
.cleanHtmlContent(_html));
}
if (mounted) {
setState(() {

10
lib/modules/browse/extension/widgets/create_extension.dart
View file

@ -284,6 +284,12 @@ class TestSource extends MProvider {
// TODO: implement
}
// Clean html up for reader
@override
Future<String> cleanHtmlContent(String html) async {
// TODO: implement
}
// For anime episode video list
@override
Future<List<MVideo>> getVideoList(String url) async {
@ -349,6 +355,10 @@ class DefaultExtension extends MProvider {
async getHtmlContent(url) {
throw new Error("getHtmlContent not implemented");
}
// Clean html up for reader
async cleanHtmlContent(html) {
throw new Error("cleanHtmlContent not implemented");
}
// For anime episode video list
async getVideoList(url) {
throw new Error("getVideoList not implemented");

2
lib/modules/manga/detail/providers/update_manga_detail_providers.g.dart
View file

@ -6,7 +6,7 @@ part of 'update_manga_detail_providers.dart';
// RiverpodGenerator
// **************************************************************************
String _$updateMangaDetailHash() => r'dcc5fd8f666959f62ee9ad6540eb0493ac9759f1';
String _$updateMangaDetailHash() => r'a86fe8fea46e411203182287c970cd80cc9a1a0c';
/// Copied from Dart SDK
class _SystemHash {

10
lib/modules/manga/download/download_page_widget.dart
View file

@ -53,10 +53,13 @@ class _ChapterPageDownloadState extends ConsumerState<ChapterPageDownload>
final cbzFile = File(p.join(mangaDir!.path, "${widget.chapter.name}.cbz"));
final mp4File = File(p.join(mangaDir.path,
"${widget.chapter.name!.replaceForbiddenCharacters(' ')}.mp4"));
final htmlFile = File(p.join(mangaDir.path, "${widget.chapter.name}.html"));
if (cbzFile.existsSync()) {
files = [XFile(cbzFile.path)];
} else if (mp4File.existsSync()) {
files = [XFile(mp4File.path)];
} else if (htmlFile.existsSync()) {
files = [XFile(htmlFile.path)];
} else {
files = path!.listSync().map((e) => XFile(e.path)).toList();
}
@ -86,6 +89,13 @@ class _ChapterPageDownloadState extends ConsumerState<ChapterPageDownload>
mp4File.deleteSync();
}
} catch (_) {}
try {
final htmlFile =
File(p.join(mangaDir!.path, "${widget.chapter.name}.html"));
if (htmlFile.existsSync()) {
htmlFile.deleteSync();
}
} catch (_) {}
path!.deleteSync(recursive: true);
} catch (_) {}
isar.writeTxnSync(() {

87
lib/modules/manga/download/providers/download_provider.dart
View file

@ -31,6 +31,8 @@ Future<List<PageUrl>> downloadChapter(
required Chapter chapter,
bool? useWifi,
}) async {
final http = MClient.init(
reqcopyWith: {'useDartHttpClient': true, 'followRedirects': false});
List<PageUrl> pageUrls = [];
List<DownloadTask> tasks = [];
final StorageProvider storageProvider = StorageProvider();
@ -48,7 +50,6 @@ Future<List<PageUrl>> downloadChapter(
final chapterName = chapter.name!.replaceForbiddenCharacters(' ');
final itemType = chapter.manga.value!.itemType;
final isManga = itemType == ItemType.manga;
final itemTypePath = itemType == ItemType.manga
? "Manga"
: itemType == ItemType.anime
@ -60,13 +61,18 @@ Future<List<PageUrl>> downloadChapter(
"${manga.source} (${manga.lang!.toUpperCase()})",
manga.name!.replaceForbiddenCharacters('_'),
];
if (isManga) {
if (itemType == ItemType.manga) {
pathSegments.add(scanlator);
pathSegments.add(chapter.name!.replaceForbiddenCharacters('_'));
}
final finalPath = p.joinAll(pathSegments);
path = Directory(p.join(path1!.path, finalPath));
Map<String, String> videoHeader = {};
Map<String, String> htmlHeader = {
"Priority": "u=0, i",
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
};
bool hasM3U8File = false;
bool nonM3U8File = false;
M3u8Downloader? m3u8Downloader;
@ -75,6 +81,7 @@ Future<List<PageUrl>> downloadChapter(
int? m3u8MediaSequence;
Future<void> processConvert() async {
if (itemType == ItemType.novel) return;
if (hasM3U8File) {
await m3u8Downloader?.mergeTsToMp4(p.join(path!.path, "$chapterName.mp4"),
p.join(path.path, chapterName));
@ -109,7 +116,7 @@ Future<List<PageUrl>> downloadChapter(
isar.settings.putSync(settings..chapterPageUrlsList = chapterPageUrls));
}
if (isManga) {
if (itemType == ItemType.manga) {
ref
.read(getChapterPagesProvider(
chapter: chapter,
@ -120,7 +127,7 @@ Future<List<PageUrl>> downloadChapter(
isOk = true;
}
});
} else {
} else if (itemType == ItemType.anime) {
ref.read(getVideoListProvider(episode: chapter).future).then((value) async {
final m3u8Urls = value.$1
.where((element) =>
@ -150,6 +157,25 @@ Future<List<PageUrl>> downloadChapter(
isOk = true;
}
});
} else if (itemType == ItemType.novel && chapter.url != null) {
final cookie = MClient.getCookiesPref(chapter.url!);
final headers = itemType == ItemType.manga
? ref.watch(headersProvider(source: manga.source!, lang: manga.lang!))
: itemType == ItemType.anime
? videoHeader
: htmlHeader;
if (cookie.isNotEmpty) {
final userAgent = isar.settings.getSync(227)!.userAgent!;
headers.addAll(cookie);
headers[HttpHeaders.userAgentHeader] = userAgent;
}
final res = await http.get(Uri.parse(chapter.url!), headers: headers);
if (res.headers.containsKey("Location")) {
pageUrls = [PageUrl(res.headers["Location"]!)];
} else {
pageUrls = [PageUrl(chapter.url!)];
}
isOk = true;
}
await Future.doWhile(() async {
@ -166,12 +192,20 @@ Future<List<PageUrl>> downloadChapter(
ref.watch(saveAsCBZArchiveStateProvider);
bool mp4FileExist =
await File(p.join(mangaDir.path, "$chapterName.mp4")).exists();
if (!cbzFileExist && isManga || !mp4FileExist && !isManga) {
bool htmlFileExist =
await File(p.join(mangaDir.path, "$chapterName.html")).exists();
if (!cbzFileExist && itemType == ItemType.manga ||
!mp4FileExist && itemType == ItemType.anime ||
!htmlFileExist && itemType == ItemType.novel) {
for (var index = 0; index < pageUrls.length; index++) {
final path2 = Directory(p.join(
path1.path,
"downloads",
isManga ? "Manga" : "Anime",
itemType == ItemType.manga
? "Manga"
: itemType == ItemType.anime
? "Anime"
: "Novel",
"${manga.source} (${manga.lang!.toUpperCase()})",
manga.name!.replaceForbiddenCharacters('_')));
if (!(await path2.exists())) {
@ -184,10 +218,12 @@ Future<List<PageUrl>> downloadChapter(
}
final page = pageUrls[index];
final cookie = MClient.getCookiesPref(page.url);
final headers = isManga
final headers = itemType == ItemType.manga
? ref.watch(
headersProvider(source: manga.source!, lang: manga.lang!))
: videoHeader;
: itemType == ItemType.anime
? videoHeader
: htmlHeader;
if (cookie.isNotEmpty) {
final userAgent = isar.settings.getSync(227)!.userAgent!;
headers.addAll(cookie);
@ -196,7 +232,7 @@ Future<List<PageUrl>> downloadChapter(
Map<String, String> pageHeaders = headers;
pageHeaders.addAll(page.headers ?? {});
if (isManga) {
if (itemType == ItemType.manga) {
final file = File(p.join(tempDir.path, "Mangayomi", finalPath,
"${padIndex(index + 1)}.jpg"));
if (file.existsSync()) {
@ -222,7 +258,7 @@ Future<List<PageUrl>> downloadChapter(
requiresWiFi: onlyOnWifi));
}
}
} else {
} else if (itemType == ItemType.anime) {
final file = File(
p.join(tempDir.path, "Mangayomi", finalPath, "$chapterName.mp4"));
if (file.existsSync()) {
@ -270,6 +306,31 @@ Future<List<PageUrl>> downloadChapter(
requiresWiFi: onlyOnWifi));
}
}
} else {
final file = File(p.join(
tempDir.path, "Mangayomi", finalPath, "$chapterName.html"));
if (file.existsSync()) {
await file.copy(p.join(path.path, "$chapterName.html"));
await file.delete();
} else {
if (!(await path.exists())) {
await path.create();
}
if (!(await File(p.join(path.path, "$chapterName.html"))
.exists())) {
tasks.add(DownloadTask(
taskId: page.url,
headers: pageHeaders,
url: page.url.trim().trimLeft().trimRight(),
filename: "$chapterName.html",
baseDirectory: BaseDirectory.temporary,
directory: p.join("Mangayomi", finalPath),
updates: Updates.statusAndProgress,
allowPause: true,
retries: 3,
requiresWiFi: onlyOnWifi));
}
}
}
}
}
@ -298,7 +359,9 @@ Future<List<PageUrl>> downloadChapter(
await FileDownloader().downloadBatch(
tasks,
batchProgressCallback: (succeeded, failed) async {
if (isManga || hasM3U8File) {
if (itemType == ItemType.manga ||
itemType == ItemType.novel ||
hasM3U8File) {
if (succeeded == tasks.length) {
await processConvert();
}
@ -335,7 +398,7 @@ Future<List<PageUrl>> downloadChapter(
},
taskProgressCallback: (taskProgress) async {
final progress = taskProgress.progress;
if (!isManga && !hasM3U8File) {
if (itemType == ItemType.anime && !hasM3U8File) {
bool isEmpty = isar.downloads
.filter()
.chapterIdEqualTo(chapter.id!)

2
lib/modules/manga/download/providers/download_provider.g.dart
View file

@ -6,7 +6,7 @@ part of 'download_provider.dart';
// RiverpodGenerator
// **************************************************************************
String _$downloadChapterHash() => r'de8e2d5b952071bc0d014fc3aa5c9b0714fbcee0';
String _$downloadChapterHash() => r'2873b00f9f4d0fd91bc90a28e2700a6c0d187a46';
/// Copied from Dart SDK
class _SystemHash {

2
lib/services/fetch_novel_sources.g.dart
View file

@ -7,7 +7,7 @@ part of 'fetch_novel_sources.dart';
// **************************************************************************
String _$fetchNovelSourcesListHash() =>
r'cc4b989c0248c3b16155444c0c429d1ed0025ecb';
r'1444d9ca12204ccf5389efe085c8a20e3498a808';
/// Copied from Dart SDK
class _SystemHash {

25
lib/services/get_html_content.dart
View file

@ -1,7 +1,9 @@
import 'package:mangayomi/eval/dart/service.dart';
import 'package:mangayomi/eval/javascript/service.dart';
import 'dart:io';
import 'package:html/parser.dart';
import 'package:mangayomi/eval/lib.dart';
import 'package:mangayomi/models/chapter.dart';
import 'package:mangayomi/models/source.dart';
import 'package:mangayomi/providers/storage_provider.dart';
import 'package:mangayomi/utils/utils.dart';
import 'package:riverpod_annotation/riverpod_annotation.dart';
import 'package:flutter_riverpod/flutter_riverpod.dart';
@ -12,13 +14,24 @@ Future<String> getHtmlContent(Ref ref, {required Chapter chapter}) async {
if (!chapter.manga.isLoaded) {
chapter.manga.loadSync();
}
final storageProvider = StorageProvider();
final mangaDirectory = await storageProvider.getMangaMainDirectory(chapter);
final htmlPath = "${mangaDirectory!.path}${chapter.name}.html";
final htmlFile = File(htmlPath);
String? htmlContent;
if (await htmlFile.exists()) {
htmlContent = await htmlFile.readAsString();
final temp = parse(htmlContent);
temp.getElementsByTagName("script").forEach((el) => el.remove());
htmlContent = temp.outerHtml;
}
final source =
getSource(chapter.manga.value!.lang!, chapter.manga.value!.source!);
String? html;
if (source!.sourceCodeLanguage == SourceCodeLanguage.dart) {
html = await DartExtensionService(source).getHtmlContent(chapter.url!);
if (htmlContent != null) {
html = await getExtensionService(source!).cleanHtmlContent(htmlContent);
} else {
html = await JsExtensionService(source).getHtmlContent(chapter.url!);
html = await getExtensionService(source!).getHtmlContent(chapter.url!);
}
return '''<div id="readerViewContent"><div style="padding: 2em;">${html.substring(1, html.length - 1)}</div></div>'''
.replaceAll("\\n", "")

2
lib/services/get_html_content.g.dart
View file

@ -6,7 +6,7 @@ part of 'get_html_content.dart';
// RiverpodGenerator
// **************************************************************************
String _$getHtmlContentHash() => r'0c964239912b7f93bfb4c80a47f7266ff1ae3f5e';
String _$getHtmlContentHash() => r'c88d61e16b50cef52da04efc5c53de7390f4910d';
/// Copied from Dart SDK
class _SystemHash {