added option to download novels

lib/eval/dart/bridge/m_provider.dart

@@ -106,6 +106,13 @@ class $MProvider extends MProvider with $Bridge<MProvider> {
               BridgeParameter('url',
                   BridgeTypeAnnotation(BridgeTypeRef(CoreTypes.string)), false),
             ])),
+        'cleanHtmlContent': BridgeMethodDef(BridgeFunctionDef(
+            returns: BridgeTypeAnnotation(BridgeTypeRef(
+                CoreTypes.future, [BridgeTypeRef(CoreTypes.string)])),
+            params: [
+              BridgeParameter('html',
+                  BridgeTypeAnnotation(BridgeTypeRef(CoreTypes.string)), false),
+            ])),
         'getFilterList': BridgeMethodDef(BridgeFunctionDef(
             returns: BridgeTypeAnnotation(BridgeTypeRef(
                 CoreTypes.list, [BridgeTypeRef(CoreTypes.dynamic)])),
@@ -1188,6 +1195,10 @@ class $MProvider extends MProvider with $Bridge<MProvider> {
   Future<String> getHtmlContent(String url) async =>
       await $_invoke('getHtmlContent', [$String(url)]);
 
+  @override
+  Future<String> cleanHtmlContent(String html) async =>
+      await $_invoke('cleanHtmlContent', [$String(html)]);
+
   @override
   Map<String, String> get headers {
     try {

lib/eval/dart/service.dart

@@ -120,6 +120,11 @@ class DartExtensionService implements ExtensionService {
     return await _executeLib().getHtmlContent(url);
   }
 
+  @override
+  Future<String> cleanHtmlContent(String html) async {
+    return await _executeLib().cleanHtmlContent(html);
+  }
+
   @override
   FilterList getFilterList() {
     List<dynamic> list;

lib/eval/interface.dart

@@ -31,6 +31,8 @@ abstract interface class ExtensionService {
 
   Future<String> getHtmlContent(String url);
 
+  Future<String> cleanHtmlContent(String html);
+
   FilterList getFilterList();
 
   List<SourcePreference> getSourcePreferences();

lib/eval/javascript/service.dart

@@ -63,6 +63,9 @@ class MProvider {
     async getHtmlContent(url) {
         throw new Error("getHtmlContent not implemented");
     }
+    async cleanHtmlContent(html) {
+        throw new Error("cleanHtmlContent not implemented");
+    }
     getFilterList() {
         throw new Error("getFilterList not implemented");
     }
@@ -147,6 +150,15 @@ var extention = new DefaultExtension();
     return res;
   }
 
+  @override
+  Future<String> cleanHtmlContent(String html) async {
+    _init();
+    final res = (await runtime.handlePromise(await runtime.evaluateAsync(
+            'jsonStringify(() => extention.cleanHtmlContent(`$html`))')))
+        .stringResult;
+    return res;
+  }
+
   @override
   FilterList getFilterList() {
     List<dynamic> list;

lib/eval/model/m_provider.dart

@@ -26,6 +26,8 @@ abstract class MProvider {
 
   Future<String> getHtmlContent(String url);
 
+  Future<String> cleanHtmlContent(String html);
+
   List<dynamic> getFilterList();
 
   List<dynamic> getSourcePreferences();

lib/modules/browse/extension/edit_code.dart

@@ -51,13 +51,15 @@ class _CodeEditorState extends ConsumerState<CodeEditor> {
         ("getDetail", 3),
         ("getPageList", 4),
         ("getVideoList", 5),
-        ("getHtmlContent", 6)
+        ("getHtmlContent", 6),
+        ("cleanHtmlContent", 7)
       ];
 
   int _serviceIndex = 0;
   int _page = 1;
   String _query = "";
   String _url = "";
+  String _html = "";
   bool _isLoading = false;
   String _errorText = "";
   bool _error = false;
@@ -226,6 +228,10 @@ class _CodeEditorState extends ConsumerState<CodeEditor> {
                               (v) {
                             _url = v;
                           }),
+                        if (_serviceIndex == 7)
+                          _textEditing("Html", context, "ex. <p>Text</p>", (v) {
+                            _html = v;
+                          }),
                         Padding(
                           padding: const EdgeInsets.all(8.0),
                           child: Wrap(
@@ -293,9 +299,12 @@ class _CodeEditorState extends ConsumerState<CodeEditor> {
                                               (await service.getVideoList(_url))
                                                   .map((e) => e.toJson())
                                                   .toList();
-                                        } else {
+                                        } else if (_serviceIndex == 6) {
                                           result = (await service
                                               .getHtmlContent(_url));
+                                        } else {
+                                          result = (await service
+                                              .cleanHtmlContent(_html));
                                         }
                                         if (mounted) {
                                           setState(() {

lib/modules/browse/extension/widgets/create_extension.dart

@@ -284,6 +284,12 @@ class TestSource extends MProvider {
     // TODO: implement
   }
   
+  // Clean html up for reader
+  @override
+  Future<String> cleanHtmlContent(String html) async {
+    // TODO: implement
+  }
+  
   // For anime episode video list
   @override
   Future<List<MVideo>> getVideoList(String url) async {
@@ -349,6 +355,10 @@ class DefaultExtension extends MProvider {
     async getHtmlContent(url) {
         throw new Error("getHtmlContent not implemented");
     }
+    // Clean html up for reader
+    async cleanHtmlContent(html) {
+        throw new Error("cleanHtmlContent not implemented");
+    }
     // For anime episode video list
     async getVideoList(url) {
         throw new Error("getVideoList not implemented");

lib/modules/manga/detail/providers/update_manga_detail_providers.g.dart

@@ -6,7 +6,7 @@ part of 'update_manga_detail_providers.dart';
 // RiverpodGenerator
 // **************************************************************************
 
-String _$updateMangaDetailHash() => r'dcc5fd8f666959f62ee9ad6540eb0493ac9759f1';
+String _$updateMangaDetailHash() => r'a86fe8fea46e411203182287c970cd80cc9a1a0c';
 
 /// Copied from Dart SDK
 class _SystemHash {

lib/modules/manga/download/download_page_widget.dart

@@ -53,10 +53,13 @@ class _ChapterPageDownloadState extends ConsumerState<ChapterPageDownload>
     final cbzFile = File(p.join(mangaDir!.path, "${widget.chapter.name}.cbz"));
     final mp4File = File(p.join(mangaDir.path,
         "${widget.chapter.name!.replaceForbiddenCharacters(' ')}.mp4"));
+    final htmlFile = File(p.join(mangaDir.path, "${widget.chapter.name}.html"));
     if (cbzFile.existsSync()) {
       files = [XFile(cbzFile.path)];
     } else if (mp4File.existsSync()) {
       files = [XFile(mp4File.path)];
+    } else if (htmlFile.existsSync()) {
+      files = [XFile(htmlFile.path)];
     } else {
       files = path!.listSync().map((e) => XFile(e.path)).toList();
     }
@@ -86,6 +89,13 @@ class _ChapterPageDownloadState extends ConsumerState<ChapterPageDownload>
           mp4File.deleteSync();
         }
       } catch (_) {}
+      try {
+        final htmlFile =
+            File(p.join(mangaDir!.path, "${widget.chapter.name}.html"));
+        if (htmlFile.existsSync()) {
+          htmlFile.deleteSync();
+        }
+      } catch (_) {}
       path!.deleteSync(recursive: true);
     } catch (_) {}
     isar.writeTxnSync(() {

lib/modules/manga/download/providers/download_provider.dart

@@ -31,6 +31,8 @@ Future<List<PageUrl>> downloadChapter(
   required Chapter chapter,
   bool? useWifi,
 }) async {
+  final http = MClient.init(
+      reqcopyWith: {'useDartHttpClient': true, 'followRedirects': false});
   List<PageUrl> pageUrls = [];
   List<DownloadTask> tasks = [];
   final StorageProvider storageProvider = StorageProvider();
@@ -48,7 +50,6 @@ Future<List<PageUrl>> downloadChapter(
   final chapterName = chapter.name!.replaceForbiddenCharacters(' ');
 
   final itemType = chapter.manga.value!.itemType;
-  final isManga = itemType == ItemType.manga;
   final itemTypePath = itemType == ItemType.manga
       ? "Manga"
       : itemType == ItemType.anime
@@ -60,13 +61,18 @@ Future<List<PageUrl>> downloadChapter(
     "${manga.source} (${manga.lang!.toUpperCase()})",
     manga.name!.replaceForbiddenCharacters('_'),
   ];
-  if (isManga) {
+  if (itemType == ItemType.manga) {
     pathSegments.add(scanlator);
     pathSegments.add(chapter.name!.replaceForbiddenCharacters('_'));
   }
   final finalPath = p.joinAll(pathSegments);
   path = Directory(p.join(path1!.path, finalPath));
   Map<String, String> videoHeader = {};
+  Map<String, String> htmlHeader = {
+    "Priority": "u=0, i",
+    "User-Agent":
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36",
+  };
   bool hasM3U8File = false;
   bool nonM3U8File = false;
   M3u8Downloader? m3u8Downloader;
@@ -75,6 +81,7 @@ Future<List<PageUrl>> downloadChapter(
   int? m3u8MediaSequence;
 
   Future<void> processConvert() async {
+    if (itemType == ItemType.novel) return;
     if (hasM3U8File) {
       await m3u8Downloader?.mergeTsToMp4(p.join(path!.path, "$chapterName.mp4"),
           p.join(path.path, chapterName));
@@ -109,7 +116,7 @@ Future<List<PageUrl>> downloadChapter(
         isar.settings.putSync(settings..chapterPageUrlsList = chapterPageUrls));
   }
 
-  if (isManga) {
+  if (itemType == ItemType.manga) {
     ref
         .read(getChapterPagesProvider(
       chapter: chapter,
@@ -120,7 +127,7 @@ Future<List<PageUrl>> downloadChapter(
         isOk = true;
       }
     });
-  } else {
+  } else if (itemType == ItemType.anime) {
     ref.read(getVideoListProvider(episode: chapter).future).then((value) async {
       final m3u8Urls = value.$1
           .where((element) =>
@@ -150,6 +157,25 @@ Future<List<PageUrl>> downloadChapter(
         isOk = true;
       }
     });
+  } else if (itemType == ItemType.novel && chapter.url != null) {
+    final cookie = MClient.getCookiesPref(chapter.url!);
+    final headers = itemType == ItemType.manga
+        ? ref.watch(headersProvider(source: manga.source!, lang: manga.lang!))
+        : itemType == ItemType.anime
+            ? videoHeader
+            : htmlHeader;
+    if (cookie.isNotEmpty) {
+      final userAgent = isar.settings.getSync(227)!.userAgent!;
+      headers.addAll(cookie);
+      headers[HttpHeaders.userAgentHeader] = userAgent;
+    }
+    final res = await http.get(Uri.parse(chapter.url!), headers: headers);
+    if (res.headers.containsKey("Location")) {
+      pageUrls = [PageUrl(res.headers["Location"]!)];
+    } else {
+      pageUrls = [PageUrl(chapter.url!)];
+    }
+    isOk = true;
   }
 
   await Future.doWhile(() async {
@@ -166,12 +192,20 @@ Future<List<PageUrl>> downloadChapter(
             ref.watch(saveAsCBZArchiveStateProvider);
     bool mp4FileExist =
         await File(p.join(mangaDir.path, "$chapterName.mp4")).exists();
-    if (!cbzFileExist && isManga || !mp4FileExist && !isManga) {
+    bool htmlFileExist =
+        await File(p.join(mangaDir.path, "$chapterName.html")).exists();
+    if (!cbzFileExist && itemType == ItemType.manga ||
+        !mp4FileExist && itemType == ItemType.anime ||
+        !htmlFileExist && itemType == ItemType.novel) {
       for (var index = 0; index < pageUrls.length; index++) {
         final path2 = Directory(p.join(
             path1.path,
             "downloads",
-            isManga ? "Manga" : "Anime",
+            itemType == ItemType.manga
+                ? "Manga"
+                : itemType == ItemType.anime
+                    ? "Anime"
+                    : "Novel",
             "${manga.source} (${manga.lang!.toUpperCase()})",
             manga.name!.replaceForbiddenCharacters('_')));
         if (!(await path2.exists())) {
@@ -184,10 +218,12 @@ Future<List<PageUrl>> downloadChapter(
         }
         final page = pageUrls[index];
         final cookie = MClient.getCookiesPref(page.url);
-        final headers = isManga
+        final headers = itemType == ItemType.manga
             ? ref.watch(
                 headersProvider(source: manga.source!, lang: manga.lang!))
-            : videoHeader;
+            : itemType == ItemType.anime
+                ? videoHeader
+                : htmlHeader;
         if (cookie.isNotEmpty) {
           final userAgent = isar.settings.getSync(227)!.userAgent!;
           headers.addAll(cookie);
@@ -196,7 +232,7 @@ Future<List<PageUrl>> downloadChapter(
         Map<String, String> pageHeaders = headers;
         pageHeaders.addAll(page.headers ?? {});
 
-        if (isManga) {
+        if (itemType == ItemType.manga) {
           final file = File(p.join(tempDir.path, "Mangayomi", finalPath,
               "${padIndex(index + 1)}.jpg"));
           if (file.existsSync()) {
@@ -222,7 +258,7 @@ Future<List<PageUrl>> downloadChapter(
                   requiresWiFi: onlyOnWifi));
             }
           }
-        } else {
+        } else if (itemType == ItemType.anime) {
           final file = File(
               p.join(tempDir.path, "Mangayomi", finalPath, "$chapterName.mp4"));
           if (file.existsSync()) {
@@ -270,6 +306,31 @@ Future<List<PageUrl>> downloadChapter(
                   requiresWiFi: onlyOnWifi));
             }
           }
+        } else {
+          final file = File(p.join(
+              tempDir.path, "Mangayomi", finalPath, "$chapterName.html"));
+          if (file.existsSync()) {
+            await file.copy(p.join(path.path, "$chapterName.html"));
+            await file.delete();
+          } else {
+            if (!(await path.exists())) {
+              await path.create();
+            }
+            if (!(await File(p.join(path.path, "$chapterName.html"))
+                .exists())) {
+              tasks.add(DownloadTask(
+                  taskId: page.url,
+                  headers: pageHeaders,
+                  url: page.url.trim().trimLeft().trimRight(),
+                  filename: "$chapterName.html",
+                  baseDirectory: BaseDirectory.temporary,
+                  directory: p.join("Mangayomi", finalPath),
+                  updates: Updates.statusAndProgress,
+                  allowPause: true,
+                  retries: 3,
+                  requiresWiFi: onlyOnWifi));
+            }
+          }
         }
       }
     }
@@ -298,7 +359,9 @@ Future<List<PageUrl>> downloadChapter(
       await FileDownloader().downloadBatch(
         tasks,
         batchProgressCallback: (succeeded, failed) async {
-          if (isManga || hasM3U8File) {
+          if (itemType == ItemType.manga ||
+              itemType == ItemType.novel ||
+              hasM3U8File) {
             if (succeeded == tasks.length) {
               await processConvert();
             }
@@ -335,7 +398,7 @@ Future<List<PageUrl>> downloadChapter(
         },
         taskProgressCallback: (taskProgress) async {
           final progress = taskProgress.progress;
-          if (!isManga && !hasM3U8File) {
+          if (itemType == ItemType.anime && !hasM3U8File) {
             bool isEmpty = isar.downloads
                 .filter()
                 .chapterIdEqualTo(chapter.id!)

lib/modules/manga/download/providers/download_provider.g.dart

@@ -6,7 +6,7 @@ part of 'download_provider.dart';
 // RiverpodGenerator
 // **************************************************************************
 
-String _$downloadChapterHash() => r'de8e2d5b952071bc0d014fc3aa5c9b0714fbcee0';
+String _$downloadChapterHash() => r'2873b00f9f4d0fd91bc90a28e2700a6c0d187a46';
 
 /// Copied from Dart SDK
 class _SystemHash {

lib/services/fetch_novel_sources.g.dart

@@ -7,7 +7,7 @@ part of 'fetch_novel_sources.dart';
 // **************************************************************************
 
 String _$fetchNovelSourcesListHash() =>
-    r'cc4b989c0248c3b16155444c0c429d1ed0025ecb';
+    r'1444d9ca12204ccf5389efe085c8a20e3498a808';
 
 /// Copied from Dart SDK
 class _SystemHash {

lib/services/get_html_content.dart

@@ -1,7 +1,9 @@
-import 'package:mangayomi/eval/dart/service.dart';
+import 'dart:io';
-import 'package:mangayomi/eval/javascript/service.dart';
+
+import 'package:html/parser.dart';
+import 'package:mangayomi/eval/lib.dart';
 import 'package:mangayomi/models/chapter.dart';
-import 'package:mangayomi/models/source.dart';
+import 'package:mangayomi/providers/storage_provider.dart';
 import 'package:mangayomi/utils/utils.dart';
 import 'package:riverpod_annotation/riverpod_annotation.dart';
 import 'package:flutter_riverpod/flutter_riverpod.dart';
@@ -12,13 +14,24 @@ Future<String> getHtmlContent(Ref ref, {required Chapter chapter}) async {
   if (!chapter.manga.isLoaded) {
     chapter.manga.loadSync();
   }
+  final storageProvider = StorageProvider();
+  final mangaDirectory = await storageProvider.getMangaMainDirectory(chapter);
+  final htmlPath = "${mangaDirectory!.path}${chapter.name}.html";
+  final htmlFile = File(htmlPath);
+  String? htmlContent;
+  if (await htmlFile.exists()) {
+    htmlContent = await htmlFile.readAsString();
+    final temp = parse(htmlContent);
+    temp.getElementsByTagName("script").forEach((el) => el.remove());
+    htmlContent = temp.outerHtml;
+  }
   final source =
       getSource(chapter.manga.value!.lang!, chapter.manga.value!.source!);
   String? html;
-  if (source!.sourceCodeLanguage == SourceCodeLanguage.dart) {
+  if (htmlContent != null) {
-    html = await DartExtensionService(source).getHtmlContent(chapter.url!);
+    html = await getExtensionService(source!).cleanHtmlContent(htmlContent);
   } else {
-    html = await JsExtensionService(source).getHtmlContent(chapter.url!);
+    html = await getExtensionService(source!).getHtmlContent(chapter.url!);
   }
   return '''<div id="readerViewContent"><div style="padding: 2em;">${html.substring(1, html.length - 1)}</div></div>'''
       .replaceAll("\\n", "")

lib/services/get_html_content.g.dart

@@ -6,7 +6,7 @@ part of 'get_html_content.dart';
 // RiverpodGenerator
 // **************************************************************************
 
-String _$getHtmlContentHash() => r'0c964239912b7f93bfb4c80a47f7266ff1ae3f5e';
+String _$getHtmlContentHash() => r'c88d61e16b50cef52da04efc5c53de7390f4910d';
 
 /// Copied from Dart SDK
 class _SystemHash {