feat: support import document from v0.1.x (#2601)

frontend/appflowy_flutter/assets/template/readme.afdoc

@@ -0,0 +1,254 @@
+{
+    "document": {
+      "type": "editor",
+      "children": [
+        { "type": "cover" },
+        {
+          "type": "text",
+          "attributes": { "subtype": "heading", "heading": "h1" },
+          "delta": [{ "insert": "Welcome to AppFlowy!" }]
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": "heading", "heading": "h2" },
+          "delta": [{ "insert": "Here are the basics" }]
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": "checkbox", "checkbox": null },
+          "delta": [{ "insert": "Click anywhere and just start typing." }]
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": "checkbox", "checkbox": false },
+          "delta": [
+            {
+              "insert": "Highlight ",
+              "attributes": { "backgroundColor": "0x4dffeb3b" }
+            },
+            { "insert": "any text, and use the editing menu to " },
+            { "insert": "style", "attributes": { "italic": true } },
+            { "insert": " " },
+            { "insert": "your", "attributes": { "bold": true } },
+            { "insert": " " },
+            { "insert": "writing", "attributes": { "underline": true } },
+            { "insert": " " },
+            { "insert": "however", "attributes": { "code": true } },
+            { "insert": " you " },
+            { "insert": "like.", "attributes": { "strikethrough": true } }
+          ]
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": "checkbox", "checkbox": null },
+          "delta": [
+            { "insert": "As soon as you type " },
+            {
+              "insert": "/",
+              "attributes": { "code": true, "color": "0xff00b5ff" }
+            },
+            { "insert": " a menu will pop up. Select " },
+            {
+              "insert": "different types",
+              "attributes": { "backgroundColor": "0x4d9c27b0" }
+            },
+            { "insert": " of content blocks you can add." }
+          ]
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": "checkbox", "checkbox": null },
+          "delta": [
+            { "insert": "Type " },
+            { "insert": "/", "attributes": { "code": true } },
+            { "insert": " followed by " },
+            { "insert": "/bullet", "attributes": { "code": true } },
+            { "insert": " or " },
+            { "insert": "/num", "attributes": { "code": true } },
+            { "insert": " to create a list.", "attributes": { "code": false } }
+          ]
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": "checkbox", "checkbox": true },
+          "delta": [
+            { "insert": "Click " },
+            { "insert": "+ New Page ", "attributes": { "code": true } },
+            {
+              "insert": "button at the bottom of your sidebar to add a new page."
+            }
+          ]
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": "checkbox", "checkbox": null },
+          "delta": [
+            { "insert": "Click " },
+            { "insert": "+", "attributes": { "code": true } },
+            { "insert": " next to any page title in the sidebar to " },
+            { "insert": "quickly", "attributes": { "color": "0xff8427e0" } },
+            { "insert": " add a new subpage, " },
+            { "insert": "Document", "attributes": { "code": true } },
+            { "insert": ", ", "attributes": { "code": false } },
+            { "insert": "Grid", "attributes": { "code": true } },
+            { "insert": ", or ", "attributes": { "code": false } },
+            { "insert": "Kanban Board", "attributes": { "code": true } },
+            { "insert": ".", "attributes": { "code": false } }
+          ]
+        },
+        { "type": "text", "delta": [] },
+        { "type": "divider" },
+        { "type": "text", "attributes": { "checkbox": null }, "delta": [] },
+        {
+          "type": "text",
+          "attributes": {
+            "subtype": "heading",
+            "checkbox": null,
+            "heading": "h2"
+          },
+          "delta": [{ "insert": "Keyboard shortcuts, markdown, and code block" }]
+        },
+        {
+          "type": "text",
+          "attributes": {
+            "subtype": "number-list",
+            "number": 1,
+            "heading": null
+          },
+          "delta": [
+            { "insert": "Keyboard shortcuts " },
+            {
+              "insert": "guide",
+              "attributes": {
+                "href": "https://appflowy.gitbook.io/docs/essential-documentation/shortcuts"
+              }
+            },
+            { "retain": 1, "attributes": { "strikethrough": true } }
+          ]
+        },
+        {
+          "type": "text",
+          "attributes": {
+            "subtype": "number-list",
+            "number": 2,
+            "heading": null
+          },
+          "delta": [
+            { "insert": "Markdown " },
+            {
+              "insert": "reference",
+              "attributes": {
+                "href": "https://appflowy.gitbook.io/docs/essential-documentation/markdown"
+              }
+            },
+            { "retain": 1, "attributes": { "strikethrough": true } }
+          ]
+        },
+        {
+          "type": "text",
+          "attributes": { "number": 3, "subtype": "number-list" },
+          "delta": [
+            { "insert": "Type " },
+            { "insert": "/code", "attributes": { "code": true } },
+            {
+              "insert": " to insert a code block",
+              "attributes": { "code": false }
+            }
+          ]
+        },
+        {
+          "type": "text",
+          "attributes": {
+            "subtype": "code_block",
+            "number": 3,
+            "heading": null,
+            "number-list": null,
+            "theme": "vs",
+            "language": "rust"
+          },
+          "delta": [
+            {
+              "insert": "// This is the main function.\nfn main() {\n    // Print text to the console.\n    println!(\"Hello World!\");\n}"
+            },
+            { "retain": 1, "attributes": { "strikethrough": true } }
+          ]
+        },
+        { "type": "text", "attributes": { "checkbox": null }, "delta": [] },
+        {
+          "type": "text",
+          "attributes": {
+            "subtype": "heading",
+            "checkbox": null,
+            "heading": "h2"
+          },
+          "delta": [{ "insert": "Have a question❓" }]
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": "quote" },
+          "delta": [
+            { "insert": "Click " },
+            { "insert": "?", "attributes": { "code": true } },
+            { "insert": " at the bottom right for help and support." }
+          ]
+        },
+        { "type": "text", "delta": [] },
+        {
+          "type": "callout",
+          "children": [
+            { "type": "text", "delta": [] },
+            {
+              "type": "text",
+              "attributes": { "subtype": "heading", "heading": "h2" },
+              "delta": [{ "insert": "Like AppFlowy? Follow us:" }]
+            },
+            {
+              "type": "text",
+              "attributes": { "subtype": "bulleted-list" },
+              "delta": [
+                {
+                  "insert": "GitHub",
+                  "attributes": {
+                    "href": "https://github.com/AppFlowy-IO/AppFlowy"
+                  }
+                }
+              ]
+            },
+            {
+              "type": "text",
+              "attributes": { "subtype": "bulleted-list" },
+              "delta": [
+                {
+                  "insert": "Twitter",
+                  "attributes": { "href": "https://twitter.com/appflowy" }
+                },
+                { "insert": ": @appflowy" }
+              ]
+            },
+            {
+              "type": "text",
+              "attributes": { "subtype": "bulleted-list" },
+              "delta": [
+                {
+                  "insert": "Newsletter",
+                  "attributes": { "href": "https://blog-appflowy.ghost.io/" }
+                }
+              ]
+            }
+          ],
+          "attributes": { "emoji": "😀" }
+        },
+        { "type": "text", "delta": [] },
+        {
+          "type": "text",
+          "attributes": { "subtype": null, "heading": null },
+          "delta": []
+        },
+        {
+          "type": "text",
+          "attributes": { "subtype": null, "heading": null },
+          "delta": []
+        }
+      ]
+    }
+  }

frontend/appflowy_flutter/lib/plugins/document/application/document_data_pb_extension.dart

@@ -3,15 +3,54 @@ import 'dart:convert';
 import 'package:appflowy_backend/log.dart';
 import 'package:appflowy_backend/protobuf/flowy-document2/protobuf.dart';
 import 'package:appflowy_editor/appflowy_editor.dart'
-    show Document, Node, Attributes, Delta, ParagraphBlockKeys;
+    show Document, Node, Attributes, Delta, ParagraphBlockKeys, NodeIterator;
 import 'package:collection/collection.dart';
+import 'package:nanoid/nanoid.dart';
 
-extension AppFlowyEditor on DocumentDataPB {
-  DocumentDataPB? fromDocument(Document document) {
-    final blocks = <String, BlockPB>{};
+extension DocumentDataPBFromTo on DocumentDataPB {
+  static DocumentDataPB? fromDocument(Document document) {
+    final startNode = document.first;
+    final endNode = document.last;
+    if (startNode == null || endNode == null) {
+      return null;
+    }
     final pageId = document.root.id;
+
+    // generate the block
+    final blocks = <String, BlockPB>{};
+    final nodes = NodeIterator(
+      document: document,
+      startNode: startNode,
+      endNode: endNode,
+    ).toList();
+    for (final node in nodes) {
+      if (blocks.containsKey(node.id)) {
+        assert(false, 'duplicate node id: ${node.id}');
+      }
+      final parentId = node.parent?.id;
+      final childrenId = nanoid(10);
+      blocks[node.id] = node.toBlock(
+        parentId: parentId,
+        childrenId: childrenId,
+      );
+    }
+    // root
+    blocks[pageId] = document.root.toBlock(
+      parentId: '',
+      childrenId: pageId,
+    );
+
+    // generate the meta
     final childrenMap = <String, ChildrenPB>{};
+    blocks.forEach((key, value) {
+      final parentId = value.parentId;
+      if (parentId.isNotEmpty) {
+        childrenMap[parentId] ??= ChildrenPB.create();
+        childrenMap[parentId]!.children.add(value.id);
+      }
+    });
     final meta = MetaPB(childrenMap: childrenMap);
+
     return DocumentDataPB(
       blocks: blocks,
       pageId: pageId,
@@ -91,6 +130,7 @@ extension BlockToNode on BlockPB {
 
 extension NodeToBlock on Node {
   BlockPB toBlock({
+    String? parentId,
     String? childrenId,
   }) {
     assert(id.isNotEmpty);
@@ -101,6 +141,9 @@ extension NodeToBlock on Node {
     if (childrenId != null && childrenId.isNotEmpty) {
       block.childrenId = childrenId;
     }
+    if (parentId != null && parentId.isNotEmpty) {
+      block.parentId = parentId;
+    }
     return block;
   }
 

frontend/appflowy_flutter/lib/workspace/application/app/app_bloc.dart

@@ -108,13 +108,13 @@ class AppBloc extends Bloc<AppEvent, AppState> {
       name: value.name,
       desc: value.desc ?? "",
       layoutType: value.pluginBuilder.layoutType!,
-      initialData: value.initialData,
+      initialDataBytes: value.initialDataBytes,
       ext: value.ext ?? {},
     );
     result.fold(
       (view) => emit(
         state.copyWith(
-          latestCreatedView: view,
+          latestCreatedView: value.openAfterCreated ? view : null,
           successOrFailure: left(unit),
         ),
       ),
@@ -151,10 +151,17 @@ class AppEvent with _$AppEvent {
     PluginBuilder pluginBuilder, {
     String? desc,
 
-    /// The initial data should be the JSON of the document
-    /// For example: {"document":{"type":"editor","children":[]}}
-    String? initialData,
+    /// ~~The initial data should be the JSON of the document~~
+    /// ~~For example: {"document":{"type":"editor","children":[]}}~~
+    ///
+    /// - Document:
+    ///   the initial data should be the string that can be converted into [DocumentDataPB]
+    ///
+    List<int>? initialDataBytes,
     Map<String, String>? ext,
+
+    /// open the view after created
+    @Default(true) bool openAfterCreated,
   }) = CreateView;
   const factory AppEvent.loadViews() = LoadApp;
   const factory AppEvent.delete() = DeleteApp;

frontend/appflowy_flutter/lib/workspace/application/app/app_service.dart

@@ -1,5 +1,4 @@
 import 'dart:async';
-import 'dart:convert';
 
 import 'package:appflowy_backend/protobuf/flowy-folder2/workspace.pb.dart';
 import 'package:dartz/dartz.dart';
@@ -14,12 +13,9 @@ class AppBackendService {
     String? desc,
     required ViewLayoutPB layoutType,
 
-    /// The initial data should be the JSON of the document.
+    /// The initial data should be a JSON that represent the DocumentDataPB.
     /// Currently, only support create document with initial data.
-    ///
-    /// The initial data must be follow this format as shown below.
-    ///  {"document":{"type":"editor","children":[]}}
-    String? initialData,
+    List<int>? initialDataBytes,
 
     /// The [ext] is used to pass through the custom configuration
     /// to the backend.
@@ -33,9 +29,7 @@ class AppBackendService {
       ..name = name
       ..desc = desc ?? ""
       ..layout = layoutType
-      ..initialData = utf8.encode(
-        initialData ?? "",
-      );
+      ..initialData = initialDataBytes ?? [];
 
     if (ext.isNotEmpty) {
       payload.ext.addAll(ext);

frontend/appflowy_flutter/lib/workspace/presentation/home/menu/app/header/add_button.dart

@@ -3,7 +3,6 @@ import 'package:appflowy/startup/plugin/plugin.dart';
 import 'package:appflowy/startup/startup.dart';
 import 'package:appflowy/workspace/presentation/home/menu/app/header/import/import_panel.dart';
 import 'package:appflowy/workspace/presentation/widgets/pop_up_action.dart';
-import 'package:appflowy_editor/appflowy_editor.dart' show Document;
 import 'package:appflowy_popover/appflowy_popover.dart';
 import 'package:flowy_infra/image.dart';
 import 'package:flowy_infra/theme_extension.dart';
@@ -15,7 +14,8 @@ import 'package:easy_localization/easy_localization.dart';
 class AddButton extends StatelessWidget {
   final Function(
     PluginBuilder,
-    Document? document,
+    List<int>? initialDataBytes,
+    bool openAfterCreated,
   ) onSelected;
 
   const AddButton({
@@ -71,14 +71,22 @@ class AddButton extends StatelessWidget {
       },
       onSelected: (action, controller) {
         if (action is AddButtonActionWrapper) {
-          onSelected(action.pluginBuilder, null);
+          onSelected(action.pluginBuilder, null, true);
         }
         if (action is ImportActionWrapper) {
-          showImportPanel(context, (document) {
-            if (document == null) {
+          showImportPanel(context, (type, initialDataBytes) {
+            if (initialDataBytes == null) {
               return;
             }
-            onSelected(action.pluginBuilder, document);
+            switch (type) {
+              case ImportType.historyDocument:
+              case ImportType.historyDatabase:
+                onSelected(action.pluginBuilder, initialDataBytes, false);
+                break;
+              case ImportType.markdownOrText:
+                onSelected(action.pluginBuilder, initialDataBytes, true);
+                break;
+            }
           });
         }
         controller.close();

frontend/appflowy_flutter/lib/workspace/presentation/home/menu/app/header/header.dart

@@ -1,5 +1,3 @@
-import 'dart:convert';
-
 import 'package:appflowy/workspace/presentation/widgets/dialogs.dart';
 import 'package:appflowy/workspace/presentation/widgets/pop_up_action.dart';
 import 'package:appflowy_backend/protobuf/flowy-folder2/view.pb.dart';
@@ -110,13 +108,13 @@ class MenuAppHeader extends StatelessWidget {
     return Tooltip(
       message: LocaleKeys.menuAppHeader_addPageTooltip.tr(),
       child: AddButton(
-        onSelected: (pluginBuilder, document) {
+        onSelected: (pluginBuilder, initialDataBytes, openAfterCreated) {
           context.read<AppBloc>().add(
                 AppEvent.createView(
                   LocaleKeys.menuAppHeader_defaultNewPageName.tr(),
                   pluginBuilder,
-                  initialData:
-                      document != null ? jsonEncode(document.toJson()) : '',
+                  initialDataBytes: initialDataBytes,
+                  openAfterCreated: openAfterCreated,
                 ),
               );
         },

frontend/appflowy_flutter/lib/workspace/presentation/home/menu/app/header/import/import_panel.dart

@@ -1,5 +1,7 @@
 import 'dart:io';
 
+import 'package:appflowy/plugins/document/application/document_data_pb_extension.dart';
+import 'package:appflowy/plugins/document/presentation/editor_plugins/migration/editor_migration.dart';
 import 'package:appflowy/startup/startup.dart';
 import 'package:appflowy/util/file_picker/file_picker_service.dart';
 import 'package:appflowy_editor/appflowy_editor.dart';
@@ -11,7 +13,7 @@ import 'package:flutter/material.dart';
 import 'package:appflowy/generated/locale_keys.g.dart';
 import 'package:easy_localization/easy_localization.dart';
 
-typedef ImportCallback = void Function(Document? document);
+typedef ImportCallback = void Function(ImportType type, List<int>? document);
 
 Future<void> showImportPanel(
   BuildContext context,
@@ -36,13 +38,19 @@ Future<void> showImportPanel(
   );
 }
 
-enum _ImportType {
+enum ImportType {
+  historyDocument,
+  historyDatabase,
   markdownOrText;
 
   @override
   String toString() {
     switch (this) {
-      case _ImportType.markdownOrText:
+      case ImportType.historyDocument:
+        return 'Document from v0.1';
+      case ImportType.historyDatabase:
+        return 'Database from v0.1';
+      case ImportType.markdownOrText:
         return 'Text & Markdown';
       default:
         assert(false, 'Unsupported Type $this');
@@ -50,23 +58,50 @@ enum _ImportType {
     }
   }
 
-  Widget? get icon {
+  Widget? Function(BuildContext context) get icon => (context) {
+        switch (this) {
+          case ImportType.historyDocument:
+          case ImportType.historyDatabase:
+            return svgWidget(
+              'editor/documents',
+              color: Theme.of(context).iconTheme.color,
+            );
+          case ImportType.markdownOrText:
+            return svgWidget(
+              'editor/documents',
+              color: Theme.of(context).iconTheme.color,
+            );
+          default:
+            assert(false, 'Unsupported Type $this');
+            return null;
+        }
+      };
+
+  List<String> get allowedExtensions {
     switch (this) {
-      case _ImportType.markdownOrText:
-        return svgWidget('editor/documents');
+      case ImportType.historyDocument:
+        return ['afdoc'];
+      case ImportType.historyDatabase:
+        // FIXME: @nathan.
+        return ['afdb'];
+      case ImportType.markdownOrText:
+        return ['md', 'txt'];
       default:
         assert(false, 'Unsupported Type $this');
-        return null;
+        return [];
     }
   }
 
-  List<String> get allowedExtensions {
+  bool get allowMultiSelect {
     switch (this) {
-      case _ImportType.markdownOrText:
-        return ['md', 'txt'];
+      case ImportType.historyDocument:
+        return true;
+      case ImportType.historyDatabase:
+      case ImportType.markdownOrText:
+        return false;
       default:
         assert(false, 'Unsupported Type $this');
-        return [];
+        return false;
     }
   }
 }
@@ -94,11 +129,11 @@ class _ImportPanelState extends State<_ImportPanel> {
       child: GridView.count(
         childAspectRatio: 1 / .2,
         crossAxisCount: 2,
-        children: _ImportType.values.map(
+        children: ImportType.values.map(
           (e) {
             return Card(
               child: FlowyButton(
-                leftIcon: e.icon,
+                leftIcon: e.icon(context),
                 leftIconSize: const Size.square(20),
                 text: FlowyText.medium(
                   e.toString(),
@@ -119,26 +154,37 @@ class _ImportPanelState extends State<_ImportPanel> {
     );
   }
 
-  Future<void> _importFile(_ImportType importType) async {
+  Future<void> _importFile(ImportType importType) async {
     final result = await getIt<FilePickerService>().pickFiles(
-      allowMultiple: false,
+      allowMultiple: importType.allowMultiSelect,
       type: FileType.custom,
       allowedExtensions: importType.allowedExtensions,
     );
     if (result == null || result.files.isEmpty) {
       return;
     }
-    final path = result.files.single.path!;
-    final plainText = await File(path).readAsString();
 
-    switch (importType) {
-      case _ImportType.markdownOrText:
-        final document = markdownToDocument(plainText);
-        widget.importCallback(document);
-        break;
-      default:
-        assert(false, 'Unsupported Type $importType');
-        widget.importCallback(null);
+    for (final file in result.files) {
+      final path = file.path;
+      if (path == null) {
+        continue;
+      }
+      final plainText = await File(path).readAsString();
+      Document? document;
+      switch (importType) {
+        case ImportType.markdownOrText:
+          document = markdownToDocument(plainText);
+          break;
+        case ImportType.historyDocument:
+          document = EditorMigration.migrateDocument(plainText);
+          break;
+        default:
+          assert(false, 'Unsupported Type $importType');
+      }
+      if (document != null) {
+        final data = DocumentDataPBFromTo.fromDocument(document);
+        widget.importCallback(importType, data?.writeToBuffer());
+      }
     }
   }
 }

frontend/appflowy_flutter/pubspec.lock

@@ -53,8 +53,8 @@ packages:
     dependency: "direct main"
     description:
       path: "."
-      ref: ead61af
-      resolved-ref: ead61afb796037e8ceb63ba4bcf439818514ed4b
+      ref: "21f686"
+      resolved-ref: "21f686d6a43137cf6c6d7d040463a1679d13f858"
       url: "https://github.com/LucasXu0/appflowy-editor.git"
     source: git
     version: "0.1.12"

frontend/appflowy_flutter/pubspec.yaml

@@ -47,7 +47,7 @@ dependencies:
     # path: /Users/lucas.xu/Desktop/appflowy-editor
     git:
       url: https://github.com/LucasXu0/appflowy-editor.git
-      ref: ead61af
+      ref: 21f686
   appflowy_popover:
     path: packages/appflowy_popover
 
@@ -179,9 +179,9 @@ flutter:
     - assets/images/login/
     - assets/images/grid/setting/
     - assets/translations/
-    - assets/template/readme.json
 
     # The following assets will be excluded in release.
     # BEGIN: EXCLUDE_IN_RELEASE
     - assets/test/workspaces/
+    - assets/template/
     # END: EXCLUDE_IN_RELEASE

frontend/appflowy_flutter/test/unit_test/editor/editor_migration_test.dart

@@ -0,0 +1,9 @@
+import 'package:flutter_test/flutter_test.dart';
+
+void main() {
+  group('editor migration, from v0.1.x to 0.2', () {
+    test('migrate readme', () {
+      // final
+    });
+  });
+}

frontend/rust-lib/flowy-core/src/deps_resolve/folder2_deps.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::convert::TryFrom;
 use std::sync::Arc;
 
 use appflowy_integrate::collab_builder::AppFlowyCollabBuilder;
@@ -9,6 +10,7 @@ use flowy_database2::entities::DatabaseLayoutPB;
 use flowy_database2::template::{make_default_board, make_default_calendar, make_default_grid};
 use flowy_database2::DatabaseManager2;
 use flowy_document2::document_data::DocumentDataWrapper;
+use flowy_document2::entities::DocumentDataPB;
 use flowy_document2::manager::DocumentManager;
 use flowy_error::FlowyError;
 use flowy_folder2::entities::ViewLayoutPB;
@@ -126,7 +128,7 @@ impl ViewDataProcessor for DocumentViewDataProcessor {
     _user_id: i64,
     view_id: &str,
     _name: &str,
-    _data: Vec<u8>,
+    data: Vec<u8>,
     layout: ViewLayout,
     _ext: HashMap<String, String>,
   ) -> FutureResult<(), FlowyError> {
@@ -134,9 +136,9 @@ impl ViewDataProcessor for DocumentViewDataProcessor {
     // TODO: implement read the document data from custom data.
     let view_id = view_id.to_string();
     let manager = self.0.clone();
-
     FutureResult::new(async move {
-      manager.create_document(view_id, DocumentDataWrapper::default())?;
+      let data = DocumentDataPB::try_from(Bytes::from(data))?;
+      manager.create_document(view_id, data.into())?;
       Ok(())
     })
   }

frontend/rust-lib/flowy-folder2/src/user_default.rs

@@ -31,14 +31,14 @@ impl DefaultFolderBuilder {
     };
 
     // create the document
-    let data = initial_read_me().into_bytes();
+    // TODO: use the initial data from the view processor
+    // let data = initial_read_me().into_bytes();
     let processor = view_processors.get(&child_view_layout).unwrap();
     processor
-      .create_view_with_custom_data(
+      .create_view_with_built_in_data(
         uid,
         &child_view.id,
         &child_view.name,
-        data,
         child_view_layout.clone(),
         HashMap::default(),
       )