refactor: cache proto file info

.gitignore

@@ -15,3 +15,4 @@
 package-lock.json
 yarn.lock
 node_modules
+**/.proto_cache

frontend/rust-lib/Cargo.lock

@@ -900,7 +900,10 @@ dependencies = [
 name = "flowy-derive"
 version = "0.1.0"
 dependencies = [
+ "dashmap",
  "flowy-ast",
+ "lazy_static",
+ "lib-infra",
  "proc-macro2",
  "quote",
  "syn",
@@ -1723,6 +1726,7 @@ dependencies = [
  "protoc-rust",
  "rand 0.8.4",
  "serde",
+ "serde_json",
  "similar",
  "syn",
  "tera",

frontend/rust-lib/flowy-folder/Cargo.toml

@@ -56,4 +56,4 @@ flowy_unit_test = ["lib-ot/flowy_unit_test", "flowy-sync/flowy_unit_test"]
 dart = ["lib-infra/dart", "flowy-folder/dart", "flowy-folder/dart",]
 
 [build-dependencies]
-lib-infra = { path = "../../../shared-lib/lib-infra", features = ["pb_gen"] }
+lib-infra = { path = "../../../shared-lib/lib-infra", features = ["pb_gen", "proto_gen"] }

frontend/rust-lib/flowy-sdk/Cargo.toml

@@ -40,6 +40,3 @@ futures-util = "0.3.15"
 http_server = ["flowy-user/http_server", "flowy-folder/http_server", "flowy-document/http_server"]
 use_bunyan = ["lib-log/use_bunyan"]
 dart = ["flowy-user/dart", "flowy-net/dart", "flowy-folder/dart", "flowy-collaboration/dart"]
-
-[build-dependencies]
-lib-infra = { path = "../../../shared-lib/lib-infra", features = ["proto_gen"] }

frontend/rust-lib/flowy-sdk/build.rs

@@ -1,9 +0,0 @@
-// use lib_infra::proto_gen;
-
-fn main() {
-    // let derive_meta_dir = format!(
-    //     "{}/{}",
-    //     env!("CARGO_MAKE_WORKING_DIRECTORY"),
-    //     env!("PROTOBUF_DERIVE_CACHE"),
-    // );
-}

shared-lib/Cargo.lock

@@ -443,7 +443,10 @@ dependencies = [
 name = "flowy-derive"
 version = "0.1.0"
 dependencies = [
+ "dashmap",
  "flowy-ast",
+ "lazy_static",
+ "lib-infra",
  "log",
  "proc-macro2",
  "quote",
@@ -802,6 +805,7 @@ dependencies = [
  "protoc-rust",
  "rand 0.8.4",
  "serde",
+ "serde_json",
  "similar",
  "syn",
  "tera",

shared-lib/flowy-derive/Cargo.toml

@@ -18,6 +18,9 @@ syn = { version = "1.0.60", features = ["extra-traits", "visit"] }
 quote = "1.0"
 proc-macro2 = "1.0"
 flowy-ast = { path = "../flowy-ast" }
+lazy_static = {version = "1.4.0"}
+dashmap = "4.0"
+lib-infra = { path = "../lib-infra"}
 
 [dev-dependencies]
 tokio = { version = "1", features = ["full"] }

shared-lib/flowy-derive/src/derive_cache/derive_cache.rs

@@ -1,4 +1,11 @@
 #![cfg_attr(rustfmt, rustfmt::skip)]
+use lazy_static::lazy_static;
+use dashmap::DashMap;
+
+lazy_static! {
+    static ref map: DashMap<String,String> = DashMap::new();
+}
+
 pub enum TypeCategory {
     Array,
     Map,
@@ -11,6 +18,13 @@ pub enum TypeCategory {
 }
 // auto generate, do not edit
 pub fn category_from_str(type_str: &str) -> TypeCategory {
+    let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
+    if !map.contains_key(&root_absolute_path) {
+        println!("😁{}", root_absolute_path);
+        map.insert(root_absolute_path, "123".to_string());
+        
+    }
+    
     match type_str {
         "Vec" => TypeCategory::Array,
         "HashMap" => TypeCategory::Map,

shared-lib/lib-infra/Cargo.toml

@@ -14,6 +14,8 @@ pin-project = "1.0"
 futures-core = { version = "0.3" }
 tokio = { version = "1.0", features = ["time", "rt"] }
 rand = "0.8.3"
+serde = { version = "1.0", features = ["derive"]}
+serde_json = "1.0"
 
 cmd_lib = { version = "1", optional = true }
 protoc-rust = { version = "2", optional = true }
@@ -28,11 +30,10 @@ tera = { version = "1.5.0", optional = true}
 itertools = {versino = "0.10", optional = true}
 phf = { version = "0.8.0", features = ["macros"], optional = true }
 console = {version = "0.14.0", optional = true}
-serde = { version = "1.0", features = ["derive"], optional = true}
+
 toml = {version = "0.5.8", optional = true}
 
 [features]
-pb_gen = ["cmd_lib", "protoc-rust", "walkdir"]
 proto_gen = [
     "flowy-ast",
     "similar",
@@ -44,7 +45,7 @@ proto_gen = [
     "phf",
     "walkdir",
     "console",
-    "serde",
     "toml"
 ]
-dart = []
+pb_gen = ["cmd_lib", "protoc-rust", "walkdir"]
+dart = ["proto_gen"]

shared-lib/lib-infra/src/lib.rs

@@ -5,7 +5,7 @@ pub mod retry;
 pub mod pb;
 
 #[cfg(feature = "proto_gen")]
-pub mod proto_gen;
+mod proto_gen;
 
 #[allow(dead_code)]
 pub fn uuid_string() -> String {

shared-lib/lib-infra/src/pb.rs

@@ -1,69 +1,44 @@
-#![allow(clippy::all)]
 #![allow(unused_imports)]
 #![allow(unused_attributes)]
 #![allow(dead_code)]
-use crate::proto_gen;
-use crate::proto_gen::proto_info::ProtobufCrate;
-use crate::proto_gen::util::suffix_relative_to_path;
-use crate::proto_gen::ProtoGenerator;
+
+#[cfg(feature = "proto_gen")]
+use crate::proto_gen::*;
 use log::info;
 use std::fs::File;
 use std::io::Write;
 use std::process::Command;
 use walkdir::WalkDir;
 
-fn gen_protos(root: &str) -> Vec<ProtobufCrate> {
-    let crate_context = ProtoGenerator::gen(root);
-    let proto_crates = crate_context
-        .iter()
-        .map(|info| info.protobuf_crate.clone())
-        .collect::<Vec<_>>();
-    crate_context
-        .into_iter()
-        .map(|info| info.files)
-        .flatten()
-        .for_each(|file| {
-            println!("cargo:rerun-if-changed={}", file.file_path);
-        });
-    proto_crates
-}
-
 pub fn gen_files(name: &str, root: &str) {
-    let root_absolute_path = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
-    for protobuf_crate in gen_protos(&root_absolute_path) {
-        let mut paths = vec![];
-        let mut file_names = vec![];
-
-        let proto_relative_path = suffix_relative_to_path(&protobuf_crate.proto_output_dir(), &root_absolute_path);
-
-        for (path, file_name) in WalkDir::new(proto_relative_path)
-            .into_iter()
-            .filter_map(|e| e.ok())
-            .map(|e| {
-                let path = e.path().to_str().unwrap().to_string();
-                let file_name = e.path().file_stem().unwrap().to_str().unwrap().to_string();
-                (path, file_name)
-            })
-        {
-            if path.ends_with(".proto") {
-                // https://stackoverflow.com/questions/49077147/how-can-i-force-build-rs-to-run-again-without-cleaning-my-whole-project
-                println!("cargo:rerun-if-changed={}", path);
-                paths.push(path);
-                file_names.push(file_name);
-            }
-        }
-        println!("cargo:rerun-if-changed=build.rs");
+    #[cfg(feature = "proto_gen")]
+    let _ = gen_protos();
 
-        #[cfg(feature = "dart")]
-        gen_pb_for_dart(name, root, &paths, &file_names);
+    let mut paths = vec![];
+    let mut file_names = vec![];
 
-        protoc_rust::Codegen::new()
-            .out_dir("./src/protobuf/model")
-            .inputs(&paths)
-            .include(root)
-            .run()
-            .expect("Running protoc failed.");
+    for (path, file_name) in WalkDir::new(root).into_iter().filter_map(|e| e.ok()).map(|e| {
+        let path = e.path().to_str().unwrap().to_string();
+        let file_name = e.path().file_stem().unwrap().to_str().unwrap().to_string();
+        (path, file_name)
+    }) {
+        if path.ends_with(".proto") {
+            // https://stackoverflow.com/questions/49077147/how-can-i-force-build-rs-to-run-again-without-cleaning-my-whole-project
+            println!("cargo:rerun-if-changed={}", path);
+            paths.push(path);
+            file_names.push(file_name);
+        }
     }
+    println!("cargo:rerun-if-changed=build.rs");
+    #[cfg(feature = "dart")]
+    gen_pb_for_dart(name, root, &paths, &file_names);
+
+    protoc_rust::Codegen::new()
+        .out_dir("./src/protobuf/model")
+        .inputs(&paths)
+        .include(root)
+        .run()
+        .expect("Running protoc failed.");
 }
 
 #[cfg(feature = "dart")]
@@ -143,3 +118,23 @@ fn run_command(cmd: &str) -> bool {
     };
     output.success()
 }
+
+#[cfg(feature = "proto_gen")]
+fn gen_protos() -> Vec<ProtobufCrate> {
+    let root = std::fs::canonicalize(".").unwrap().as_path().display().to_string();
+    let crate_context = ProtoGenerator::gen(&root);
+    let proto_crates = crate_context
+        .iter()
+        .map(|info| info.protobuf_crate.clone())
+        .collect::<Vec<_>>();
+
+    crate_context
+        .into_iter()
+        .map(|info| info.files)
+        .flatten()
+        .for_each(|file| {
+            println!("cargo:rerun-if-changed={}", file.file_path);
+        });
+
+    proto_crates
+}

shared-lib/lib-infra/src/proto_gen/ast.rs

@@ -1,6 +1,6 @@
-use crate::proto_gen::proto_info::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext};
 use crate::proto_gen::template::{EnumTemplate, StructTemplate};
 use crate::proto_gen::util::*;
+use crate::proto_gen::{parse_crate_info_from_path, ProtoFile, ProtobufCrateContext};
 use fancy_regex::Regex;
 use flowy_ast::*;
 use lazy_static::lazy_static;

shared-lib/lib-infra/src/proto_gen/mod.rs

@@ -1,8 +1,9 @@
 mod ast;
 mod flowy_toml;
 mod proto_gen;
-pub mod proto_info;
+mod proto_info;
 mod template;
 pub mod util;
 
-pub(crate) use proto_gen::*;
+pub use proto_gen::*;
+pub use proto_info::*;

shared-lib/lib-infra/src/proto_gen/proto_gen.rs

@@ -2,6 +2,8 @@ use crate::proto_gen::ast::parse_crate_protobuf;
 use crate::proto_gen::proto_info::ProtobufCrateContext;
 use crate::proto_gen::template::write_derive_meta;
 use crate::proto_gen::util::*;
+use crate::proto_gen::ProtoFile;
+use std::fs::File;
 use std::{fs::OpenOptions, io::Write};
 
 pub(crate) struct ProtoGenerator();
@@ -16,6 +18,25 @@ impl ProtoGenerator {
             crate_info.create_crate_mod_file();
         }
 
+        let cache = ProtoCache::from_crate_contexts(&crate_contexts);
+        let cache_str = serde_json::to_string(&cache).unwrap();
+        let protobuf_dart = format!("{}/.proto_cache", root);
+        match std::fs::OpenOptions::new()
+            .create(true)
+            .write(true)
+            .append(false)
+            .truncate(true)
+            .open(&protobuf_dart)
+        {
+            Ok(ref mut file) => {
+                file.write_all(cache_str.as_bytes()).unwrap();
+                File::flush(file).unwrap();
+            }
+            Err(err) => {
+                panic!("Failed to open file: {}", err);
+            }
+        }
+
         crate_contexts
     }
 }
@@ -61,3 +82,23 @@ fn write_rust_crate_mod_file(crate_contexts: &[ProtobufCrateContext]) {
         }
     }
 }
+
+#[derive(serde::Serialize)]
+pub struct ProtoCache {
+    pub structs: Vec<String>,
+    pub enums: Vec<String>,
+}
+
+impl ProtoCache {
+    fn from_crate_contexts(crate_contexts: &[ProtobufCrateContext]) -> Self {
+        let proto_files = crate_contexts
+            .iter()
+            .map(|ref crate_info| &crate_info.files)
+            .flatten()
+            .collect::<Vec<&ProtoFile>>();
+
+        let structs: Vec<String> = proto_files.iter().map(|info| info.structs.clone()).flatten().collect();
+        let enums: Vec<String> = proto_files.iter().map(|info| info.enums.clone()).flatten().collect();
+        Self { structs, enums }
+    }
+}