|
@@ -4,7 +4,7 @@ use crate::cache::{
|
|
|
};
|
|
|
use crate::disk::{RevisionState, SyncRecord};
|
|
|
use crate::memory::RevisionMemoryCache;
|
|
|
-use crate::RevisionCompress;
|
|
|
+use crate::RevisionMergeable;
|
|
|
use flowy_error::{internal_error, FlowyError, FlowyResult};
|
|
|
use flowy_sync::entities::revision::{Revision, RevisionRange};
|
|
|
use std::collections::VecDeque;
|
|
@@ -14,30 +14,58 @@ use tokio::task::spawn_blocking;
|
|
|
|
|
|
pub const REVISION_WRITE_INTERVAL_IN_MILLIS: u64 = 600;
|
|
|
|
|
|
+pub struct RevisionPersistenceConfiguration {
|
|
|
+ merge_threshold: usize,
|
|
|
+}
|
|
|
+
|
|
|
+impl RevisionPersistenceConfiguration {
|
|
|
+ pub fn new(merge_threshold: usize) -> Self {
|
|
|
+ debug_assert!(merge_threshold > 1);
|
|
|
+ if merge_threshold > 1 {
|
|
|
+ Self { merge_threshold }
|
|
|
+ } else {
|
|
|
+ Self { merge_threshold: 2 }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+impl std::default::Default for RevisionPersistenceConfiguration {
|
|
|
+ fn default() -> Self {
|
|
|
+ Self { merge_threshold: 2 }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
pub struct RevisionPersistence<Connection> {
|
|
|
user_id: String,
|
|
|
object_id: String,
|
|
|
disk_cache: Arc<dyn RevisionDiskCache<Connection, Error = FlowyError>>,
|
|
|
memory_cache: Arc<RevisionMemoryCache>,
|
|
|
sync_seq: RwLock<DeferSyncSequence>,
|
|
|
+ configuration: RevisionPersistenceConfiguration,
|
|
|
}
|
|
|
|
|
|
impl<Connection> RevisionPersistence<Connection>
|
|
|
where
|
|
|
Connection: 'static,
|
|
|
{
|
|
|
- pub fn new<C>(user_id: &str, object_id: &str, disk_cache: C) -> RevisionPersistence<Connection>
|
|
|
+ pub fn new<C>(
|
|
|
+ user_id: &str,
|
|
|
+ object_id: &str,
|
|
|
+ disk_cache: C,
|
|
|
+ configuration: RevisionPersistenceConfiguration,
|
|
|
+ ) -> RevisionPersistence<Connection>
|
|
|
where
|
|
|
C: 'static + RevisionDiskCache<Connection, Error = FlowyError>,
|
|
|
{
|
|
|
let disk_cache = Arc::new(disk_cache) as Arc<dyn RevisionDiskCache<Connection, Error = FlowyError>>;
|
|
|
- Self::from_disk_cache(user_id, object_id, disk_cache)
|
|
|
+ Self::from_disk_cache(user_id, object_id, disk_cache, configuration)
|
|
|
}
|
|
|
|
|
|
pub fn from_disk_cache(
|
|
|
user_id: &str,
|
|
|
object_id: &str,
|
|
|
disk_cache: Arc<dyn RevisionDiskCache<Connection, Error = FlowyError>>,
|
|
|
+ configuration: RevisionPersistenceConfiguration,
|
|
|
) -> RevisionPersistence<Connection> {
|
|
|
let object_id = object_id.to_owned();
|
|
|
let user_id = user_id.to_owned();
|
|
@@ -49,6 +77,7 @@ where
|
|
|
disk_cache,
|
|
|
memory_cache,
|
|
|
sync_seq,
|
|
|
+ configuration,
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -64,7 +93,7 @@ where
|
|
|
pub(crate) async fn sync_revision(&self, revision: &Revision) -> FlowyResult<()> {
|
|
|
tracing::Span::current().record("rev_id", &revision.rev_id);
|
|
|
self.add(revision.clone(), RevisionState::Sync, false).await?;
|
|
|
- self.sync_seq.write().await.add(revision.rev_id)?;
|
|
|
+ self.sync_seq.write().await.dry_push(revision.rev_id)?;
|
|
|
Ok(())
|
|
|
}
|
|
|
|
|
@@ -72,44 +101,39 @@ where
|
|
|
#[tracing::instrument(level = "trace", skip_all, fields(rev_id, compact_range, object_id=%self.object_id), err)]
|
|
|
pub(crate) async fn add_sync_revision<'a>(
|
|
|
&'a self,
|
|
|
- revision: &'a Revision,
|
|
|
- rev_compress: &Arc<dyn RevisionCompress + 'a>,
|
|
|
+ new_revision: &'a Revision,
|
|
|
+ rev_compress: &Arc<dyn RevisionMergeable + 'a>,
|
|
|
) -> FlowyResult<i64> {
|
|
|
let mut sync_seq_write_guard = self.sync_seq.write().await;
|
|
|
- let result = sync_seq_write_guard.compact();
|
|
|
- match result {
|
|
|
- None => {
|
|
|
- tracing::Span::current().record("rev_id", &revision.rev_id);
|
|
|
- self.add(revision.clone(), RevisionState::Sync, true).await?;
|
|
|
- sync_seq_write_guard.add(revision.rev_id)?;
|
|
|
- Ok(revision.rev_id)
|
|
|
- }
|
|
|
- Some((range, mut compact_seq)) => {
|
|
|
- tracing::Span::current().record("compact_range", &format!("{}", range).as_str());
|
|
|
- let mut revisions = self.revisions_in_range(&range).await?;
|
|
|
- if range.to_rev_ids().len() != revisions.len() {
|
|
|
- debug_assert_eq!(range.to_rev_ids().len(), revisions.len());
|
|
|
- }
|
|
|
-
|
|
|
- // append the new revision
|
|
|
- revisions.push(revision.clone());
|
|
|
-
|
|
|
- // compact multiple revisions into one
|
|
|
- let compact_revision = rev_compress.compress_revisions(&self.user_id, &self.object_id, revisions)?;
|
|
|
- let rev_id = compact_revision.rev_id;
|
|
|
- tracing::Span::current().record("rev_id", &rev_id);
|
|
|
-
|
|
|
- // insert new revision
|
|
|
- compact_seq.push_back(rev_id);
|
|
|
-
|
|
|
- // replace the revisions in range with compact revision
|
|
|
- self.compact(&range, compact_revision).await?;
|
|
|
- //
|
|
|
- debug_assert_eq!(compact_seq.len(), 2);
|
|
|
- debug_assert_eq!(sync_seq_write_guard.len(), compact_seq.len());
|
|
|
- sync_seq_write_guard.reset(compact_seq);
|
|
|
- Ok(rev_id)
|
|
|
- }
|
|
|
+ if sync_seq_write_guard.step > self.configuration.merge_threshold {
|
|
|
+ let compact_seq = sync_seq_write_guard.compact();
|
|
|
+ let range = RevisionRange {
|
|
|
+ start: *compact_seq.front().unwrap(),
|
|
|
+ end: *compact_seq.back().unwrap(),
|
|
|
+ };
|
|
|
+
|
|
|
+ tracing::Span::current().record("compact_range", &format!("{}", range).as_str());
|
|
|
+ let mut revisions = self.revisions_in_range(&range).await?;
|
|
|
+ debug_assert_eq!(range.len() as usize, revisions.len());
|
|
|
+ // append the new revision
|
|
|
+ revisions.push(new_revision.clone());
|
|
|
+
|
|
|
+ // compact multiple revisions into one
|
|
|
+ let compact_revision = rev_compress.merge_revisions(&self.user_id, &self.object_id, revisions)?;
|
|
|
+ let rev_id = compact_revision.rev_id;
|
|
|
+ tracing::Span::current().record("rev_id", &rev_id);
|
|
|
+
|
|
|
+ // insert new revision
|
|
|
+ let _ = sync_seq_write_guard.dry_push(rev_id)?;
|
|
|
+
|
|
|
+ // replace the revisions in range with compact revision
|
|
|
+ self.compact(&range, compact_revision).await?;
|
|
|
+ Ok(rev_id)
|
|
|
+ } else {
|
|
|
+ tracing::Span::current().record("rev_id", &new_revision.rev_id);
|
|
|
+ self.add(new_revision.clone(), RevisionState::Sync, true).await?;
|
|
|
+ sync_seq_write_guard.push(new_revision.rev_id)?;
|
|
|
+ Ok(new_revision.rev_id)
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -132,6 +156,10 @@ where
|
|
|
self.sync_seq.read().await.next_rev_id()
|
|
|
}
|
|
|
|
|
|
+ pub(crate) fn number_of_sync_records(&self) -> usize {
|
|
|
+ self.memory_cache.number_of_sync_records()
|
|
|
+ }
|
|
|
+
|
|
|
/// The cache gets reset while it conflicts with the remote revisions.
|
|
|
#[tracing::instrument(level = "trace", skip(self, revisions), err)]
|
|
|
pub(crate) async fn reset(&self, revisions: Vec<Revision>) -> FlowyResult<()> {
|
|
@@ -257,27 +285,42 @@ impl<C> RevisionMemoryCacheDelegate for Arc<dyn RevisionDiskCache<C, Error = Flo
|
|
|
}
|
|
|
|
|
|
#[derive(Default)]
|
|
|
-struct DeferSyncSequence(VecDeque<i64>);
|
|
|
+struct DeferSyncSequence {
|
|
|
+ rev_ids: VecDeque<i64>,
|
|
|
+ start: Option<usize>,
|
|
|
+ step: usize,
|
|
|
+}
|
|
|
+
|
|
|
impl DeferSyncSequence {
|
|
|
fn new() -> Self {
|
|
|
DeferSyncSequence::default()
|
|
|
}
|
|
|
|
|
|
- fn add(&mut self, new_rev_id: i64) -> FlowyResult<()> {
|
|
|
+ fn push(&mut self, new_rev_id: i64) -> FlowyResult<()> {
|
|
|
+ let _ = self.dry_push(new_rev_id)?;
|
|
|
+
|
|
|
+ self.step += 1;
|
|
|
+ if self.start.is_none() && !self.rev_ids.is_empty() {
|
|
|
+ self.start = Some(self.rev_ids.len() - 1);
|
|
|
+ }
|
|
|
+ Ok(())
|
|
|
+ }
|
|
|
+
|
|
|
+ fn dry_push(&mut self, new_rev_id: i64) -> FlowyResult<()> {
|
|
|
// The last revision's rev_id must be greater than the new one.
|
|
|
- if let Some(rev_id) = self.0.back() {
|
|
|
+ if let Some(rev_id) = self.rev_ids.back() {
|
|
|
if *rev_id >= new_rev_id {
|
|
|
return Err(
|
|
|
FlowyError::internal().context(format!("The new revision's id must be greater than {}", rev_id))
|
|
|
);
|
|
|
}
|
|
|
}
|
|
|
- self.0.push_back(new_rev_id);
|
|
|
+ self.rev_ids.push_back(new_rev_id);
|
|
|
Ok(())
|
|
|
}
|
|
|
|
|
|
fn ack(&mut self, rev_id: &i64) -> FlowyResult<()> {
|
|
|
- let cur_rev_id = self.0.front().cloned();
|
|
|
+ let cur_rev_id = self.rev_ids.front().cloned();
|
|
|
if let Some(pop_rev_id) = cur_rev_id {
|
|
|
if &pop_rev_id != rev_id {
|
|
|
let desc = format!(
|
|
@@ -286,38 +329,38 @@ impl DeferSyncSequence {
|
|
|
);
|
|
|
return Err(FlowyError::internal().context(desc));
|
|
|
}
|
|
|
- let _ = self.0.pop_front();
|
|
|
+ let _ = self.rev_ids.pop_front();
|
|
|
}
|
|
|
Ok(())
|
|
|
}
|
|
|
|
|
|
fn next_rev_id(&self) -> Option<i64> {
|
|
|
- self.0.front().cloned()
|
|
|
- }
|
|
|
-
|
|
|
- fn reset(&mut self, new_seq: VecDeque<i64>) {
|
|
|
- self.0 = new_seq;
|
|
|
+ self.rev_ids.front().cloned()
|
|
|
}
|
|
|
|
|
|
fn clear(&mut self) {
|
|
|
- self.0.clear();
|
|
|
- }
|
|
|
-
|
|
|
- fn len(&self) -> usize {
|
|
|
- self.0.len()
|
|
|
+ self.start = None;
|
|
|
+ self.step = 0;
|
|
|
+ self.rev_ids.clear();
|
|
|
}
|
|
|
|
|
|
// Compact the rev_ids into one except the current synchronizing rev_id.
|
|
|
- fn compact(&self) -> Option<(RevisionRange, VecDeque<i64>)> {
|
|
|
- // Make sure there are two rev_id going to sync. No need to compact if there is only
|
|
|
- // one rev_id in queue.
|
|
|
- self.next_rev_id()?;
|
|
|
-
|
|
|
- let mut new_seq = self.0.clone();
|
|
|
- let mut drained = new_seq.drain(1..).collect::<VecDeque<_>>();
|
|
|
+ fn compact(&mut self) -> VecDeque<i64> {
|
|
|
+ if self.start.is_none() {
|
|
|
+ return VecDeque::default();
|
|
|
+ }
|
|
|
|
|
|
- let start = drained.pop_front()?;
|
|
|
- let end = drained.pop_back().unwrap_or(start);
|
|
|
- Some((RevisionRange { start, end }, new_seq))
|
|
|
+ let start = self.start.unwrap();
|
|
|
+ let compact_seq = self.rev_ids.split_off(start);
|
|
|
+ self.start = None;
|
|
|
+ self.step = 0;
|
|
|
+ compact_seq
|
|
|
+
|
|
|
+ // let mut new_seq = self.rev_ids.clone();
|
|
|
+ // let mut drained = new_seq.drain(1..).collect::<VecDeque<_>>();
|
|
|
+ //
|
|
|
+ // let start = drained.pop_front()?;
|
|
|
+ // let end = drained.pop_back().unwrap_or(start);
|
|
|
+ // Some((RevisionRange { start, end }, new_seq))
|
|
|
}
|
|
|
}
|