steven/gpx.studio
1
0 Fork 0
mirror of https://github.com/gpxstudio/gpx.studio.git synced 2026-10-27 02:27:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

progress

Browse Source
This commit is contained in:
vcoppe committed 2026-10-08 21:04:29 +02:00
1 parent b085a1d476
commit 42a55e6759
51 files changed
+5756 -3854
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

gpx-rs/engine/src/storage/codec.rs
-286
View File
@@ -1,286 +0,0 @@
//! The binary form of what is stored: a version byte, then the data (postcard).
use std::rc::Rc;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::{
File, FileInfo, Track, TrackId, TrackInfo, TrackSegment, TrackSegmentId, Trackpoint,
TrackpointChunk, TrackpointChunkId, Waypoint, WaypointChunk, WaypointChunkId, Waypoints,
};
/// Changes when the form of the records does.
const VERSION: u8 = 1;
pub fn encode<T: Serialize + ?Sized>(value: &T) -> Vec<u8> {
postcard::to_extend(value, vec![VERSION]).expect("serializing to memory")
}
pub fn decode<'a, T: Deserialize<'a>>(bytes: &'a [u8]) -> Option<T> {
match bytes.split_first() {
Some((&VERSION, data)) => postcard::from_bytes(data).ok(),
_ => None,
}
}
/// A file without its trackpoints and waypoints, which are in chunks.
#[derive(Serialize, Deserialize)]
pub struct FileRecord {
info: FileInfo,
tracks: Vec<TrackRecord>,
waypoints: Vec<WaypointChunkId>,
}
#[derive(Serialize, Deserialize)]
struct TrackRecord {
id: TrackId,
info: TrackInfo,
segments: Vec<SegmentRecord>,
}
#[derive(Serialize, Deserialize)]
struct SegmentRecord {
id: TrackSegmentId,
chunks: Vec<TrackpointChunkId>,
}
impl FileRecord {
pub fn new(file: &File) -> Self {
Self {
info: file.info.clone(),
tracks: file
.trk
.iter()
.map(|track| TrackRecord {
id: track.id,
info: track.info.clone(),
segments: track
.trkseg
.iter()
.map(|segment| SegmentRecord {
id: segment.id,
chunks: segment.chunks().iter().map(|chunk| chunk.id).collect(),
})
.collect(),
})
.collect(),
waypoints: file.wpt.chunks().iter().map(|chunk| chunk.id).collect(),
}
}
/// The file made of the record and its chunks, `None` if one is missing. Chunks that several
/// files use stay shared.
pub fn build(
self,
id: crate::FileId,
trackpoint_chunks: &impl Fn(&TrackpointChunkId) -> Option<Rc<TrackpointChunk>>,
waypoint_chunks: &impl Fn(&WaypointChunkId) -> Option<Rc<WaypointChunk>>,
) -> Option<File> {
let mut file = File {
id,
info: self.info,
..Default::default()
};
for track in self.tracks {
let mut trkseg = Vec::with_capacity(track.segments.len());
for segment in track.segments {
let mut built = TrackSegment::default();
built.id = segment.id;
for chunk in &segment.chunks {
built.push_shared(trackpoint_chunks(chunk)?);
}
trkseg.push(built);
}
file.trk.push(Track {
id: track.id,
info: track.info,
trkseg,
});
}
let mut wpt = Waypoints::default();
for chunk in &self.waypoints {
wpt.push_shared(waypoint_chunks(chunk)?);
}
file.wpt = wpt;
Some(file)
}
}
pub fn encode_trackpoints(chunk: &TrackpointChunk) -> Vec<u8> {
encode(&chunk.trkpt)
}
pub fn decode_trackpoints(id: Uuid, bytes: &[u8]) -> Option<TrackpointChunk> {
Some(TrackpointChunk {
id: TrackpointChunkId(id),
trkpt: decode::<Vec<Trackpoint>>(bytes)?,
})
}
pub fn encode_waypoints(chunk: &WaypointChunk) -> Vec<u8> {
encode(&chunk.wpt)
}
pub fn decode_waypoints(id: Uuid, bytes: &[u8]) -> Option<WaypointChunk> {
Some(WaypointChunk {
id: WaypointChunkId(id),
wpt: decode::<Vec<Waypoint>>(bytes)?,
})
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use crate::Chunk;
use super::*;
fn trackpoints(n: usize) -> TrackpointChunk {
TrackpointChunk::new(
(0..n)
.map(|i| Trackpoint {
ele: i as f64,
..Default::default()
})
.collect(),
)
}
#[test]
fn test_encode_decode_round_trip() {
let value = vec![(1_u32, "a".to_string()), (2, "b".to_string())];
let bytes = encode(&value);
assert_eq!(bytes[0], VERSION);
assert_eq!(decode::<Vec<(u32, String)>>(&bytes), Some(value));
}
#[test]
fn test_decode_rejects_other_versions_and_garbage() {
let mut bytes = encode(&42_u32);
assert_eq!(decode::<u32>(&bytes), Some(42));
bytes[0] = VERSION + 1;
assert_eq!(decode::<u32>(&bytes), None);
assert_eq!(decode::<u32>(&[]), None);
// the right version, but not what was asked for
assert_eq!(decode::<Vec<String>>(&[VERSION, 0xff, 0xff, 0xff]), None);
assert_eq!(decode::<u32>(&[VERSION]), None);
}
#[test]
fn test_chunks_keep_their_identity_and_items() {
let chunk = trackpoints(3);
let decoded = decode_trackpoints(chunk.id.0, &encode_trackpoints(&chunk)).unwrap();
assert_eq!(decoded.id, chunk.id);
let eles: Vec<f64> = decoded.trkpt.iter().map(|p| p.ele).collect();
assert_eq!(eles, [0.0, 1.0, 2.0]);
assert!(decode_trackpoints(chunk.id.0, &[VERSION, 9, 9]).is_none());
let chunk = WaypointChunk::new(vec![Waypoint {
name: Some("summit".into()),
..Default::default()
}]);
let decoded = decode_waypoints(chunk.id.0, &encode_waypoints(&chunk)).unwrap();
assert_eq!(decoded.id, chunk.id);
assert_eq!(decoded.wpt[0].name.as_deref(), Some("summit"));
assert!(decode_waypoints(chunk.id.0, &[]).is_none());
}
fn file() -> File {
let mut file = File::default();
file.info.name = "file".into();
for name in ["a", "b"] {
let mut track = Track::default();
track.info.name = Some(name.into());
let mut segment = TrackSegment::default();
segment.push(trackpoints(2));
segment.push(trackpoints(1));
track.trkseg.push(segment);
track.trkseg.push(TrackSegment::default());
file.trk.push(track);
}
file.wpt.push(WaypointChunk::new(vec![Waypoint::default()]));
file
}
/// The chunks of a file, by identity.
fn chunks_of(
file: &File,
) -> (
HashMap<TrackpointChunkId, Rc<TrackpointChunk>>,
HashMap<WaypointChunkId, Rc<WaypointChunk>>,
) {
let trackpoints = file
.trk
.iter()
.flat_map(|track| &track.trkseg)
.flat_map(|segment| segment.chunks().iter().map(|c| (c.id, c.clone())))
.collect();
let waypoints = file
.wpt
.chunks()
.iter()
.map(|c| (c.id, c.clone()))
.collect();
(trackpoints, waypoints)
}
#[test]
fn test_file_record_rebuilds_the_file_with_the_same_chunks() {
let file = file();
let (trackpoints, waypoints) = chunks_of(&file);
let record: FileRecord = decode(&encode(&FileRecord::new(&file))).unwrap();
let built = record
.build(file.id, &|id| trackpoints.get(id).cloned(), &|id| {
waypoints.get(id).cloned()
})
.unwrap();
assert_eq!(built.id, file.id);
assert_eq!(built.info, file.info);
assert_eq!(built.trk.len(), 2);
for (built, original) in built.trk.iter().zip(&file.trk) {
assert_eq!(built.id, original.id);
assert_eq!(built.info, original.info);
assert_eq!(built.trkseg.len(), 2);
for (built, original) in built.trkseg.iter().zip(&original.trkseg) {
assert_eq!(built.id, original.id);
assert_eq!(built.len(), original.len());
for (a, b) in built.chunks().iter().zip(original.chunks()) {
// the chunks are the ones that were given, not copies
assert!(Rc::ptr_eq(a, b));
}
}
}
assert_eq!(built.wpt.len(), 1);
assert!(Rc::ptr_eq(&built.wpt.chunks()[0], &file.wpt.chunks()[0]));
}
#[test]
fn test_file_record_of_a_missing_chunk_builds_nothing() {
let file = file();
let (trackpoints, waypoints) = chunks_of(&file);
let record = || FileRecord::new(&file);
assert!(
record()
.build(file.id, &|_| None, &|id| waypoints.get(id).cloned())
.is_none()
);
assert!(
record()
.build(file.id, &|id| trackpoints.get(id).cloned(), &|_| None)
.is_none()
);
}
#[test]
fn test_file_record_without_data_builds_an_empty_file() {
let file = File::default();
let built = FileRecord::new(&file)
.build(file.id, &|_| None, &|_| None)
.unwrap();
assert!(built.trk.is_empty() && built.wpt.is_empty());
}
}
gpx-rs/engine/src/storage/codec/mod.rs
+579
View File
@@ -0,0 +1,579 @@
//! The binary form of what is stored.
//!
//! Every blob starts with a version byte, then the data (postcard, which does not name the fields:
//! the form is exactly the types that wrote it). So a form never changes once it has been
//! written: the types of each version live in their own module (see [`v1`]) and are frozen, and
//! a change is a new version next to them. Blobs of any known version can be read; they are
//! written again in the latest version when they are written again. Chunks are never rewritten
//! unless they change, so blobs of several versions live together in a storage.
//!
//! To change the form: add `vN`, make `LATEST` its number, convert from it in the decoders, and
//! keep the golden bytes of the older versions in the tests.
mod v1;
use std::rc::Rc;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::{
File, FileId, FileInfo, Track, TrackId, TrackInfo, TrackSegment, TrackSegmentId, Trackpoint,
TrackpointCategories, TrackpointChunk, TrackpointChunkId, Waypoint, WaypointChunk,
WaypointChunkId, Waypoints,
};
/// The version that is written.
pub const LATEST: u8 = 1;
/// Why a blob could not be read.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DecodeError {
/// It was written by a version that this one does not know: a newer one, or not a blob.
UnknownVersion(u8),
/// It has a known version, but not what it should have.
Corrupt,
}
fn encode<T: Serialize + ?Sized>(value: &T) -> Vec<u8> {
postcard::to_extend(value, vec![LATEST]).expect("serializing to memory")
}
/// The version of a blob, and its data.
fn split(bytes: &[u8]) -> Result<(u8, &[u8]), DecodeError> {
bytes
.split_first()
.map(|(v, data)| (*v, data))
.ok_or(DecodeError::Corrupt)
}
fn parse<'a, T: Deserialize<'a>>(data: &'a [u8]) -> Result<T, DecodeError> {
postcard::from_bytes(data).map_err(|_| DecodeError::Corrupt)
}
/// A file without its trackpoints and waypoints, which are in chunks.
pub struct FileRecord {
info: FileInfo,
tracks: Vec<TrackRecord>,
waypoints: Vec<WaypointChunkId>,
}
struct TrackRecord {
id: TrackId,
info: TrackInfo,
segments: Vec<SegmentRecord>,
}
struct SegmentRecord {
id: TrackSegmentId,
chunks: Vec<TrackpointChunkId>,
}
impl From<v1::FileRecord1> for FileRecord {
fn from(record: v1::FileRecord1) -> Self {
Self {
info: record.info.into(),
tracks: record
.tracks
.into_iter()
.map(|track| TrackRecord {
id: TrackId(track.id),
info: track.info.into(),
segments: track
.segments
.into_iter()
.map(|segment| SegmentRecord {
id: TrackSegmentId(segment.id),
chunks: segment.chunks.into_iter().map(TrackpointChunkId).collect(),
})
.collect(),
})
.collect(),
waypoints: record.waypoints.into_iter().map(WaypointChunkId).collect(),
}
}
}
impl FileRecord {
/// The file made of the record and its chunks, `None` if one is missing. Chunks that several
/// files use stay shared.
pub fn build(
self,
id: FileId,
trackpoint_chunks: &impl Fn(&TrackpointChunkId) -> Option<Rc<TrackpointChunk>>,
waypoint_chunks: &impl Fn(&WaypointChunkId) -> Option<Rc<WaypointChunk>>,
) -> Option<File> {
let mut file = File {
id,
info: self.info,
..Default::default()
};
for track in self.tracks {
let mut trkseg = Vec::with_capacity(track.segments.len());
for segment in track.segments {
let mut built = TrackSegment::default();
built.id = segment.id;
for chunk in &segment.chunks {
built.push_shared(trackpoint_chunks(chunk)?);
}
trkseg.push(built);
}
file.trk.push(Track {
id: track.id,
info: track.info,
trkseg,
});
}
let mut wpt = Waypoints::default();
for chunk in &self.waypoints {
wpt.push_shared(waypoint_chunks(chunk)?);
}
file.wpt = wpt;
Some(file)
}
}
pub fn encode_file(file: &File) -> Vec<u8> {
encode(&v1::FileRecord1::new(file))
}
pub fn decode_file(bytes: &[u8]) -> Result<FileRecord, DecodeError> {
match split(bytes)? {
(1, data) => Ok(parse::<v1::FileRecord1>(data)?.into()),
(version, _) => Err(DecodeError::UnknownVersion(version)),
}
}
pub fn encode_trackpoints(chunk: &TrackpointChunk) -> Vec<u8> {
encode(
&chunk
.trkpt
.iter()
.map(v1::Trackpoint1::from)
.collect::<Vec<_>>(),
)
}
pub fn decode_trackpoints(id: Uuid, bytes: &[u8]) -> Result<TrackpointChunk, DecodeError> {
let trkpt = match split(bytes)? {
(1, data) => parse::<Vec<v1::Trackpoint1>>(data)?
.into_iter()
.map(Trackpoint::from)
.collect(),
(version, _) => return Err(DecodeError::UnknownVersion(version)),
};
Ok(TrackpointChunk {
id: TrackpointChunkId(id),
trkpt,
})
}
pub fn encode_waypoints(chunk: &WaypointChunk) -> Vec<u8> {
encode(
&chunk
.wpt
.iter()
.map(v1::Waypoint1::from)
.collect::<Vec<_>>(),
)
}
pub fn decode_waypoints(id: Uuid, bytes: &[u8]) -> Result<WaypointChunk, DecodeError> {
let wpt = match split(bytes)? {
(1, data) => parse::<Vec<v1::Waypoint1>>(data)?
.into_iter()
.map(Waypoint::from)
.collect(),
(version, _) => return Err(DecodeError::UnknownVersion(version)),
};
Ok(WaypointChunk {
id: WaypointChunkId(id),
wpt,
})
}
pub fn encode_order(order: &[FileId]) -> Vec<u8> {
encode(&order.iter().map(|id| id.0).collect::<Vec<_>>())
}
pub fn decode_order(bytes: &[u8]) -> Result<Vec<FileId>, DecodeError> {
match split(bytes)? {
(1, data) => Ok(parse::<Vec<Uuid>>(data)?.into_iter().map(FileId).collect()),
(version, _) => Err(DecodeError::UnknownVersion(version)),
}
}
pub fn encode_categories(categories: &TrackpointCategories) -> Vec<u8> {
encode(&v1::Categories1::from(categories))
}
pub fn decode_categories(bytes: &[u8]) -> Result<TrackpointCategories, DecodeError> {
match split(bytes)? {
(1, data) => Ok(parse::<v1::Categories1>(data)?.into()),
(version, _) => Err(DecodeError::UnknownVersion(version)),
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use crate::{Author, Link, LngLat};
use super::*;
fn id(n: u128) -> Uuid {
Uuid::from_u128(n)
}
fn link(href: &str) -> Link {
Link {
href: href.into(),
text: Some(format!("text of {href}")),
}
}
fn trackpoint(n: u8) -> Trackpoint {
let f = f64::from(n);
Trackpoint {
coordinates: LngLat {
lng: 4.0 + f,
lat: 50.0 + f,
},
ele: 100.5 + f,
time: Some(1_700_000_000_000 + i64::from(n)),
atemp: Some(-3),
hr: Some(140 + u16::from(n)),
cad: Some(80),
power: Some(250),
surface: Some(1),
highway: Some(2),
sac_scale: Some(3),
mtb_scale: Some(4),
anchor: Some(n),
}
}
fn waypoint() -> Waypoint {
Waypoint {
id: crate::WaypointId(id(7)),
coordinates: LngLat {
lng: 5.5,
lat: 51.5,
},
ele: 321.0,
time: Some(1_700_000_000_000),
name: Some("summit".into()),
cmt: Some("comment".into()),
desc: Some("description".into()),
link: Some(link("https://example.com/summit")),
sym: Some("Summit".into()),
type_: Some("peak".into()),
}
}
/// A file that has a value for every field, with fixed ids.
fn sample() -> File {
let mut file = File {
id: FileId(id(1)),
info: FileInfo {
name: "sample".into(),
desc: Some("about".into()),
author: Some(Author {
name: Some("someone".into()),
email: Some("someone@example.com".into()),
link: Some(link("https://example.com/someone")),
}),
link: Some(link("https://example.com/file")),
time: Some(1_700_000_000_000),
},
..Default::default()
};
let mut segment = TrackSegment::default();
segment.id = TrackSegmentId(id(3));
segment.push(TrackpointChunk {
id: TrackpointChunkId(id(4)),
trkpt: vec![trackpoint(0), trackpoint(1), Trackpoint::default()],
});
let mut empty = TrackSegment::default();
empty.id = TrackSegmentId(id(6));
file.trk.push(Track {
id: TrackId(id(2)),
info: TrackInfo {
name: Some("track".into()),
cmt: Some("cmt".into()),
desc: Some("desc".into()),
src: Some("src".into()),
link: Some(link("https://example.com/track")),
type_: Some("Cycling".into()),
color: Some("ff0000".into()),
opacity: Some(0.5),
width: Some(6.0),
},
trkseg: vec![segment, empty],
});
file.wpt.push(WaypointChunk {
id: WaypointChunkId(id(5)),
wpt: vec![
waypoint(),
Waypoint {
id: crate::WaypointId(id(8)),
..Default::default()
},
],
});
file
}
fn categories() -> TrackpointCategories {
let mut categories = TrackpointCategories::default();
categories.surface.code("asphalt");
categories.surface.code("gravel");
categories.highway.code("path");
categories.sac_scale.code("hiking");
categories.mtb_scale.code("mtb0");
categories
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
fn unhex(text: &str) -> Vec<u8> {
(0..text.len())
.step_by(2)
.map(|i| u8::from_str_radix(&text[i..i + 2], 16).unwrap())
.collect()
}
/// The chunks of a file, by identity.
fn chunks_of(
file: &File,
) -> (
HashMap<TrackpointChunkId, Rc<TrackpointChunk>>,
HashMap<WaypointChunkId, Rc<WaypointChunk>>,
) {
let trackpoints = file
.trk
.iter()
.flat_map(|track| &track.trkseg)
.flat_map(|segment| segment.chunks().iter().map(|c| (c.id, c.clone())))
.collect();
let waypoints = file
.wpt
.chunks()
.iter()
.map(|c| (c.id, c.clone()))
.collect();
(trackpoints, waypoints)
}
fn rebuilt(file: &File, record: FileRecord) -> Option<File> {
let (trackpoints, waypoints) = chunks_of(file);
record.build(file.id, &|id| trackpoints.get(id).cloned(), &|id| {
waypoints.get(id).cloned()
})
}
fn assert_same_file(a: &File, b: &File) {
assert_eq!(a.id, b.id);
assert_eq!(a.info, b.info);
assert_eq!(a.trk.len(), b.trk.len());
for (a, b) in a.trk.iter().zip(&b.trk) {
assert_eq!((a.id, &a.info), (b.id, &b.info));
assert_eq!(a.trkseg.len(), b.trkseg.len());
for (a, b) in a.trkseg.iter().zip(&b.trkseg) {
assert_eq!(a.id, b.id);
assert_eq!(a.len(), b.len());
for (a, b) in a.chunks().iter().zip(b.chunks()) {
assert!(Rc::ptr_eq(a, b), "the chunks are the ones that were given");
}
}
}
assert_eq!(a.wpt.len(), b.wpt.len());
for (a, b) in a.wpt.chunks().iter().zip(b.wpt.chunks()) {
assert!(Rc::ptr_eq(a, b));
}
}
#[test]
fn test_a_file_record_rebuilds_the_file_with_the_same_chunks() {
let file = sample();
let record = decode_file(&encode_file(&file)).unwrap();
assert_same_file(&rebuilt(&file, record).unwrap(), &file);
}
#[test]
fn test_a_file_record_of_a_missing_chunk_builds_nothing() {
let file = sample();
let (trackpoints, waypoints) = chunks_of(&file);
let record = || decode_file(&encode_file(&file)).unwrap();
assert!(
record()
.build(file.id, &|_| None, &|id| waypoints.get(id).cloned())
.is_none()
);
assert!(
record()
.build(file.id, &|id| trackpoints.get(id).cloned(), &|_| None)
.is_none()
);
}
#[test]
fn test_a_file_without_data_builds_an_empty_file() {
let file = File::default();
let built = decode_file(&encode_file(&file))
.unwrap()
.build(file.id, &|_| None, &|_| None)
.unwrap();
assert!(built.trk.is_empty() && built.wpt.is_empty());
}
#[test]
fn test_chunks_keep_their_identity_and_items() {
let chunk = &sample().trk[0].trkseg[0].chunks()[0].clone();
let decoded = decode_trackpoints(chunk.id.0, &encode_trackpoints(chunk)).unwrap();
assert_eq!(decoded.id, chunk.id);
assert_eq!(decoded.trkpt.len(), 3);
for (a, b) in decoded.trkpt.iter().zip(&chunk.trkpt) {
assert_eq!(format!("{a:?}"), format!("{b:?}"));
}
let chunk = &sample().wpt.chunks()[0].clone();
let decoded = decode_waypoints(chunk.id.0, &encode_waypoints(chunk)).unwrap();
assert_eq!(decoded.id, chunk.id);
for (a, b) in decoded.wpt.iter().zip(&chunk.wpt) {
assert_eq!(a.id, b.id);
assert!(a.same_content(b));
}
}
#[test]
fn test_order_and_categories_round_trip() {
let order = vec![FileId(id(9)), FileId(id(8))];
assert_eq!(decode_order(&encode_order(&order)).unwrap(), order);
let categories = categories();
assert_eq!(
decode_categories(&encode_categories(&categories)).unwrap(),
categories
);
}
#[test]
fn test_blobs_say_their_version_and_unknown_ones_are_told_apart_from_corrupt_ones() {
assert_eq!(encode_file(&sample())[0], LATEST);
assert_eq!(encode_order(&[])[0], LATEST);
let mut bytes = encode_order(&[FileId(id(1))]);
bytes[0] = LATEST + 1;
assert_eq!(
decode_order(&bytes),
Err(DecodeError::UnknownVersion(LATEST + 1))
);
assert_eq!(decode_order(&[0]), Err(DecodeError::UnknownVersion(0)));
assert!(matches!(
decode_file(&bytes),
Err(DecodeError::UnknownVersion(_))
));
assert!(matches!(
decode_trackpoints(id(1), &bytes),
Err(DecodeError::UnknownVersion(_))
));
assert!(matches!(
decode_waypoints(id(1), &bytes),
Err(DecodeError::UnknownVersion(_))
));
assert!(matches!(
decode_categories(&bytes),
Err(DecodeError::UnknownVersion(_))
));
// nothing at all, or a known version with something else
assert_eq!(decode_order(&[]), Err(DecodeError::Corrupt));
assert_eq!(
decode_file(&[LATEST, 0xff, 0xff]).err(),
Some(DecodeError::Corrupt)
);
assert_eq!(
decode_trackpoints(id(1), &[LATEST, 9, 9, 9]).err(),
Some(DecodeError::Corrupt)
);
assert_eq!(decode_categories(&[LATEST]), Err(DecodeError::Corrupt));
}
// The stored forms of version 1, as they were written when the version was made. They must
// stay readable, whatever happens to the engine types: if a test below fails, the form was
// changed by mistake, or a version was not added when it should have been.
const FILE_V1: &str = concat!(
"010673616d706c65010561626f7574010107736f6d656f6e650113736f6d656f6e65406578616d706c652e63",
"6f6d011b68747470733a2f2f6578616d706c652e636f6d2f736f6d656f6e65012374657874206f6620687474",
"70733a2f2f6578616d706c652e636f6d2f736f6d656f6e65011868747470733a2f2f6578616d706c652e636f",
"6d2f66696c65012074657874206f662068747470733a2f2f6578616d706c652e636f6d2f66696c650180a0ab",
"fef9620110000000000000000000000000000000020105747261636b0103636d740104646573630103737263",
"011968747470733a2f2f6578616d706c652e636f6d2f747261636b012174657874206f662068747470733a2f",
"2f6578616d706c652e636f6d2f747261636b01074379636c696e67010666663030303001000000000000e03f",
"0100000000000018400210000000000000000000000000000000030110000000000000000000000000000000",
"04100000000000000000000000000000000600011000000000000000000000000000000005",
);
const TRACKPOINTS_V1: &str = concat!(
"01030000000000001040000000000000494000000000002059400180a0abfef9620105018c01015001fa0101",
"0101020103010401000000000000001440000000000080494000000000006059400182a0abfef9620105018d",
"01015001fa010101010201030104010100000000000000000000000000000000000000000000000000000000",
"000000000000",
);
const WAYPOINTS_V1: &str = concat!(
"0102100000000000000000000000000000000700000000000016400000000000c04940000000000010744001",
"80a0abfef962010673756d6d69740107636f6d6d656e74010b6465736372697074696f6e011a68747470733a",
"2f2f6578616d706c652e636f6d2f73756d6d6974012274657874206f662068747470733a2f2f6578616d706c",
"652e636f6d2f73756d6d6974010653756d6d697401047065616b100000000000000000000000000000000800",
"000000000000000000000000000000000000000000000000000000000000",
);
const ORDER_V1: &str =
"010210000000000000000000000000000000091000000000000000000000000000000008";
const CATEGORIES_V1: &str =
"01020761737068616c740667726176656c010470617468010668696b696e6701046d746230";
#[test]
fn test_version_1_is_still_written_as_it_was() {
let file = sample();
assert_eq!(hex(&encode_file(&file)), FILE_V1);
assert_eq!(
hex(&encode_trackpoints(&file.trk[0].trkseg[0].chunks()[0])),
TRACKPOINTS_V1
);
assert_eq!(hex(&encode_waypoints(&file.wpt.chunks()[0])), WAYPOINTS_V1);
assert_eq!(
hex(&encode_order(&[FileId(id(9)), FileId(id(8))])),
ORDER_V1
);
assert_eq!(hex(&encode_categories(&categories())), CATEGORIES_V1);
}
#[test]
fn test_version_1_is_still_read_as_it_was() {
let file = sample();
assert_same_file(
&rebuilt(&file, decode_file(&unhex(FILE_V1)).unwrap()).unwrap(),
&file,
);
let chunk = decode_trackpoints(id(4), &unhex(TRACKPOINTS_V1)).unwrap();
let expected = &file.trk[0].trkseg[0].chunks()[0];
assert_eq!(
format!("{:?}", chunk.trkpt),
format!("{:?}", expected.trkpt)
);
let chunk = decode_waypoints(id(5), &unhex(WAYPOINTS_V1)).unwrap();
assert_eq!(chunk.wpt.len(), 2);
assert!(chunk.wpt[0].same_content(&waypoint()));
assert_eq!(chunk.wpt[0].id, waypoint().id);
assert_eq!(
decode_order(&unhex(ORDER_V1)).unwrap(),
vec![FileId(id(9)), FileId(id(8))]
);
assert_eq!(
decode_categories(&unhex(CATEGORIES_V1)).unwrap(),
categories()
);
}
}
gpx-rs/engine/src/storage/codec/v1.rs
+318
View File
@@ -0,0 +1,318 @@
//! Version 1 of the stored form. It is frozen: the engine types may change, these may not (a test
//! compares the bytes they give with the ones of a fixture). A change of the form is a new
//! version, next to this one, that the decoders convert from.
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::{
Author, Categories, File, FileInfo, Link, LngLat, TrackInfo, Trackpoint, TrackpointCategories,
Waypoint, WaypointId,
};
#[derive(Serialize, Deserialize)]
pub struct Link1 {
href: String,
text: Option<String>,
}
impl From<&Link> for Link1 {
fn from(link: &Link) -> Self {
Self {
href: link.href.clone(),
text: link.text.clone(),
}
}
}
impl From<Link1> for Link {
fn from(link: Link1) -> Self {
Self {
href: link.href,
text: link.text,
}
}
}
#[derive(Serialize, Deserialize)]
pub struct Author1 {
name: Option<String>,
email: Option<String>,
link: Option<Link1>,
}
#[derive(Serialize, Deserialize)]
pub struct FileInfo1 {
name: String,
desc: Option<String>,
author: Option<Author1>,
link: Option<Link1>,
time: Option<i64>,
}
impl From<&FileInfo> for FileInfo1 {
fn from(info: &FileInfo) -> Self {
Self {
name: info.name.clone(),
desc: info.desc.clone(),
author: info.author.as_ref().map(|author| Author1 {
name: author.name.clone(),
email: author.email.clone(),
link: author.link.as_ref().map(Into::into),
}),
link: info.link.as_ref().map(Into::into),
time: info.time,
}
}
}
impl From<FileInfo1> for FileInfo {
fn from(info: FileInfo1) -> Self {
Self {
name: info.name,
desc: info.desc,
author: info.author.map(|author| Author {
name: author.name,
email: author.email,
link: author.link.map(Into::into),
}),
link: info.link.map(Into::into),
time: info.time,
}
}
}
#[derive(Serialize, Deserialize)]
pub struct TrackInfo1 {
name: Option<String>,
cmt: Option<String>,
desc: Option<String>,
src: Option<String>,
link: Option<Link1>,
type_: Option<String>,
color: Option<String>,
opacity: Option<f64>,
width: Option<f64>,
}
impl From<&TrackInfo> for TrackInfo1 {
fn from(info: &TrackInfo) -> Self {
Self {
name: info.name.clone(),
cmt: info.cmt.clone(),
desc: info.desc.clone(),
src: info.src.clone(),
link: info.link.as_ref().map(Into::into),
type_: info.type_.clone(),
color: info.color.clone(),
opacity: info.opacity,
width: info.width,
}
}
}
impl From<TrackInfo1> for TrackInfo {
fn from(info: TrackInfo1) -> Self {
Self {
name: info.name,
cmt: info.cmt,
desc: info.desc,
src: info.src,
link: info.link.map(Into::into),
type_: info.type_,
color: info.color,
opacity: info.opacity,
width: info.width,
}
}
}
/// A file without its trackpoints and waypoints, which are in chunks.
#[derive(Serialize, Deserialize)]
pub struct FileRecord1 {
pub info: FileInfo1,
pub tracks: Vec<TrackRecord1>,
pub waypoints: Vec<Uuid>,
}
#[derive(Serialize, Deserialize)]
pub struct TrackRecord1 {
pub id: Uuid,
pub info: TrackInfo1,
pub segments: Vec<SegmentRecord1>,
}
#[derive(Serialize, Deserialize)]
pub struct SegmentRecord1 {
pub id: Uuid,
pub chunks: Vec<Uuid>,
}
impl FileRecord1 {
pub fn new(file: &File) -> Self {
Self {
info: (&file.info).into(),
tracks: file
.trk
.iter()
.map(|track| TrackRecord1 {
id: track.id.0,
info: (&track.info).into(),
segments: track
.trkseg
.iter()
.map(|segment| SegmentRecord1 {
id: segment.id.0,
chunks: segment.chunks().iter().map(|chunk| chunk.id.0).collect(),
})
.collect(),
})
.collect(),
waypoints: file.wpt.chunks().iter().map(|chunk| chunk.id.0).collect(),
}
}
}
#[derive(Serialize, Deserialize)]
pub struct Trackpoint1 {
lng: f64,
lat: f64,
ele: f64,
time: Option<i64>,
atemp: Option<i16>,
hr: Option<u16>,
cad: Option<u16>,
power: Option<u16>,
surface: Option<u8>,
highway: Option<u8>,
sac_scale: Option<u8>,
mtb_scale: Option<u8>,
anchor: Option<u8>,
}
impl From<&Trackpoint> for Trackpoint1 {
fn from(point: &Trackpoint) -> Self {
Self {
lng: point.coordinates.lng,
lat: point.coordinates.lat,
ele: point.ele,
time: point.time,
atemp: point.atemp,
hr: point.hr,
cad: point.cad,
power: point.power,
surface: point.surface,
highway: point.highway,
sac_scale: point.sac_scale,
mtb_scale: point.mtb_scale,
anchor: point.anchor,
}
}
}
impl From<Trackpoint1> for Trackpoint {
fn from(point: Trackpoint1) -> Self {
Self {
coordinates: LngLat {
lng: point.lng,
lat: point.lat,
},
ele: point.ele,
time: point.time,
atemp: point.atemp,
hr: point.hr,
cad: point.cad,
power: point.power,
surface: point.surface,
highway: point.highway,
sac_scale: point.sac_scale,
mtb_scale: point.mtb_scale,
anchor: point.anchor,
}
}
}
#[derive(Serialize, Deserialize)]
pub struct Waypoint1 {
id: Uuid,
lng: f64,
lat: f64,
ele: f64,
time: Option<i64>,
name: Option<String>,
cmt: Option<String>,
desc: Option<String>,
link: Option<Link1>,
sym: Option<String>,
type_: Option<String>,
}
impl From<&Waypoint> for Waypoint1 {
fn from(waypoint: &Waypoint) -> Self {
Self {
id: waypoint.id.0,
lng: waypoint.coordinates.lng,
lat: waypoint.coordinates.lat,
ele: waypoint.ele,
time: waypoint.time,
name: waypoint.name.clone(),
cmt: waypoint.cmt.clone(),
desc: waypoint.desc.clone(),
link: waypoint.link.as_ref().map(Into::into),
sym: waypoint.sym.clone(),
type_: waypoint.type_.clone(),
}
}
}
impl From<Waypoint1> for Waypoint {
fn from(waypoint: Waypoint1) -> Self {
Self {
id: WaypointId(waypoint.id),
coordinates: LngLat {
lng: waypoint.lng,
lat: waypoint.lat,
},
ele: waypoint.ele,
time: waypoint.time,
name: waypoint.name,
cmt: waypoint.cmt,
desc: waypoint.desc,
link: waypoint.link.map(Into::into),
sym: waypoint.sym,
type_: waypoint.type_,
}
}
}
/// The names of each table of categories, in the order of their codes.
#[derive(Serialize, Deserialize)]
pub struct Categories1 {
surface: Vec<String>,
highway: Vec<String>,
sac_scale: Vec<String>,
mtb_scale: Vec<String>,
}
impl From<&TrackpointCategories> for Categories1 {
fn from(categories: &TrackpointCategories) -> Self {
let names = |table: &Categories| table.names().to_vec();
Self {
surface: names(&categories.surface),
highway: names(&categories.highway),
sac_scale: names(&categories.sac_scale),
mtb_scale: names(&categories.mtb_scale),
}
}
}
impl From<Categories1> for TrackpointCategories {
fn from(categories: Categories1) -> Self {
Self {
surface: Categories::from_names(categories.surface),
highway: Categories::from_names(categories.highway),
sac_scale: Categories::from_names(categories.sac_scale),
mtb_scale: Categories::from_names(categories.mtb_scale),
}
}
}
gpx-rs/engine/src/storage/memory.rs
+50 -43
View File
@@ -1,9 +1,7 @@
use std::cell::RefCell;
use std::collections::HashMap;
use uuid::Uuid;
use crate::{Batch, FileId, Storage, StorageError, StoredData};
use crate::{Batch, ChunkKey, ChunkKind, FileId, Storage, StorageError, StoredData};
/// A storage that lives as long as the session. It is what an engine without a host storage uses,
/// and what the tests use.
@@ -15,22 +13,30 @@ pub struct MemoryStorage {
#[derive(Debug, Default)]
struct State {
files: HashMap<FileId, Vec<u8>>,
trackpoint_chunks: HashMap<Uuid, Vec<u8>>,
waypoint_chunks: HashMap<Uuid, Vec<u8>>,
chunks: HashMap<ChunkKey, Vec<u8>>,
order: Option<Vec<u8>>,
categories: Option<Vec<u8>>,
settings: HashMap<String, String>,
}
/// How much there is in a storage.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct StorageCounts {
pub files: usize,
pub trackpoint_chunks: usize,
pub waypoint_chunks: usize,
}
impl MemoryStorage {
/// How many files and chunks (trackpoints, waypoints) are stored.
pub fn counts(&self) -> (usize, usize, usize) {
/// How many files and chunks are stored.
pub fn counts(&self) -> StorageCounts {
let state = self.state.borrow();
(
state.files.len(),
state.trackpoint_chunks.len(),
state.waypoint_chunks.len(),
)
let chunks = |kind| state.chunks.keys().filter(|key| key.kind == kind).count();
StorageCounts {
files: state.files.len(),
trackpoint_chunks: chunks(ChunkKind::Trackpoints),
waypoint_chunks: chunks(ChunkKind::Waypoints),
}
}
}
@@ -39,16 +45,7 @@ impl Storage for MemoryStorage {
let state = self.state.borrow();
Ok(StoredData {
files: state.files.iter().map(|(k, v)| (*k, v.clone())).collect(),
trackpoint_chunks: state
.trackpoint_chunks
.iter()
.map(|(k, v)| (*k, v.clone()))
.collect(),
waypoint_chunks: state
.waypoint_chunks
.iter()
.map(|(k, v)| (*k, v.clone()))
.collect(),
chunks: state.chunks.iter().map(|(k, v)| (*k, v.clone())).collect(),
order: state.order.clone(),
categories: state.categories.clone(),
})
@@ -60,17 +57,9 @@ impl Storage for MemoryStorage {
for id in &batch.delete_files {
state.files.remove(id);
}
state
.trackpoint_chunks
.extend(batch.put_trackpoint_chunks.iter().cloned());
for id in &batch.delete_trackpoint_chunks {
state.trackpoint_chunks.remove(id);
}
state
.waypoint_chunks
.extend(batch.put_waypoint_chunks.iter().cloned());
for id in &batch.delete_waypoint_chunks {
state.waypoint_chunks.remove(id);
state.chunks.extend(batch.put_chunks.iter().cloned());
for key in &batch.delete_chunks {
state.chunks.remove(key);
}
if let Some(order) = &batch.order {
state.order = Some(order.clone());
@@ -109,6 +98,8 @@ mod tests {
use std::future::Future;
use std::task::{Context, Poll, Waker};
use uuid::Uuid;
use super::*;
fn block_on<F: Future>(future: F) -> F::Output {
@@ -120,12 +111,20 @@ mod tests {
}
}
fn counts(files: usize, trackpoint_chunks: usize, waypoint_chunks: usize) -> StorageCounts {
StorageCounts {
files,
trackpoint_chunks,
waypoint_chunks,
}
}
#[test]
fn test_empty_storage_has_nothing() {
let storage = MemoryStorage::default();
let data = block_on(storage.load()).unwrap();
assert_eq!(data, StoredData::default());
assert_eq!(storage.counts(), (0, 0, 0));
assert_eq!(storage.counts(), counts(0, 0, 0));
assert!(block_on(storage.load_settings()).unwrap().is_empty());
}
@@ -136,20 +135,28 @@ mod tests {
let (t, w) = (Uuid::new_v4(), Uuid::new_v4());
block_on(storage.commit(&Batch {
put_files: vec![(a, vec![1]), (b, vec![2])],
put_trackpoint_chunks: vec![(t, vec![3])],
put_waypoint_chunks: vec![(w, vec![4])],
put_chunks: vec![
(ChunkKey::trackpoints(t), vec![3]),
(ChunkKey::waypoints(w), vec![4]),
],
order: Some(vec![5]),
categories: Some(vec![6]),
..Default::default()
}))
.unwrap();
assert_eq!(storage.counts(), (2, 1, 1));
assert_eq!(storage.counts(), counts(2, 1, 1));
let mut data = block_on(storage.load()).unwrap();
data.files.sort_by_key(|(_, bytes)| bytes.clone());
assert_eq!(data.files, vec![(a, vec![1]), (b, vec![2])]);
assert_eq!(data.trackpoint_chunks, vec![(t, vec![3])]);
assert_eq!(data.waypoint_chunks, vec![(w, vec![4])]);
data.chunks.sort_by_key(|(_, bytes)| bytes.clone());
assert_eq!(
data.chunks,
vec![
(ChunkKey::trackpoints(t), vec![3]),
(ChunkKey::waypoints(w), vec![4])
]
);
assert_eq!(
(data.order, data.categories),
(Some(vec![5]), Some(vec![6]))
@@ -159,11 +166,11 @@ mod tests {
block_on(storage.commit(&Batch {
put_files: vec![(a, vec![10])],
delete_files: vec![b],
delete_trackpoint_chunks: vec![t],
delete_chunks: vec![ChunkKey::trackpoints(t)],
..Default::default()
}))
.unwrap();
assert_eq!(storage.counts(), (1, 0, 1));
assert_eq!(storage.counts(), counts(1, 0, 1));
let data = block_on(storage.load()).unwrap();
assert_eq!(data.files, vec![(a, vec![10])]);
assert_eq!(
@@ -172,12 +179,12 @@ mod tests {
);
block_on(storage.commit(&Batch {
delete_waypoint_chunks: vec![w],
delete_chunks: vec![ChunkKey::waypoints(w)],
order: Some(vec![]),
..Default::default()
}))
.unwrap();
assert_eq!(storage.counts(), (1, 0, 0));
assert_eq!(storage.counts(), counts(1, 0, 0));
assert_eq!(block_on(storage.load()).unwrap().order, Some(vec![]));
}
gpx-rs/engine/src/storage/mod.rs
+51 -8
View File
@@ -30,13 +30,42 @@ impl std::fmt::Display for StorageError {
impl std::error::Error for StorageError {}
/// What a chunk holds.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ChunkKind {
Trackpoints,
Waypoints,
}
/// Where a chunk is stored: chunks are told apart by their identity within their kind.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct ChunkKey {
pub kind: ChunkKind,
pub id: Uuid,
}
impl ChunkKey {
pub fn trackpoints(id: Uuid) -> Self {
Self {
kind: ChunkKind::Trackpoints,
id,
}
}
pub fn waypoints(id: Uuid) -> Self {
Self {
kind: ChunkKind::Waypoints,
id,
}
}
}
/// Everything that is stored about the files, as opaque bytes (see the module documentation).
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct StoredData {
/// The record of each file.
pub files: Vec<(FileId, Vec<u8>)>,
pub trackpoint_chunks: Vec<(Uuid, Vec<u8>)>,
pub waypoint_chunks: Vec<(Uuid, Vec<u8>)>,
pub chunks: Vec<(ChunkKey, Vec<u8>)>,
/// The order of the files.
pub order: Option<Vec<u8>>,
/// The categories of the trackpoints, which their chunks refer to.
@@ -49,10 +78,8 @@ pub struct StoredData {
pub struct Batch {
pub put_files: Vec<(FileId, Vec<u8>)>,
pub delete_files: Vec<FileId>,
pub put_trackpoint_chunks: Vec<(Uuid, Vec<u8>)>,
pub delete_trackpoint_chunks: Vec<Uuid>,
pub put_waypoint_chunks: Vec<(Uuid, Vec<u8>)>,
pub delete_waypoint_chunks: Vec<Uuid>,
pub put_chunks: Vec<(ChunkKey, Vec<u8>)>,
pub delete_chunks: Vec<ChunkKey>,
pub order: Option<Vec<u8>>,
pub categories: Option<Vec<u8>>,
}
@@ -61,6 +88,22 @@ impl Batch {
pub fn is_empty(&self) -> bool {
self == &Self::default()
}
/// How many chunks of the kind the batch writes.
pub fn puts(&self, kind: ChunkKind) -> usize {
self.put_chunks
.iter()
.filter(|(key, _)| key.kind == kind)
.count()
}
/// How many chunks of the kind the batch deletes.
pub fn deletes(&self, kind: ChunkKind) -> usize {
self.delete_chunks
.iter()
.filter(|key| key.kind == kind)
.count()
}
}
/// Where the data is kept. The implementation belongs to the host; [`MemoryStorage`] is one that
@@ -101,11 +144,11 @@ mod tests {
..Default::default()
},
Batch {
delete_trackpoint_chunks: vec![Uuid::new_v4()],
put_chunks: vec![(ChunkKey::waypoints(Uuid::new_v4()), vec![])],
..Default::default()
},
Batch {
delete_waypoint_chunks: vec![Uuid::new_v4()],
delete_chunks: vec![ChunkKey::trackpoints(Uuid::new_v4())],
..Default::default()
},
Batch {
gpx-rs/engine/src/storage/persister.rs
+154 -64
View File
@@ -1,16 +1,17 @@
use std::cell::RefCell;
use std::cell::{Cell, RefCell};
use std::collections::{HashMap, HashSet};
use std::rc::Rc;
use uuid::Uuid;
use super::codec::{
FileRecord, decode, decode_trackpoints, decode_waypoints, encode, encode_trackpoints,
DecodeError, decode_categories, decode_file, decode_order, decode_trackpoints,
decode_waypoints, encode_categories, encode_file, encode_order, encode_trackpoints,
encode_waypoints,
};
use crate::{
Batch, File, FileId, StackEntry, Storage, StorageError, TrackpointCategories, TrackpointChunk,
WaypointChunk,
Batch, ChunkKey, File, FileId, StackEntry, Storage, StorageError, TrackpointCategories,
TrackpointChunk, WaypointChunk,
};
/// What there is to keep of the engine: the files of the current state, their order, and the
@@ -30,6 +31,10 @@ pub struct Restored {
pub categories: TrackpointCategories,
/// The files that could not be read: they are dropped at the next save.
pub unreadable: usize,
/// Whether some of what is stored was written by a newer version, which this one cannot read.
/// Nothing is written then, so that the newer version finds its data again: the files that
/// could be read are only kept in memory, and the persister does not save.
pub read_only: bool,
}
/// Works out what to write to a [`Storage`] so that it holds a [`Snapshot`], knowing what it
@@ -44,18 +49,35 @@ pub struct Persister {
files: HashMap<FileId, Rc<File>>,
/// The files that have a record, including the ones that could not be read.
stored_files: HashSet<FileId>,
trackpoint_chunks: HashSet<Uuid>,
waypoint_chunks: HashSet<Uuid>,
chunks: HashSet<ChunkKey>,
order: Option<Vec<FileId>>,
categories: Option<TrackpointCategories>,
/// Set by `restore`, see [`Restored::read_only`].
read_only: bool,
}
/// Remembers whether something that was read comes from a newer version.
#[derive(Default)]
struct Newer(Cell<bool>);
impl Newer {
fn read<T>(&self, result: Result<T, DecodeError>) -> Option<T> {
match result {
Ok(value) => Some(value),
Err(error) => {
self.0
.set(self.0.get() || matches!(error, DecodeError::UnknownVersion(_)));
None
}
}
}
}
/// What a save writes, and what is stored once it is written.
struct Plan {
batch: Batch,
snapshot: Snapshot,
trackpoint_chunks: HashSet<Uuid>,
waypoint_chunks: HashSet<Uuid>,
chunks: HashSet<ChunkKey>,
}
impl Persister {
@@ -64,11 +86,18 @@ impl Persister {
let data = storage.load().await?;
*self = Self::default();
let categories: Option<TrackpointCategories> = data.categories.as_deref().and_then(decode);
let stored_order: Option<Vec<FileId>> = data.order.as_deref().and_then(decode);
// what comes from a newer version is not corrupt: it must not be overwritten
let newer = Newer::default();
let categories: Option<TrackpointCategories> = data
.categories
.as_deref()
.and_then(|bytes| newer.read(decode_categories(bytes)));
let stored_order: Option<Vec<FileId>> = data
.order
.as_deref()
.and_then(|bytes| newer.read(decode_order(bytes)));
let trackpoint_bytes: HashMap<Uuid, Vec<u8>> = data.trackpoint_chunks.into_iter().collect();
let waypoint_bytes: HashMap<Uuid, Vec<u8>> = data.waypoint_chunks.into_iter().collect();
let chunk_bytes: HashMap<ChunkKey, Vec<u8>> = data.chunks.into_iter().collect();
// chunks are decoded once, whatever the number of files that use them
let trackpoint_cache: RefCell<HashMap<Uuid, Option<Rc<TrackpointChunk>>>> =
Default::default();
@@ -78,8 +107,8 @@ impl Persister {
.borrow_mut()
.entry(id.0)
.or_insert_with(|| {
let bytes = trackpoint_bytes.get(&id.0)?;
decode_trackpoints(id.0, bytes).map(Rc::new)
let bytes = chunk_bytes.get(&ChunkKey::trackpoints(id.0))?;
newer.read(decode_trackpoints(id.0, bytes)).map(Rc::new)
})
.clone()
};
@@ -88,8 +117,8 @@ impl Persister {
.borrow_mut()
.entry(id.0)
.or_insert_with(|| {
let bytes = waypoint_bytes.get(&id.0)?;
decode_waypoints(id.0, bytes).map(Rc::new)
let bytes = chunk_bytes.get(&ChunkKey::waypoints(id.0))?;
newer.read(decode_waypoints(id.0, bytes)).map(Rc::new)
})
.clone()
};
@@ -101,7 +130,8 @@ impl Persister {
let mut files: HashMap<FileId, Rc<File>> = HashMap::new();
for (id, bytes) in &data.files {
self.stored_files.insert(*id);
let file = decode::<FileRecord>(bytes)
let file = newer
.read(decode_file(bytes))
.and_then(|record| record.build(*id, &trackpoint_chunk, &waypoint_chunk));
match file {
Some(file) => {
@@ -124,10 +154,11 @@ impl Persister {
restored.files = restored.order.iter().map(|id| files[id].clone()).collect();
self.files = files;
self.trackpoint_chunks = trackpoint_bytes.into_keys().collect();
self.waypoint_chunks = waypoint_bytes.into_keys().collect();
self.chunks = chunk_bytes.into_keys().collect();
self.order = stored_order;
self.categories = categories;
self.read_only = newer.0.get();
restored.read_only = self.read_only;
Ok(restored)
}
@@ -137,6 +168,9 @@ impl Persister {
storage: &S,
snapshot: Snapshot,
) -> Result<(), StorageError> {
if self.read_only {
return Ok(());
}
let plan = self.plan(snapshot);
if plan.batch.is_empty() {
self.files = plan.snapshot.files;
@@ -146,8 +180,7 @@ impl Persister {
self.stored_files = plan.snapshot.files.keys().copied().collect();
self.files = plan.snapshot.files;
self.trackpoint_chunks = plan.trackpoint_chunks;
self.waypoint_chunks = plan.waypoint_chunks;
self.chunks = plan.chunks;
self.order = Some(plan.snapshot.order);
self.categories = Some(plan.snapshot.categories);
Ok(())
@@ -156,8 +189,7 @@ impl Persister {
fn plan(&self, snapshot: Snapshot) -> Plan {
let mut batch = Batch::default();
// the chunks that the files use, which the storage has afterwards
let mut trackpoint_chunks = HashSet::new();
let mut waypoint_chunks = HashSet::new();
let mut chunks = HashSet::new();
for (id, file) in &snapshot.files {
let changed = self
@@ -166,23 +198,19 @@ impl Persister {
.is_none_or(|saved| !Rc::ptr_eq(saved, file));
let segments = file.trk.iter().flat_map(|track| &track.trkseg);
for chunk in segments.flat_map(|segment| segment.chunks()) {
let new = trackpoint_chunks.insert(chunk.id.0);
if changed && new && !self.trackpoint_chunks.contains(&chunk.id.0) {
batch
.put_trackpoint_chunks
.push((chunk.id.0, encode_trackpoints(chunk)));
let key = ChunkKey::trackpoints(chunk.id.0);
if chunks.insert(key) && changed && !self.chunks.contains(&key) {
batch.put_chunks.push((key, encode_trackpoints(chunk)));
}
}
for chunk in file.wpt.chunks() {
let new = waypoint_chunks.insert(chunk.id.0);
if changed && new && !self.waypoint_chunks.contains(&chunk.id.0) {
batch
.put_waypoint_chunks
.push((chunk.id.0, encode_waypoints(chunk)));
let key = ChunkKey::waypoints(chunk.id.0);
if chunks.insert(key) && changed && !self.chunks.contains(&key) {
batch.put_chunks.push((key, encode_waypoints(chunk)));
}
}
if changed || !self.stored_files.contains(id) {
batch.put_files.push((*id, encode(&FileRecord::new(file))));
batch.put_files.push((*id, encode_file(file)));
}
}
@@ -192,28 +220,18 @@ impl Persister {
.filter(|id| !snapshot.files.contains_key(id))
.copied()
.collect();
batch.delete_trackpoint_chunks = self
.trackpoint_chunks
.difference(&trackpoint_chunks)
.copied()
.collect();
batch.delete_waypoint_chunks = self
.waypoint_chunks
.difference(&waypoint_chunks)
.copied()
.collect();
batch.delete_chunks = self.chunks.difference(&chunks).copied().collect();
if self.order.as_ref() != Some(&snapshot.order) {
batch.order = Some(encode(&snapshot.order));
batch.order = Some(encode_order(&snapshot.order));
}
if self.categories.as_ref() != Some(&snapshot.categories) {
batch.categories = Some(encode(&snapshot.categories));
batch.categories = Some(encode_categories(&snapshot.categories));
}
Plan {
batch,
snapshot,
trackpoint_chunks,
waypoint_chunks,
chunks,
}
}
}
@@ -225,7 +243,8 @@ mod tests {
use std::task::{Context, Poll, Waker};
use crate::{
Action, Command, Engine, ExportOptions, Load, MemoryStorage, StoredData, parse, write,
Action, ChunkKind, Command, Engine, ExportOptions, Load, MemoryStorage, StoredData, parse,
write,
};
use super::*;
@@ -354,9 +373,8 @@ mod tests {
let batches = storage.batches.borrow();
let batch = batches.last().unwrap();
assert_eq!(batch.put_files.len(), 1);
assert!(batch.put_trackpoint_chunks.is_empty());
assert!(batch.put_waypoint_chunks.is_empty());
assert!(batch.delete_trackpoint_chunks.is_empty());
assert!(batch.put_chunks.is_empty());
assert!(batch.delete_chunks.is_empty());
assert!(batch.order.is_none() && batch.categories.is_none());
}
assert_eq!(storage.inner.counts(), counts);
@@ -376,8 +394,9 @@ mod tests {
let batches = storage.batches.borrow();
let batch = batches.last().unwrap();
assert_eq!(batch.put_files.len(), 1);
assert_eq!(batch.put_trackpoint_chunks.len(), 1);
assert_eq!(batch.delete_trackpoint_chunks.len(), 1);
assert_eq!(batch.puts(ChunkKind::Trackpoints), 1);
assert_eq!(batch.deletes(ChunkKind::Trackpoints), 1);
assert_eq!(batch.puts(ChunkKind::Waypoints), 0);
}
assert_eq!(storage.inner.counts(), counts);
assert!(chunks_before >= 1);
@@ -386,9 +405,9 @@ mod tests {
snapshot.files.remove(&id);
snapshot.order.retain(|other| *other != id);
block_on(persister.save(&storage, snapshot)).unwrap();
let (files, trackpoint_chunks, _) = storage.inner.counts();
assert_eq!(files, counts.0 - 1);
assert!(trackpoint_chunks < counts.1);
let after = storage.inner.counts();
assert_eq!(after.files, counts.files - 1);
assert!(after.trackpoint_chunks < counts.trackpoint_chunks);
}
#[test]
@@ -404,7 +423,7 @@ mod tests {
let mut persister = Persister::default();
block_on(persister.save(&storage, snapshot)).unwrap();
let chunks = original.trk[0].trkseg[0].chunks().len();
assert_eq!(storage.inner.counts().1, chunks);
assert_eq!(storage.inner.counts().trackpoint_chunks, chunks);
let restored = block_on(Persister::default().restore(&storage)).unwrap();
assert_eq!(restored.files.len(), 2);
@@ -424,13 +443,14 @@ mod tests {
// damage a record, and lose a chunk of another file
let broken = snapshot.order[0];
block_on(storage.inner.commit(&Batch {
put_files: vec![(broken, vec![99, 1, 2])],
put_files: vec![(broken, vec![super::super::codec::LATEST, 0xff, 0xff])],
..Default::default()
}))
.unwrap();
let mut persister = Persister::default();
let restored = block_on(persister.restore(&storage)).unwrap();
assert_eq!(restored.unreadable, 1);
assert!(!restored.read_only);
assert_eq!(restored.files.len(), FILES.len() - 1);
assert!(restored.files.iter().all(|file| file.id != broken));
@@ -440,7 +460,77 @@ mod tests {
categories: restored.categories,
};
block_on(persister.save(&storage, kept)).unwrap();
assert_eq!(storage.inner.counts().0, FILES.len() - 1);
assert_eq!(storage.inner.counts().files, FILES.len() - 1);
}
/// A storage holding the files, with one blob replaced by `damage`.
fn stored_with(damage: impl FnOnce(&Snapshot, &mut Batch)) -> (Recording, Snapshot) {
let storage = Recording::default();
let snapshot = snapshot_of(&FILES);
block_on(Persister::default().save(&storage, snapshot.clone())).unwrap();
let mut batch = Batch::default();
damage(&snapshot, &mut batch);
block_on(storage.inner.commit(&batch)).unwrap();
storage.batches.borrow_mut().clear();
(storage, snapshot)
}
fn from_the_future() -> Vec<u8> {
vec![super::super::codec::LATEST + 1, 1, 2, 3]
}
#[test]
fn test_what_a_newer_version_wrote_is_left_alone() {
type Damage = (&'static str, fn(&Snapshot, &mut Batch));
let damages: [Damage; 4] = [
("a file", |snapshot, batch| {
batch.put_files.push((snapshot.order[0], from_the_future()));
}),
("a chunk", |snapshot, batch| {
let file = &snapshot.files[&snapshot.order[0]];
let chunk = &file.trk[0].trkseg[0].chunks()[0];
batch
.put_chunks
.push((ChunkKey::trackpoints(chunk.id.0), from_the_future()));
}),
("the order", |_, batch| {
batch.order = Some(from_the_future())
}),
("the categories", |_, batch| {
batch.categories = Some(from_the_future())
}),
];
for (what, damage) in damages {
let (storage, snapshot) = stored_with(damage);
let before = storage.inner.counts();
let mut persister = Persister::default();
let restored = block_on(persister.restore(&storage)).unwrap();
assert!(restored.read_only, "{what}");
// whatever is saved afterwards, nothing is written: the newer version needs its data
let mut changed = snapshot.clone();
changed.order.clear();
changed.files.clear();
block_on(persister.save(&storage, changed)).unwrap();
assert!(storage.batches.borrow().is_empty(), "{what}");
assert_eq!(storage.inner.counts(), before, "{what}");
}
}
#[test]
fn test_restoring_again_forgets_that_the_storage_was_newer() {
let (storage, _) = stored_with(|snapshot, batch| {
batch.put_files.push((snapshot.order[0], from_the_future()));
});
let mut persister = Persister::default();
assert!(block_on(persister.restore(&storage)).unwrap().read_only);
// the same persister, on a storage that is not newer
let clean = Recording::default();
let restored = block_on(persister.restore(&clean)).unwrap();
assert!(!restored.read_only);
block_on(persister.save(&clean, snapshot_of(&FILES))).unwrap();
assert_eq!(clean.inner.counts().files, FILES.len());
}
#[test]
@@ -449,7 +539,7 @@ mod tests {
let mut engine = Engine::default();
for path in FILES {
let data = std::fs::read(path).unwrap();
assert!(engine.execute(Action::Edit(Command::Load(Load {
assert!(engine.run(Action::Edit(Command::Load(Load {
data: &data,
name: "file",
}))));
@@ -471,14 +561,14 @@ mod tests {
// the files that were restored are not undone
assert!(!engine.can_undo());
assert!(!engine.execute(Action::Undo));
assert!(!engine.run(Action::Undo));
let data = std::fs::read(FILES[0]).unwrap();
assert!(engine.execute(Action::Edit(Command::Load(Load {
assert!(engine.run(Action::Edit(Command::Load(Load {
data: &data,
name: "file",
}))));
assert!(engine.can_undo());
assert!(engine.execute(Action::Undo));
assert!(engine.run(Action::Undo));
assert_eq!(engine.snapshot().files.len(), FILES.len());
assert!(!engine.can_undo());
}