Differences

This shows you the differences between two versions of the page.

--- customanalysis:example [2024/07/23 12:22] – liuxzh12023
+++ customanalysis:example [2024/07/29 11:19] (current) – liuxzh12023
@@ Line 83: / Line 83: @@
     dataset_id: null
     evaluation_value:
-      ate_max_maximun: null
+      ate_max_maximum: null
       ate_max_minimum: null
       ate_max_nolimitation: 1
-      ate_mean_maximun: null
+      ate_mean_maximum: null
       ate_mean_minimum: null
       ate_mean_nolimitation: 1
-      ate_median_maximun: null
+      ate_median_maximum: null
       ate_median_minimum: null
       ate_median_nolimitation: 1
-      ate_min_maximun: null
+      ate_min_maximum: null
       ate_min_minimum: null
       ate_min_nolimitation: 1
-      ate_rmse_maximun: null
+      ate_rmse_maximum: null
       ate_rmse_minimum: null
       ate_rmse_nolimitation: 1
-      ate_sse_maximun: null
+      ate_sse_maximum: null
       ate_sse_minimum: null
       ate_sse_nolimitation: 1
-      ate_std_maximun: null
+      ate_std_maximum: null
       ate_std_minimum: null
       ate_std_nolimitation: 1
-      cpu_max_maximun: null
+      cpu_max_maximum: null
       cpu_max_minimum: null
       cpu_max_nolimitation: 1
-      cpu_mean_maximun: null
+      cpu_mean_maximum: null
       cpu_mean_minimum: null
       cpu_mean_nolimitation: 1
-      ram_max_maximun: null
+      ram_max_maximum: null
       ram_max_minimum: null
       ram_max_nolimitation: 1
-      rpe_max_maximun: null
+      rpe_max_maximum: null
       rpe_max_minimum: null
       rpe_max_nolimitation: 1
-      rpe_mean_maximun: null
+      rpe_mean_maximum: null
       rpe_mean_minimum: null
       rpe_mean_nolimitation: 1
-      rpe_median_maximun: null
+      rpe_median_maximum: null
       rpe_median_minimum: null
       rpe_median_nolimitation: 1
-      rpe_min_maximun: null
+      rpe_min_maximum: null
       rpe_min_minimum: null
       rpe_min_nolimitation: 1
-      rpe_rmse_maximun: null
+      rpe_rmse_maximum: null
       rpe_rmse_minimum: null
       rpe_rmse_nolimitation: 1
-      rpe_sse_maximun: null
+      rpe_sse_maximum: null
       rpe_sse_minimum: null
       rpe_sse_nolimitation: 1
-      rpe_std_maximun: null
+      rpe_std_maximum: null
       rpe_std_minimum: null
       rpe_std_nolimitation: 1
@@ Line 190: / Line 190: @@
 image resolution - totally 1500 configurations
 group_name: Paper_Vision_SLAM_Image_Exploration_Experiment
+</file>
+------
+=== Paper_3_accuracy_metrics_comparison_ATE_RMSE_Experiment ===
+Explanation:
+This analysis task is to calculate the metric of some configurations
+modes
+dataset sequences
+totally: 50
+This YAML file's metric is "ATE RMSE", you can also change to select other metrics.
+<file yaml>
+configuration_choose:
+  comb_configuration_id: []
+  combination_rule:
+    first_one:
+    - 0
+    first_rule:
+    - I
+    second_one:
+    - 2
+    - 1
+    second_rule:
+    - U
+  configuration_id:
+  - 274
+  - 275
+  - 276
+  - 277
+  - 278
+  - 279
+  - 282
+  - 285
+  - 287
+  - 288
+  - 706
+  - 1735
+  - 1585
+  - 722
+  - 754
+  - 762
+  - 1885
+  - 2040
+  - 2190
+  - 2340
+  - 1765
+  - 1615
+  - 778
+  - 794
+  - 826
+  - 834
+  - 1915
+  - 2070
+  - 2220
+  - 2370
+  - 1795
+  - 1645
+  - 850
+  - 866
+  - 882
+  - 890
+  - 1945
+  - 2100
+  - 2250
+  - 2400
+  - 1825
+  - 1675
+  - 906
+  - 978
+  - 954
+  - 962
+  - 1975
+  - 2130
+  - 2280
+  - 2430
+  limitation_rules:
+    algorithm_id: null
+    dataset_id: null
+    evaluation_value:
+      ate_max_maximum: null
+      ate_max_minimum: null
+      ate_max_nolimitation: 1
+      ate_mean_maximum: null
+      ate_mean_minimum: null
+      ate_mean_nolimitation: 1
+      ate_median_maximum: null
+      ate_median_minimum: null
+      ate_median_nolimitation: 1
+      ate_min_maximum: null
+      ate_min_minimum: null
+      ate_min_nolimitation: 1
+      ate_rmse_maximum: null
+      ate_rmse_minimum: null
+      ate_rmse_nolimitation: 1
+      ate_sse_maximum: null
+      ate_sse_minimum: null
+      ate_sse_nolimitation: 1
+      ate_std_maximum: null
+      ate_std_minimum: null
+      ate_std_nolimitation: 1
+      cpu_max_maximum: null
+      cpu_max_minimum: null
+      cpu_max_nolimitation: 1
+      cpu_mean_maximum: null
+      cpu_mean_minimum: null
+      cpu_mean_nolimitation: 1
+      ram_max_maximum: null
+      ram_max_minimum: null
+      ram_max_nolimitation: 1
+      rpe_max_maximum: null
+      rpe_max_minimum: null
+      rpe_max_nolimitation: 1
+      rpe_mean_maximum: null
+      rpe_mean_minimum: null
+      rpe_mean_nolimitation: 1
+      rpe_median_maximum: null
+      rpe_median_minimum: null
+      rpe_median_nolimitation: 1
+      rpe_min_maximum: null
+      rpe_min_minimum: null
+      rpe_min_nolimitation: 1
+      rpe_rmse_maximum: null
+      rpe_rmse_minimum: null
+      rpe_rmse_nolimitation: 1
+      rpe_sse_maximum: null
+      rpe_sse_minimum: null
+      rpe_sse_nolimitation: 1
+      rpe_std_maximum: null
+      rpe_std_minimum: null
+      rpe_std_nolimitation: 1
+    parameters_value: []
+evaluation_form:
+_trajectory_comparison:
+    choose: 0
+_accuracy_metrics_comparison:
+    choose: 0
+_accuracy_metrics_comparison: # Because we only provide one configuration for one group(algorithm+dataset), so this task actually just select the value of the metric. In general case, we provide 2 ways to calculate the metric (average and best).
+    algorithm_id:
+    - 2
+    - 3
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    - 11
+    - 12
+    calculate_method: 1
+    choose: 1
+    dataset_id:
+    - 2
+    - 3
+    - 5
+    - 6
+    - 7
+    metric: ate_rmse # change here to change metric
+_usage_metrics_comparison:
+    choose: 0
+_scatter_diagram:
+    choose: 0
+    x-axis: ate_mean
+    y-axis: cpu_mean
+_scatter_diagram:
+    choose: 0
+    x-axis: null
+    y-axis: null
+_3d_scatter_diagram:
+    choose: 0
+    x-axis: general+image_width
+    y-axis: general+imu_frequency
+_repeatability_test:
+    choose: 0
+    metric: ate_mean
+  algorithm_dataset_type: 3
+group_description: test3
+group_name: Paper_3_accuracy_metrics_comparison_ATE_RMSE_Experiment
+</file>
+------
+=== Paper_1_2_4_Lidar_SLAM_Trajectory_Evo_Usage_Experiment ===
+Explanation:
+This analysis task is to compare some Lidar based SLAM algorithms in trajectory, Evo metrics and CPU and RAM usage.
+modes
+dataset sequences
+totally: 6
+note: 1) Don't select too many configurations to draw them in one diagram. 2) If your selected configurations contain failed trajectories, system will not generate diagram and return an error.
+<file yaml>
+configuration_choose:
+  comb_configuration_id:
+  - 5
+  combination_rule:
+    first_one:
+    - 0
+    first_rule:
+    - I
+    second_one:
+    - 2
+    - 1
+    second_rule:
+    - I
+  configuration_id: [994, 995, 996, 997, 998, 999, 1000]
+  limitation_rules:
+    algorithm_id:
+    - 12
+    - 11
+    - 10
+    dataset_id:
+    - 15
+    evaluation_value:
+      ate_max_maximum: null
+      ate_max_minimum: null
+      ate_max_nolimitation: 1
+      ate_mean_maximum: null
+      ate_mean_minimum: null
+      ate_mean_nolimitation: 1
+      ate_median_maximum: null
+      ate_median_minimum: null
+      ate_median_nolimitation: 1
+      ate_min_maximum: null
+      ate_min_minimum: null
+      ate_min_nolimitation: 1
+      ate_rmse_maximum: null
+      ate_rmse_minimum: null
+      ate_rmse_nolimitation: 1
+      ate_sse_maximum: null
+      ate_sse_minimum: null
+      ate_sse_nolimitation: 1
+      ate_std_maximum: null
+      ate_std_minimum: null
+      ate_std_nolimitation: 1
+      cpu_max_maximum: null
+      cpu_max_minimum: null
+      cpu_max_nolimitation: 1
+      cpu_mean_maximum: null
+      cpu_mean_minimum: null
+      cpu_mean_nolimitation: 1
+      ram_max_maximum: null
+      ram_max_minimum: null
+      ram_max_nolimitation: 1
+      rpe_max_maximum: null
+      rpe_max_minimum: null
+      rpe_max_nolimitation: 1
+      rpe_mean_maximum: null
+      rpe_mean_minimum: null
+      rpe_mean_nolimitation: 1
+      rpe_median_maximum: null
+      rpe_median_minimum: null
+      rpe_median_nolimitation: 1
+      rpe_min_maximum: null
+      rpe_min_minimum: null
+      rpe_min_nolimitation: 1
+      rpe_rmse_maximum: null
+      rpe_rmse_minimum: null
+      rpe_rmse_nolimitation: 1
+      rpe_sse_maximum: null
+      rpe_sse_minimum: null
+      rpe_sse_nolimitation: 1
+      rpe_std_maximum: null
+      rpe_std_minimum: null
+      rpe_std_nolimitation: 1
+    parameters_value:
+    - nFeatures < 4000
+evaluation_form:
+_trajectory_comparison:
+    choose: 1
+_accuracy_metrics_comparison:
+    choose: 1
+_accuracy_metrics_comparison:
+    algorithm_id:
+    - 2
+    - 5
+    calculate_method: 1
+    choose: 0
+    dataset_id:
+    - 2
+    - 13
+    metric: ate_rmse
+_usage_metrics_comparison:
+    choose: 1
+_scatter_diagram:
+    choose: 0
+    x-axis: cpu_mean
+    y-axis: ate_mean
+_3d_scatter_diagram:
+    choose: 0
+    x-axis: null
+    y-axis: null
+    z-axis: null
+_repeatability_test:
+    choose: 0
+    metric: null
+  algorithm_dataset_type: 1
+group_description: Lidar SLAM
+group_name: Paper_1_2_4_Lidar_SLAM_Trajectory_Evo_Usage_Experiment
+</file>
+-------
+=== Paper_8_repeatability_Experiment ===
+Explanation:
+This analysis task is to compare the stability of a configuration. We run 20 times of one configuration.
+configuration with 20 times
+totally: 20
+<file yaml>
+configuration_choose:
+  comb_configuration_id: []
+  combination_rule:
+    first_one:
+    - 0
+    first_rule:
+    - U
+    second_one:
+    - 1
+    - 2
+    second_rule:
+    - I
+  configuration_id:
+  - 531
+  limitation_rules:
+    algorithm_id: null
+    dataset_id: null
+    evaluation_value:
+      ate_max_maximum: null
+      ate_max_minimum: null
+      ate_max_nolimitation: 1
+      ate_mean_maximum: null
+      ate_mean_minimum: null
+      ate_mean_nolimitation: 1
+      ate_median_maximum: null
+      ate_median_minimum: null
+      ate_median_nolimitation: 1
+      ate_min_maximum: null
+      ate_min_minimum: null
+      ate_min_nolimitation: 1
+      ate_rmse_maximum: null
+      ate_rmse_minimum: null
+      ate_rmse_nolimitation: 1
+      ate_sse_maximum: null
+      ate_sse_minimum: null
+      ate_sse_nolimitation: 1
+      ate_std_maximum: null
+      ate_std_minimum: null
+      ate_std_nolimitation: 1
+      cpu_max_maximum: null
+      cpu_max_minimum: null
+      cpu_max_nolimitation: 1
+      cpu_mean_maximum: null
+      cpu_mean_minimum: null
+      cpu_mean_nolimitation: 1
+      ram_max_maximum: null
+      ram_max_minimum: null
+      ram_max_nolimitation: 1
+      rpe_max_maximum: null
+      rpe_max_minimum: null
+      rpe_max_nolimitation: 1
+      rpe_mean_maximum: null
+      rpe_mean_minimum: null
+      rpe_mean_nolimitation: 1
+      rpe_median_maximum: null
+      rpe_median_minimum: null
+      rpe_median_nolimitation: 1
+      rpe_min_maximum: null
+      rpe_min_minimum: null
+      rpe_min_nolimitation: 1
+      rpe_rmse_maximum: null
+      rpe_rmse_minimum: null
+      rpe_rmse_nolimitation: 1
+      rpe_sse_maximum: null
+      rpe_sse_minimum: null
+      rpe_sse_nolimitation: 1
+      rpe_std_maximum: null
+      rpe_std_minimum: null
+      rpe_std_nolimitation: 1
+    parameters_value:
+    - nFeatures < 4000
+evaluation_form:
+_trajectory_comparison:
+    choose: 0
+_accuracy_metrics_comparison:
+    choose: 0
+_accuracy_metrics_comparison:
+    algorithm_id:
+    - 12
+    calculate_method: 1
+    choose: 0
+    dataset_id:
+    - 2
+    metric: ate_rmse
+_usage_metrics_comparison:
+    choose: 0
+_scatter_diagram:
+    choose: 0
+    x-axis: ate_mean
+    y-axis: cpu_mean
+_scatter_diagram:
+    choose: 0
+    x-axis: general+image_frequency
+    y-axis: ate_mean
+_3d_scatter_diagram:
+    choose: 0
+    x-axis: general+image_width
+    y-axis: general+imu_frequency
+    z-axis: ate_mean
+_repeatability_test:
+    choose: 1
+    metric: ate_rmse
+  algorithm_dataset_type: 0
+group_description: random test; 531 (bad dataset)
+group_name: Paper_8_repeatability_Experiment
+</file>
+------
+=== Paper_Vision_SLAM_Image_Exploration_Experiment_limitation ===
+Explanation:
+This experiment explores how the quality of Image data (resolution and framerate) affects Visual-based SLAM algorithms.
+	We run 4 visual-based SLAM algorithms on different sequences of the EuRoC dataset and create 1500 configurations totall. The detail parameter space is shown in \ref{table: exploration3}. And in this experiment, the Extend metric parameters are: internal range: $[1, 0.75), [0.75, 0.5), [0.5, 0.25), [0.25, 0]$; multiple value: $[1, 2, 2.5, 3]$.
+	In the practical application of SLAM algorithm, it is often necessary to meet some specific scenarios, such as: limited computing resources, accuracy requirements, etc.
+	Therefore, in order to analyze such a large number of tasks succinctly, we utilize our system search engine to performs a simple pre-process of the data - Filtering through these 1500 configurations using the conditions: $[0.02 \le ATE.Mean \le 0.1; 0.6 \le CPU.Mean \le 2.0; 600 \le Memory.Max \le 1000]$ results in 278 matching configurations, which are then analyzed with Extend metrics.
+Category:
+orb-slam2-mono:
+  - 147
+  - 148
+  - 149
+  - 150
+  - 151
+orb-slam3-mono:
+  - 133
+  - 134
+  - 135
+  - 136
+  - 137
+orb-slam3-inertial:
+  - 128
+  - 129
+  - 130
+  - 131
+  - 132
+vins-mono:
+  - 152
+  - 153
+  - 154
+  - 155
+  - 156
+vins-fusion-mono-imu:
+  - 163
+  - 164
+  - 165
+  - 166
+  - 167
+orb-slam2-stereo:
+  - 141
+  - 142
+  - 143
+  - 144
+  - 145
+orb-slam3-stereo:
+  - 123
+  - 124
+  - 125
+  - 126
+  - 127
+vins-fusion-stereo:
+  - 169
+  - 170
+  - 171
+  - 172
+  - 173
+orb-slam3-stereo-inertial:
+  - 118
+  - 119
+  - 120
+  - 121
+  - 122
+vins-fusion-stereo-imu:
+  - 174
+  - 175
+  - 176
+  - 177
+  - 178
+<file yaml>
+configuration_choose:
+  comb_configuration_id:
+  - 147
+  - 148
+  - 149
+  - 150
+  - 151
+  - 141
+  - 142
+  - 143
+  - 144
+  - 145
+  - 133
+  - 134
+  - 135
+  - 136
+  - 137
+  - 128
+  - 129
+  - 130
+  - 131
+  - 132
+  - 123
+  - 124
+  - 125
+  - 126
+  - 127
+  - 118
+  - 119
+  - 120
+  - 121
+  - 122
+  - 152
+  - 153
+  - 154
+  - 155
+  - 156
+  - 163
+  - 164
+  - 165
+  - 166
+  - 167
+  - 169
+  - 170
+  - 171
+  - 172
+  - 173
+  - 174
+  - 175
+  - 176
+  - 177
+  - 178
+  combination_rule:
+    first_one:
+    - 1
+    - 2
+    first_rule:
+    - I
+    second_one:
+    - 0
+    second_rule:
+    - I
+  configuration_id: []
+  limitation_rules:
+    algorithm_id:
+    - 2
+    - 3
+    - 5
+    - 6
+    - 7
+    - 8
+    - 9
+    - 10
+    - 11
+    - 12
+    dataset_id:
+    - 2
+    - 3
+    - 5
+    - 6
+    - 7
+    evaluation_value:
+      ate_max_maximum: null
+      ate_max_minimum: null
+      ate_max_nolimitation: 1
+      ate_mean_maximum: null
+      ate_mean_minimum: null
+      ate_mean_nolimitation: 1
+      ate_median_maximum: null
+      ate_median_minimum: null
+      ate_median_nolimitation: 1
+      ate_min_maximum: null
+      ate_min_minimum: null
+      ate_min_nolimitation: 1
+      ate_rmse_maximum: 0.1
+      ate_rmse_minimum: 0.02
+      ate_rmse_nolimitation: 0
+      ate_sse_maximum: null
+      ate_sse_minimum: null
+      ate_sse_nolimitation: 1
+      ate_std_maximum: null
+      ate_std_minimum: null
+      ate_std_nolimitation: 1
+      cpu_max_maximum: null
+      cpu_max_minimum: null
+      cpu_max_nolimitation: 1
+      cpu_mean_maximum: 2
+      cpu_mean_minimum: 0.6
+      cpu_mean_nolimitation: 0
+      ram_max_maximum: 1000
+      ram_max_minimum: 600
+      ram_max_nolimitation: 0
+      rpe_max_maximum: null
+      rpe_max_minimum: null
+      rpe_max_nolimitation: 1
+      rpe_mean_maximum: null
+      rpe_mean_minimum: null
+      rpe_mean_nolimitation: 1
+      rpe_median_maximum: null
+      rpe_median_minimum: null
+      rpe_median_nolimitation: 1
+      rpe_min_maximum: null
+      rpe_min_minimum: null
+      rpe_min_nolimitation: 1
+      rpe_rmse_maximum: null
+      rpe_rmse_minimum: null
+      rpe_rmse_nolimitation: 1
+      rpe_sse_maximum: null
+      rpe_sse_minimum: null
+      rpe_sse_nolimitation: 1
+      rpe_std_maximum: null
+      rpe_std_minimum: null
+      rpe_std_nolimitation: 1
+    parameters_value: []
+evaluation_form:
+_trajectory_comparison:
+    choose: 0
+_accuracy_metrics_comparison:
+    choose: 0
+_accuracy_metrics_comparison:
+    algorithm_id:
+    - 12
+    calculate_method: 1
+    choose: 0
+    dataset_id:
+    - 2
+    metric: ate_rmse
+_usage_metrics_comparison:
+    choose: 0
+_scatter_diagram:
+    choose: 0
+    x-axis: ate_mean
+    y-axis: cpu_mean
+_scatter_diagram:
+    choose: 1
+    extend_choose: 1
+    extend_multiple:
+    - 1
+    - 2
+    - 2.5
+    - 3
+    extend_threshold:
+    - 0.75
+    - 0.5
+    - 0.25
+    x-axis: cpu_mean
+    y-axis: ate_mean
+_3d_scatter_diagram:
+    choose: 1
+    extend_choose: 1
+    extend_multiple:
+    - 1
+    - 2
+    - 2.5
+    - 3
+    extend_threshold:
+    - 0.75
+    - 0.5
+    - 0.25
+    x-axis: general+image_width
+    y-axis: general+image_frequency
+    z-axis: ate_mean
+_repeatability_test:
+    choose: 0
+    metric: null
+  algorithm_dataset_type: 3
+group_description: 10 modes of 4 vision algorithm on 5 sequences; 5 image rate and
+image resolution - totally 1500 configurations
+group_name: Paper_Vision_SLAM_Image_Exploration_Experiment_limitation
 </file>