Add anki21 format and fix multi import bug

src/lang.rs

@@ -1,8 +1,7 @@
 use crate::data_provider::card_stats::{
-    add_stat_list, delete_stat, load_stats_of_set, update_stat_score,
+    delete_stat, load_stats_of_set, update_stat_score,
 };
 use crate::data_provider::history::{push_note, HistoryItem};
-use crate::repetitions::CardSetSettings;
 use crate::AppState;
 use chrono::{DateTime, Utc};
 use rand::distr::weighted::WeightedIndex;
@@ -10,6 +9,9 @@ use rand::distr::Distribution;
 use rand::prelude::SliceRandom;
 use rand::rng;
 use rand::rngs::ThreadRng;
+use rayon::iter::IntoParallelRefIterator;
+use rayon::iter::ParallelIterator;
+use rhai::{Engine, Scope};
 use serde::{Deserialize, Serialize};
 use std::cmp::min;
 use std::collections::HashMap;
@@ -315,33 +317,32 @@ impl CardSet {
 
         let mut current_set = load_stats_of_set(settings, &state_locked.connection);
         let last_list = settings.get_word_list(&state_locked);
-        let saved_ids = current_set.iter().map(|l| l.word_id).collect::<Vec<u32>>();
         let word_ids = last_list.iter().map(|l| l.id).collect::<Vec<u32>>();
 
-        let new_stats: &mut Vec<CardStatistics> = &mut last_list
-            .iter()
-            .filter(|word| !saved_ids.contains(&word.id))
-            .map(|word| CardStatistics {
-                id: 0,
-                word_id: word.id,
-                last_open: Utc::now(),
-                score: 1,
-                set_id: settings.id,
-            })
-            .collect();
-
-        let time = Instant::now();
-
-        if !new_stats.is_empty() {
-            add_stat_list(new_stats, &state_locked.connection);
-            current_set.append(new_stats);
-        }
-
-        println!(
-            "Added {} stats: {}",
-            new_stats.len(),
-            time.elapsed().as_millis()
-        );
+        // let new_stats: &mut Vec<CardStatistics> = &mut last_list
+        //     .iter()
+        //     .filter(|word| !saved_ids.contains(&word.id))
+        //     .map(|word| CardStatistics {
+        //         id: 0,
+        //         word_id: word.id,
+        //         last_open: Utc::now(),
+        //         score: 1,
+        //         set_id: settings.id,
+        //     })
+        //     .collect();
+        //
+        // let time = Instant::now();
+        //
+        // if !new_stats.is_empty() {
+        //     add_stat_list(new_stats, &state_locked.connection);
+        //     current_set.append(new_stats);
+        // }
+        //
+        // println!(
+        //     "Added {} stats: {}",
+        //     new_stats.len(),
+        //     time.elapsed().as_millis()
+        // );
         let mut index = 0;
         for stat in current_set.clone() {
             if !word_ids.contains(&stat.word_id) {
@@ -358,11 +359,11 @@ impl CardSet {
             current_word_index: None,
             state: state_for,
             order_module: match settings.open_mode {
-                SetOrderMode::Default => OrderModule::SemiRandomSRS(SemiRandomSRSModule::new()),
-                SetOrderMode::TrainWorstFirst => {
+                OrderMode::Default => OrderModule::SemiRandomSRS(SemiRandomSRSModule::new()),
+                OrderMode::TrainWorstFirst => {
                     OrderModule::WorstWordsSRS(WorstWordsSRSModule::new())
                 }
-                SetOrderMode::FullRandom => OrderModule::RandomSRS(RandomSRSModule::new()),
+                OrderMode::FullRandom => OrderModule::RandomSRS(RandomSRSModule::new()),
             },
             settings: settings.clone(),
         }
@@ -444,12 +445,18 @@ impl CardSet {
 }
 
 #[derive(Clone, PartialEq, Copy, Eq)]
-pub enum SetOrderMode {
+pub enum OrderMode {
     Default,
     TrainWorstFirst,
     FullRandom,
 }
 
+#[derive(Clone, Copy, Eq, PartialEq)]
+pub enum AppendMode{
+    Full,
+    Manual
+}
+
 #[derive(Clone)]
 enum OrderModule {
     SemiRandomSRS(SemiRandomSRSModule),
@@ -619,3 +626,112 @@ impl WorstWordsSRSModule {
         self.pool = worst;
     }
 }
+
+#[derive(Clone)]
+pub struct CardSetSettings {
+    pub id: u32,
+    pub name: String,
+    pub forward: String,
+    pub backward: String,
+    pub filter: String,
+    pub count: Option<usize>,
+    pub worst_words_list: Option<Vec<WordData>>,
+    pub open_mode: OrderMode,
+    pub append_mode: AppendMode
+}
+
+impl CardSetSettings {
+    pub(crate) fn with_name(name: String) -> CardSetSettings {
+        CardSetSettings {
+            id: 0,
+            name,
+            forward: "".to_string(),
+            backward: "".to_string(),
+            filter: "true".to_string(),
+            count: None,
+            worst_words_list: None,
+            open_mode: OrderMode::Default,
+            append_mode: AppendMode::Full
+        }
+    }
+
+    pub(crate) fn check_filter(&self) -> bool {
+        let engine = Engine::new();
+        let ast = engine.compile(&self.filter);
+        ast.is_ok()
+    }
+
+    pub fn get_word_list(&self, state: &AppState) -> Vec<WordData> {
+        let time = Instant::now();
+        let mut list = vec![];
+        let engine = Engine::new();
+
+        let ast = engine.compile(&self.filter);
+        if ast.is_err() {
+            return list;
+        }
+
+        let ast = ast.unwrap();
+
+        let groups = &state.word_groups;
+
+        list = state.dictionary.par_iter().filter(|word| {
+            let mut more = rhai::Map::new();
+            for iced in &word.additional {
+                more.insert(iced.0.clone().into(), iced.1.clone().into());
+            }
+            let mut scope = Scope::new();
+            scope
+                .push_constant("id", word.id)
+                .push_constant("key", word.key.clone())
+                .push_constant("value", word.value.clone())
+                .push_constant("tags", word.tags.clone())
+                .push_constant("more", more)
+                .push_constant(
+                    "group",
+                    groups
+                        .iter()
+                        .find(|g| g.id == word.group_id)
+                        .cloned()
+                        .unwrap()
+                        .name,
+                );
+
+            let result = engine.eval_ast_with_scope::<bool>(&mut scope, &ast);
+            result.is_ok() && result.unwrap()
+        }).cloned().collect();
+
+        println!("Collecting available words is {:?}", time.elapsed());
+
+        list
+    }
+
+    pub fn require_speech(&self) -> bool {
+        self.forward == "speech" || self.backward == "speech"
+    }
+
+    pub(crate) fn update_worst_words(&mut self, state: &AppState) {
+        if self.worst_words_list.is_some() {
+            return;
+        }
+
+        let connection = &state.connection;
+        let mut stats = load_stats_of_set(self, connection);
+        stats.sort_by_key(|s| s.calculated_score() as i32);
+        let avg = stats.iter().map(|s| s.calculated_score()).sum::<f32>() / stats.len() as f32;
+        let avg = avg * 0.7;
+        let bad: Vec<WordData> = stats
+            .iter()
+            .take_while(|word| word.calculated_score() < avg)
+            .map(|stat| {
+                state.dictionary[state
+                    .dictionary
+                    .binary_search_by_key(&stat.word_id, |x| x.id)
+                    .unwrap()]
+                    .clone()
+            })
+            .collect();
+
+        self.worst_words_list = Some(bad.clone());
+    }
+}