Add anki21 format and fix multi import bug

Cargo.lock

@@ -2320,6 +2320,7 @@ dependencies = [
  "iced_core",
  "mimalloc",
  "rand",
+ "rayon",
  "reqwest",
  "rfd",
  "rhai",
@@ -2792,6 +2793,15 @@ dependencies = [
  "jni-sys 0.3.0",
 ]
 
+[[package]]
+name = "no-std-compat"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
+dependencies = [
+ "spin",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.4.6"
@@ -3995,6 +4005,7 @@ checksum = "dd4dd0f8c36625202a4ba553c416c19b719947cd2a31d1bda06126e4a5727daf"
 dependencies = [
  "ahash",
  "bitflags 2.10.0",
+ "no-std-compat",
  "num-traits",
  "once_cell",
  "rhai_codegen",
@@ -4591,6 +4602,12 @@ dependencies = [
  "x11rb",
 ]
 
+[[package]]
+name = "spin"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
+
 [[package]]
 name = "spirv"
 version = "0.3.0+sdk-1.3.268.0"

Cargo.toml

@@ -10,7 +10,7 @@ rand = "0.10.2"
 dirs = "6.0.0"
 serde = { version = "1.0.229", features = ["derive"] }
 serde_json = "1.0.151"
-rhai = "1.25.1"
+rhai = { version = "1.25.1", features = ["sync"] }
 chrono = "0.4.45"
 rusqlite = { version = "0.40.2", features = ["chrono", "bundled"] }
 reqwest = { version = "0.13.4", features = ["json", "stream", "multipart"] }
@@ -23,6 +23,7 @@ zip = "8.6.0"
 rfd = "0.17.2"
 mimalloc = "0.1.52"
 hashbrown = "0.17.1"
+rayon = "1.12.0"
 
 [profile.super-release]
 inherits = "release"

src/data_provider/card_sets.rs

@@ -1,5 +1,5 @@
-use crate::lang::SetOrderMode;
+use crate::lang::{AppendMode, CardSetSettings, OrderMode};
-use crate::repetitions::CardSetSettings;
+
 use rusqlite::Connection;
 
 pub fn load_sets(connection: &Connection) -> Vec<CardSetSettings> {
@@ -16,7 +16,8 @@ pub fn load_sets(connection: &Connection) -> Vec<CardSetSettings> {
                 filter: row.get(4)?,
                 count: None,
                 worst_words_list: None,
-                open_mode: SetOrderMode::Default,
+                open_mode: OrderMode::Default,
+                append_mode: AppendMode::Full,
             })
         })
         .unwrap();

src/data_provider/card_stats.rs

@@ -1,5 +1,4 @@
-use crate::lang::CardStatistics;
+use crate::lang::{CardSetSettings, CardStatistics};
-use crate::repetitions::CardSetSettings;
 use rusqlite::Connection;
 use std::time::Instant;
 

src/import.rs

@@ -80,10 +80,12 @@ impl NavigatedPage<ImportMessage> for ImportState {
             Back => {}
             UpdateFile(path) => {
                 self.path = Some(path);
+                self.import_data = None;
                 return self.load_package();
             }
             SelectFile => return Self::select_import_file(),
             UpdateImport(import) => {
+                self.selected_index = 0;
                 self.import_data = Some(import);
             }
             NextGroup => {
@@ -129,7 +131,8 @@ impl NavigatedPage<ImportMessage> for ImportState {
             ImportFinished => {
                 self.progress = None;
                 let group = self
-                    .import_data.as_mut()
+                    .import_data
+                    .as_mut()
                     .unwrap()
                     .0
                     .get_mut(self.selected_index)
@@ -200,7 +203,9 @@ impl ImportState {
 
     fn import_button(&self, group: &ImportGroup) -> Element<'_, ImportMessage> {
         if group.imported {
-            return container(text!("Группа слов успешно импортирована")).padding(HALF_SPACING).align_x(Center)
+            return container(text!("Группа слов успешно импортирована"))
+                .padding(HALF_SPACING)
+                .align_x(Center)
                 .style(success)
                 .width(Fill)
                 .into();
@@ -379,8 +384,6 @@ impl ImportState {
         let file = File::open(path).map_err(|_| ())?;
         let reader = BufReader::new(file);
         let mut archive = ZipArchive::new(reader).map_err(|_| ())?;
-        let db_file = archive.by_name("collection.anki2");
-        if let Ok(mut file) = db_file {
         let temp_file_path = app_data_dir().join("import");
         let mut temp_file = OpenOptions::new()
             .write(true)
@@ -388,6 +391,10 @@ impl ImportState {
             .truncate(true)
             .open(&temp_file_path)
             .map_err(|_| ())?;
+        if let Ok(mut file) = archive.by_name("collection.anki21") {
+            std::io::copy(&mut file, &mut temp_file).map_err(|_| ())?;
+            Ok(())
+        } else if let Ok(mut file) = archive.by_name("collection.anki2") {
             std::io::copy(&mut file, &mut temp_file).map_err(|_| ())?;
             Ok(())
         } else {
@@ -476,15 +483,14 @@ impl ImportState {
                             }
                         }
 
-                        println!("{word:?}");
                         words_list.push(word);
                     }
 
                     let mut state = state.lock().unwrap();
                     let connection = &mut state.connection;
                     let mut index = 0;
-                    let total_len = words_list.len() as f32 / 256.0;
+                    let total_len = words_list.len() as f32 / 1024.0;
-                    for word in &mut words_list.chunks_mut(256) {
+                    for word in &mut words_list.chunks_mut(1024) {
                         add_words(word, connection);
                         index += 1;
                         let mut progress = progress.lock().unwrap();

src/lang.rs

@@ -1,8 +1,7 @@
 use crate::data_provider::card_stats::{
-    add_stat_list, delete_stat, load_stats_of_set, update_stat_score,
+    delete_stat, load_stats_of_set, update_stat_score,
 };
 use crate::data_provider::history::{push_note, HistoryItem};
-use crate::repetitions::CardSetSettings;
 use crate::AppState;
 use chrono::{DateTime, Utc};
 use rand::distr::weighted::WeightedIndex;
@@ -10,6 +9,9 @@ use rand::distr::Distribution;
 use rand::prelude::SliceRandom;
 use rand::rng;
 use rand::rngs::ThreadRng;
+use rayon::iter::IntoParallelRefIterator;
+use rayon::iter::ParallelIterator;
+use rhai::{Engine, Scope};
 use serde::{Deserialize, Serialize};
 use std::cmp::min;
 use std::collections::HashMap;
@@ -315,33 +317,32 @@ impl CardSet {
 
         let mut current_set = load_stats_of_set(settings, &state_locked.connection);
         let last_list = settings.get_word_list(&state_locked);
-        let saved_ids = current_set.iter().map(|l| l.word_id).collect::<Vec<u32>>();
         let word_ids = last_list.iter().map(|l| l.id).collect::<Vec<u32>>();
 
-        let new_stats: &mut Vec<CardStatistics> = &mut last_list
+        // let new_stats: &mut Vec<CardStatistics> = &mut last_list
-            .iter()
+        //     .iter()
-            .filter(|word| !saved_ids.contains(&word.id))
+        //     .filter(|word| !saved_ids.contains(&word.id))
-            .map(|word| CardStatistics {
+        //     .map(|word| CardStatistics {
-                id: 0,
+        //         id: 0,
-                word_id: word.id,
+        //         word_id: word.id,
-                last_open: Utc::now(),
+        //         last_open: Utc::now(),
-                score: 1,
+        //         score: 1,
-                set_id: settings.id,
+        //         set_id: settings.id,
-            })
+        //     })
-            .collect();
+        //     .collect();
-
+        //
-        let time = Instant::now();
+        // let time = Instant::now();
-
+        //
-        if !new_stats.is_empty() {
+        // if !new_stats.is_empty() {
-            add_stat_list(new_stats, &state_locked.connection);
+        //     add_stat_list(new_stats, &state_locked.connection);
-            current_set.append(new_stats);
+        //     current_set.append(new_stats);
-        }
+        // }
-
+        //
-        println!(
+        // println!(
-            "Added {} stats: {}",
+        //     "Added {} stats: {}",
-            new_stats.len(),
+        //     new_stats.len(),
-            time.elapsed().as_millis()
+        //     time.elapsed().as_millis()
-        );
+        // );
         let mut index = 0;
         for stat in current_set.clone() {
             if !word_ids.contains(&stat.word_id) {
@@ -358,11 +359,11 @@ impl CardSet {
             current_word_index: None,
             state: state_for,
             order_module: match settings.open_mode {
-                SetOrderMode::Default => OrderModule::SemiRandomSRS(SemiRandomSRSModule::new()),
+                OrderMode::Default => OrderModule::SemiRandomSRS(SemiRandomSRSModule::new()),
-                SetOrderMode::TrainWorstFirst => {
+                OrderMode::TrainWorstFirst => {
                     OrderModule::WorstWordsSRS(WorstWordsSRSModule::new())
                 }
-                SetOrderMode::FullRandom => OrderModule::RandomSRS(RandomSRSModule::new()),
+                OrderMode::FullRandom => OrderModule::RandomSRS(RandomSRSModule::new()),
             },
             settings: settings.clone(),
         }
@@ -444,12 +445,18 @@ impl CardSet {
 }
 
 #[derive(Clone, PartialEq, Copy, Eq)]
-pub enum SetOrderMode {
+pub enum OrderMode {
     Default,
     TrainWorstFirst,
     FullRandom,
 }
 
+#[derive(Clone, Copy, Eq, PartialEq)]
+pub enum AppendMode{
+    Full,
+    Manual
+}
+
 #[derive(Clone)]
 enum OrderModule {
     SemiRandomSRS(SemiRandomSRSModule),
@@ -619,3 +626,112 @@ impl WorstWordsSRSModule {
         self.pool = worst;
     }
 }
+
+#[derive(Clone)]
+pub struct CardSetSettings {
+    pub id: u32,
+    pub name: String,
+    pub forward: String,
+    pub backward: String,
+    pub filter: String,
+    pub count: Option<usize>,
+    pub worst_words_list: Option<Vec<WordData>>,
+    pub open_mode: OrderMode,
+    pub append_mode: AppendMode
+}
+
+impl CardSetSettings {
+    pub(crate) fn with_name(name: String) -> CardSetSettings {
+        CardSetSettings {
+            id: 0,
+            name,
+            forward: "".to_string(),
+            backward: "".to_string(),
+            filter: "true".to_string(),
+            count: None,
+            worst_words_list: None,
+            open_mode: OrderMode::Default,
+            append_mode: AppendMode::Full
+        }
+    }
+
+    pub(crate) fn check_filter(&self) -> bool {
+        let engine = Engine::new();
+        let ast = engine.compile(&self.filter);
+        ast.is_ok()
+    }
+
+    pub fn get_word_list(&self, state: &AppState) -> Vec<WordData> {
+        let time = Instant::now();
+        let mut list = vec![];
+        let engine = Engine::new();
+
+        let ast = engine.compile(&self.filter);
+        if ast.is_err() {
+            return list;
+        }
+
+        let ast = ast.unwrap();
+
+        let groups = &state.word_groups;
+
+        list = state.dictionary.par_iter().filter(|word| {
+            let mut more = rhai::Map::new();
+            for iced in &word.additional {
+                more.insert(iced.0.clone().into(), iced.1.clone().into());
+            }
+            let mut scope = Scope::new();
+            scope
+                .push_constant("id", word.id)
+                .push_constant("key", word.key.clone())
+                .push_constant("value", word.value.clone())
+                .push_constant("tags", word.tags.clone())
+                .push_constant("more", more)
+                .push_constant(
+                    "group",
+                    groups
+                        .iter()
+                        .find(|g| g.id == word.group_id)
+                        .cloned()
+                        .unwrap()
+                        .name,
+                );
+
+            let result = engine.eval_ast_with_scope::<bool>(&mut scope, &ast);
+            result.is_ok() && result.unwrap()
+        }).cloned().collect();
+
+        println!("Collecting available words is {:?}", time.elapsed());
+
+        list
+    }
+
+    pub fn require_speech(&self) -> bool {
+        self.forward == "speech" || self.backward == "speech"
+    }
+
+    pub(crate) fn update_worst_words(&mut self, state: &AppState) {
+        if self.worst_words_list.is_some() {
+            return;
+        }
+
+        let connection = &state.connection;
+        let mut stats = load_stats_of_set(self, connection);
+        stats.sort_by_key(|s| s.calculated_score() as i32);
+        let avg = stats.iter().map(|s| s.calculated_score()).sum::<f32>() / stats.len() as f32;
+        let avg = avg * 0.7;
+        let bad: Vec<WordData> = stats
+            .iter()
+            .take_while(|word| word.calculated_score() < avg)
+            .map(|stat| {
+                state.dictionary[state
+                    .dictionary
+                    .binary_search_by_key(&stat.word_id, |x| x.id)
+                    .unwrap()]
+                    .clone()
+            })
+            .collect();
+
+        self.worst_words_list = Some(bad.clone());
+    }
+}

src/main.rs

@@ -16,15 +16,15 @@ mod word;
 mod writing;
 mod repetition_settings;
 pub mod import;
+pub mod helpers;
 
 use crate::data_provider::card_sets::load_sets;
 use crate::data_provider::settings::get_setting;
 use crate::data_provider::sqlite::default_connection;
 use crate::data_provider::words::{load_word_groups, load_words};
-use crate::lang::{WordData, WordGroup};
+use crate::lang::{CardSetSettings, WordData, WordGroup};
 use crate::navigation::{AppSettings, RootMessage, ScreenState};
 use crate::quiz::*;
-use crate::repetitions::CardSetSettings;
 use crate::RootMessage::Keyboard;
 use chrono::NaiveDate;
 use iced::{keyboard, Subscription, Theme};

src/repetition.rs

@@ -1,8 +1,7 @@
 use crate::data_provider::voice::get_voice;
-use crate::lang::{CardSet, CardStatistics, WordData, WordOpenMode};
+use crate::lang::{CardSet, CardSetSettings, CardStatistics, WordData, WordOpenMode};
 use crate::navigation::Page::PreviousPage;
 use crate::navigation::{KeyPressedPage, NavigatedPage, Page};
-use crate::repetitions::CardSetSettings;
 use crate::styling::*;
 use crate::{AppState, RootMessage};
 use chrono::Local;

src/repetition_settings.rs

@@ -2,7 +2,6 @@ use crate::data_provider::card_sets::{delete_set, update_card_set};
 use crate::navigation::Page::PreviousPage;
 use crate::navigation::{NavigatedPage, Page, RootMessage};
 use crate::repetition_settings::RepetitionSettingsMessage::*;
-use crate::repetitions::CardSetSettings;
 use crate::styling::*;
 use crate::AppState;
 use iced::widget::button::danger;
@@ -12,6 +11,7 @@ use iced_core::Length::Fill;
 use iced_core::Padding;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
+use crate::lang::CardSetSettings;
 
 pub struct RepetitionSettingsState {
     set: CardSetSettings,

src/repetitions.rs

@@ -1,7 +1,6 @@
 use crate::data_provider::card_sets::{delete_set, update_card_set};
-use crate::data_provider::card_stats::load_stats_of_set;
 use crate::history::HistoryState;
-use crate::lang::{SetOrderMode, WordData};
+use crate::lang::{AppendMode, CardSetSettings, OrderMode};
 use crate::navigation::Page::{History, PreviousPage, Repetition, RepetitionSettings};
 use crate::navigation::{NavigatedPage, Page};
 use crate::repetition::RepetitionState;
@@ -19,7 +18,6 @@ use iced::{Background, Border, Center, Color, Element, Fill, Length, Shadow, Tas
 use iced_core::border::Radius;
 use iced_core::svg::Handle;
 use iced_core::Padding;
-use rhai::{Engine, Scope};
 use std::sync::{Arc, Mutex};
 
 #[derive(Clone)]
@@ -121,6 +119,10 @@ impl NavigatedPage<RepetitionsMessage> for RepetitionsState {
                     .unwrap()
                     .open_mode = mode;
             }
+            SetAppendMode(mode) => {
+                let set = state.card_sets.get_mut(self.selected_set.unwrap()).unwrap();
+                set.append_mode = mode;
+            }
         }
         Task::none()
     }
@@ -230,11 +232,12 @@ impl RepetitionsState {
                         },
                         radio(
                             "Обычный режим",
-                            SetOrderMode::Default,
+                            OrderMode::Default,
                             Some(set.open_mode),
                             SetOpenMode
                         ),
-                        self.words_words_view(&set),
+                        // self.words_words_view(&set),
+                        self.word_append_panel(&set),
                         button("История").style(jl_button).on_press(GoToHistory),
                     ]
                     .spacing(DEFAULT_SPACING)
@@ -342,13 +345,13 @@ impl RepetitionsState {
             container(scrollable(self.worst_words_list(set)).height(200)).style(bordered_box),
             radio(
                 "Начать с плохих слов",
-                SetOrderMode::TrainWorstFirst,
+                OrderMode::TrainWorstFirst,
                 Some(set.open_mode),
                 SetOpenMode
             ),
             radio(
                 "Полностью случайно",
-                SetOrderMode::FullRandom,
+                OrderMode::FullRandom,
                 Some(set.open_mode),
                 SetOpenMode
             )
@@ -357,6 +360,13 @@ impl RepetitionsState {
         .into()
     }
 
+    fn word_append_panel(&self, set: &CardSetSettings) -> Element<'_, RepetitionsMessage> {
+        column![
+            text!("Режим добавления карточек"),
+            row![radio("Добавлять все доступные", AppendMode::Full, Some(set.append_mode), SetAppendMode), radio("Добавлять вручную", AppendMode::Manual, Some(set.append_mode), SetAppendMode)].spacing(HALF_SPACING),
+        ].spacing(HALF_SPACING).into()
+    }
+
     fn worst_words_list(&self, set: &CardSetSettings) -> Element<'_, RepetitionsMessage> {
         let mut column = Column::new();
 
@@ -421,111 +431,9 @@ pub enum RepetitionsMessage {
     SetBackward(String),
     SetFilter(String),
     TryFilter,
-    SetOpenMode(SetOrderMode),
+    SetOpenMode(OrderMode),
+    SetAppendMode(AppendMode),
     GoToHistory,
     GoToSettings,
 }
 
-#[derive(Clone)]
-pub struct CardSetSettings {
-    pub id: u32,
-    pub name: String,
-    pub forward: String,
-    pub backward: String,
-    pub filter: String,
-    pub count: Option<usize>,
-    pub worst_words_list: Option<Vec<WordData>>,
-    pub open_mode: SetOrderMode,
-}
-
-impl CardSetSettings {
-    fn with_name(name: String) -> CardSetSettings {
-        CardSetSettings {
-            id: 0,
-            name,
-            forward: "".to_string(),
-            backward: "".to_string(),
-            filter: "true".to_string(),
-            count: None,
-            worst_words_list: None,
-            open_mode: SetOrderMode::Default,
-        }
-    }
-
-    fn check_filter(&self) -> bool {
-        let engine = Engine::new();
-        let ast = engine.compile(&self.filter);
-        ast.is_ok()
-    }
-
-    pub fn get_word_list(&self, state: &AppState) -> Vec<WordData> {
-        let mut list = vec![];
-        let engine = Engine::new();
-        let ast = engine.compile(&self.filter);
-        if ast.is_err() {
-            return list;
-        }
-
-        let ast = ast.unwrap();
-
-        let groups = &state.word_groups;
-
-        for word in &state.dictionary {
-            let mut more = rhai::Map::new();
-            for iced in &word.additional {
-                more.insert(iced.0.clone().into(), iced.1.clone().into());
-            }
-            let mut scope = Scope::new();
-            scope
-                .push_constant("key", word.key.clone())
-                .push_constant("value", word.value.clone())
-                .push_constant("tags", word.tags.clone())
-                .push_constant("more", more)
-                .push_constant(
-                    "group",
-                    groups
-                        .iter()
-                        .find(|g| g.id == word.group_id)
-                        .cloned()
-                        .unwrap()
-                        .name,
-                );
-
-            let result = engine.eval_ast_with_scope::<bool>(&mut scope, &ast);
-            if result.is_ok() && result.unwrap() {
-                list.push(word.clone());
-            }
-        }
-
-        list
-    }
-
-    pub fn require_speech(&self) -> bool {
-        self.forward == "speech" || self.backward == "speech"
-    }
-
-    fn update_worst_words(&mut self, state: &AppState) {
-        if self.worst_words_list.is_some() {
-            return;
-        }
-
-        let connection = &state.connection;
-        let mut stats = load_stats_of_set(self, connection);
-        stats.sort_by_key(|s| s.calculated_score() as i32);
-        let avg = stats.iter().map(|s| s.calculated_score()).sum::<f32>() / stats.len() as f32;
-        let avg = avg * 0.7;
-        let bad: Vec<WordData> = stats
-            .iter()
-            .take_while(|word| word.calculated_score() < avg)
-            .map(|stat| {
-                state.dictionary[state
-                    .dictionary
-                    .binary_search_by_key(&stat.word_id, |x| x.id)
-                    .unwrap()]
-                .clone()
-            })
-            .collect();
-
-        self.worst_words_list = Some(bad.clone());
-    }
-}