Files
jap_learn/src/lang.rs
T
2026-08-18 12:47:06 +03:00

737 lines
23 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
use crate::data_provider::card_stats::{delete_stat, load_stats_of_set, update_stat_score};
use crate::data_provider::history::{push_note, HistoryItem};
use crate::AppState;
use chrono::{DateTime, Utc};
use rand::distr::weighted::WeightedIndex;
use rand::distr::Distribution;
use rand::prelude::SliceRandom;
use rand::rng;
use rand::rngs::ThreadRng;
use rayon::iter::IndexedParallelIterator;
use rayon::iter::IntoParallelRefIterator;
use rayon::iter::ParallelIterator;
use rhai::{Engine, Scope};
use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSqlOutput, ValueRef};
use rusqlite::ToSql;
use std::cmp::PartialEq;
use std::collections::HashMap;
use std::fmt::{Display, Formatter};
use std::sync::{Arc, Mutex};
use std::time::Instant;
const MAX_HISTORY_LEN: usize = 20;
const MAX_HISTORY_LEN_PART: f32 = 0.33;
const MAX_SCORE: u8 = 25;
const FADE_PER_DAY: f32 = 0.95;
#[derive(Clone, PartialEq, Eq, Debug, Hash, Copy)]
pub(crate) struct Id(u32);
pub const INVALID_ID: Id = Id(0);
#[derive(Clone, Debug)]
pub struct KanaSet {
name: String,
pub(crate) chars_type: KanaType,
pub(crate) dictionary: Vec<Vec<(String, String)>>,
pub(crate) include_map: [bool; 10],
}
#[derive(Clone, Debug)]
pub enum KanaType {
Hiragana,
Katakana,
}
impl KanaSet {
pub fn hiragana() -> Self {
Self {
name: "Хирагана".to_string(),
chars_type: KanaType::Hiragana,
dictionary: vec![
vec![
(String::from("ぁ"), String::from("a")),
(String::from("ぃ"), String::from("i")),
(String::from("ぅ"), String::from("u")),
(String::from("ぇ"), String::from("e")),
(String::from("ぉ"), String::from("o")),
],
vec![
(String::from("さ"), String::from("sa")),
(String::from("し"), String::from("shi")),
(String::from("す"), String::from("su")),
(String::from("せ"), String::from("se")),
(String::from("そ"), String::from("so")),
],
vec![
(String::from("か"), String::from("ka")),
(String::from("き"), String::from("ki")),
(String::from("く"), String::from("ku")),
(String::from("け"), String::from("ke")),
(String::from("こ"), String::from("ko")),
],
// Ряд «та» (たちつてと)
vec![
(String::from("た"), String::from("ta")),
(String::from("ち"), String::from("chi")),
(String::from("つ"), String::from("tsu")),
(String::from("て"), String::from("te")),
(String::from("と"), String::from("to")),
],
// Ряд «на» (なにぬねの)
vec![
(String::from("な"), String::from("na")),
(String::from("に"), String::from("ni")),
(String::from("ぬ"), String::from("nu")),
(String::from("ね"), String::from("ne")),
(String::from("の"), String::from("no")),
],
// Ряд «ха» (はひふへほ)
vec![
(String::from("は"), String::from("ha")),
(String::from("ひ"), String::from("hi")),
(String::from("ふ"), String::from("fu")),
(String::from("へ"), String::from("he")),
(String::from("ほ"), String::from("ho")),
],
// Ряд «ма» (まみむめも)
vec![
(String::from("ま"), String::from("ma")),
(String::from("み"), String::from("mi")),
(String::from("む"), String::from("mu")),
(String::from("め"), String::from("me")),
(String::from("も"), String::from("mo")),
],
// Ряд «я» (やゆよ) — только 3 символа
vec![
(String::from("や"), String::from("ya")),
(String::from("ゆ"), String::from("yu")),
(String::from("よ"), String::from("yo")),
],
// Ряд «ра» (らりるれろ)
vec![
(String::from("ら"), String::from("ra")),
(String::from("り"), String::from("ri")),
(String::from("る"), String::from("ru")),
(String::from("れ"), String::from("re")),
(String::from("ろ"), String::from("ro")),
],
vec![
(String::from("わ"), String::from("wa")),
(String::from("を"), String::from("wo")),
(String::from("ん"), String::from("n")),
],
],
include_map: [true; 10],
}
}
pub fn katakana() -> Self {
Self {
name: "Катакана".to_string(),
chars_type: KanaType::Katakana,
dictionary: vec![
vec![
(String::from("ァ"), String::from("a")),
(String::from("ィ"), String::from("i")),
(String::from("ゥ"), String::from("u")),
(String::from("ェ"), String::from("e")),
(String::from("ォ"), String::from("o")),
],
vec![
(String::from("サ"), String::from("sa")),
(String::from("シ"), String::from("shi")),
(String::from("ス"), String::from("su")),
(String::from("セ"), String::from("se")),
(String::from("ソ"), String::from("so")),
],
vec![
(String::from("カ"), String::from("ka")),
(String::from("キ"), String::from("ki")),
(String::from("ク"), String::from("ku")),
(String::from("ケ"), String::from("ke")),
(String::from("コ"), String::from("ko")),
],
// Ряд «та» (タチツテト)
vec![
(String::from("タ"), String::from("ta")),
(String::from("チ"), String::from("chi")),
(String::from("ツ"), String::from("tsu")),
(String::from("テ"), String::from("te")),
(String::from("ト"), String::from("to")),
],
// Ряд «на» (ナニヌネノ)
vec![
(String::from("ナ"), String::from("na")),
(String::from("ニ"), String::from("ni")),
(String::from("ヌ"), String::from("nu")),
(String::from("ネ"), String::from("ne")),
(String::from(""), String::from("no")),
],
// Ряд «ха» (ハヒフヘホ)
vec![
(String::from("ハ"), String::from("ha")),
(String::from("ヒ"), String::from("hi")),
(String::from("フ"), String::from("fu")),
(String::from("ヘ"), String::from("he")),
(String::from("ホ"), String::from("ho")),
],
// Ряд «ма» (マミムメモ)
vec![
(String::from("マ"), String::from("ma")),
(String::from("ミ"), String::from("mi")),
(String::from("ム"), String::from("mu")),
(String::from("メ"), String::from("me")),
(String::from("モ"), String::from("mo")),
],
// Ряд «я» (ヤユヨ) — только 3 символа
vec![
(String::from("ヤ"), String::from("ya")),
(String::from("ユ"), String::from("yu")),
(String::from("ヨ"), String::from("yo")),
],
// Ряд «ра» (ラリルレロ)
vec![
(String::from("ラ"), String::from("ra")),
(String::from("リ"), String::from("ri")),
(String::from("ル"), String::from("ru")),
(String::from("レ"), String::from("re")),
(String::from("ロ"), String::from("ro")),
],
vec![
(String::from("ワ"), String::from("wa")),
(String::from("ヲ"), String::from("wo")),
(String::from("ン"), String::from("n")),
],
],
include_map: [true; 10],
}
}
pub fn list(&self) -> Vec<(String, String)> {
let mut current_set: Vec<(String, String)> = Vec::new();
for i in 0..10 {
if self.include_map[i] {
self.dictionary[i].iter().for_each(|v| {
current_set.push(v.clone());
});
}
}
current_set
}
}
impl Default for KanaSet {
fn default() -> Self {
KanaSet::hiragana()
}
}
impl PartialEq<Self> for KanaSet {
fn eq(&self, other: &Self) -> bool {
self.name == other.name
}
}
#[derive(Clone, Debug)]
pub struct WordData {
pub id: Id,
pub key: String,
pub value: String,
pub tags: String,
pub additional: HashMap<String, String>,
pub group_id: Id,
}
impl WordData {
pub fn new() -> Self {
Self {
id: 0.into(),
key: String::new(),
value: String::new(),
tags: String::new(),
additional: Default::default(),
group_id: 1.into(),
}
}
}
#[derive(Clone)]
pub struct WordGroup {
pub id: Id,
pub name: String,
}
#[derive(Clone, PartialEq)]
pub struct CardStatistics {
pub id: Id,
pub word_id: Id,
pub set_id: Id,
pub last_open: DateTime<Utc>,
pub score: u8,
}
impl CardStatistics {
pub fn update(&mut self, status: WordOpenMode) {
match status {
WordOpenMode::Easy => {
self.score = (self.calculated_score() + 5.0).round() as u8;
}
WordOpenMode::Ok => self.score = (self.calculated_score() + 2.0).round() as u8,
WordOpenMode::Hard => {
self.score = (self.calculated_score() * 0.75).round() as u8;
}
WordOpenMode::None => {
self.score = (self.calculated_score() * 0.4) as u8;
}
}
self.score = self.score.clamp(1, MAX_SCORE);
self.last_open = Utc::now();
}
pub fn calculated_score(&self) -> f32 {
let time = Utc::now() - self.last_open;
let days = time.num_days();
let multiplier = FADE_PER_DAY.powi(days as i32);
(self.score as f32 * multiplier).max(1.0)
}
}
#[derive(Clone, Copy)]
pub enum WordOpenMode {
Easy,
Ok,
Hard,
None,
}
#[derive(Clone)]
pub struct DeckData {
words: Vec<WordData>,
set: Vec<CardStatistics>,
current_word_index: Option<usize>,
state: Arc<Mutex<AppState>>,
order_module: OrderModule,
settings: DeckSettings,
}
impl DeckData {
pub fn new(settings: &DeckSettings, state: Arc<Mutex<AppState>>) -> Self {
let state_for = state.clone();
let state_locked = state.lock().unwrap();
let mut current_set = load_stats_of_set(settings, &state_locked.connection);
let last_list: Vec<_> = settings
.get_word_list(&state_locked)
.iter()
.map(|w| state_locked.dictionary.get(*w).unwrap())
.cloned()
.collect();
let word_ids = last_list.iter().map(|l| l.id).collect::<Vec<Id>>();
let mut index = 0;
for stat in current_set.clone() {
if !word_ids.contains(&stat.word_id) {
delete_stat(&stat, &state_locked.connection);
current_set.remove(index);
} else {
index += 1;
}
}
Self {
set: current_set,
words: last_list,
current_word_index: None,
state: state_for,
order_module: match settings.open_mode {
OrderMode::Default => OrderModule::SemiRandomSRS(SemiRandomSRSModule::new()),
OrderMode::TrainWorstFirst => {
OrderModule::WorstWordsSRS(WorstWordsSRSModule::new())
}
OrderMode::FullRandom => OrderModule::RandomSRS(RandomSRSModule::new()),
},
settings: settings.clone(),
}
}
pub fn next(&mut self) -> (WordData, CardStatistics) {
let index = match self.order_module.clone() {
OrderModule::SemiRandomSRS(mut module) => {
if !module.initialized {
module.init(self)
}
let index = module.next(self);
self.order_module = OrderModule::SemiRandomSRS(module);
index
}
OrderModule::RandomSRS(mut module) => {
if !module.initialized {
module.init(self)
}
let index = module.next(self);
self.order_module = OrderModule::RandomSRS(module);
index
}
OrderModule::WorstWordsSRS(mut module) => {
if !module.initialized {
module.init(self)
}
let index = module.next(self);
self.order_module = OrderModule::WorstWordsSRS(module);
index
}
};
self.current_word_index = Some(index);
(self.words[index].clone(), self.set[index].clone())
}
pub fn open(&mut self, status: WordOpenMode) {
if self.current_word_index.is_none() {
return;
}
let index = self.current_word_index.unwrap();
let word = &mut self.set[index];
let old_score = word.score;
word.update(status);
let new_score = word.score;
match self.order_module.clone() {
OrderModule::SemiRandomSRS(mut module) => {
module.open(status, index, word.clone());
self.order_module = OrderModule::SemiRandomSRS(module);
}
OrderModule::RandomSRS(mut module) => {
module.open(status, index, word.clone());
self.order_module = OrderModule::RandomSRS(module);
}
OrderModule::WorstWordsSRS(mut module) => {
module.open(status, index, word.clone());
self.order_module = OrderModule::WorstWordsSRS(module);
}
}
update_stat_score(word, &self.state.lock().unwrap().connection);
push_note(
self.settings.id,
HistoryItem {
timestamp: Utc::now(),
word_id: word.word_id.into(),
mode: WordOpenMode::Easy,
before: old_score,
after: new_score,
},
)
}
pub fn len(&self) -> usize {
self.set.len()
}
}
#[derive(Clone, PartialEq, Copy, Eq)]
pub enum OrderMode {
Default,
TrainWorstFirst,
FullRandom,
}
#[derive(Clone, Copy, Eq, PartialEq, Default)]
pub enum AppendMode {
Full,
#[default]
Manual,
}
#[derive(Clone)]
enum OrderModule {
SemiRandomSRS(SemiRandomSRSModule),
RandomSRS(RandomSRSModule),
WorstWordsSRS(WorstWordsSRSModule),
}
trait SRSModule {
fn next(&mut self, set: &mut DeckData) -> usize;
fn open(&mut self, status: WordOpenMode, index: usize, updated_word: CardStatistics);
fn init(&mut self, set: &mut DeckData);
}
#[derive(Clone)]
struct RandomSRSModule {
basket: Vec<usize>,
initialized: bool,
}
impl RandomSRSModule {
fn new() -> RandomSRSModule {
Self {
basket: vec![],
initialized: false,
}
}
}
impl SRSModule for RandomSRSModule {
fn next(&mut self, set: &mut DeckData) -> usize {
if self.basket.is_empty() {
self.basket = (0..set.words.len()).collect::<Vec<usize>>();
self.basket.shuffle(&mut rand::rng())
}
self.basket.pop().unwrap()
}
fn open(&mut self, _: WordOpenMode, _: usize, _: CardStatistics) {}
fn init(&mut self, set: &mut DeckData) {
self.initialized = true;
self.basket = (0..set.words.len()).collect::<Vec<usize>>();
self.basket.shuffle(&mut rand::rng())
}
}
#[derive(Clone)]
struct SemiRandomSRSModule {
history: Vec<usize>,
last_weights: WeightedIndex<f32>,
generator: ThreadRng,
initialized: bool,
}
impl SemiRandomSRSModule {
fn new() -> SemiRandomSRSModule {
SemiRandomSRSModule {
history: vec![],
last_weights: WeightedIndex::new([1.0]).unwrap(),
generator: rng(),
initialized: false,
}
}
}
impl SRSModule for SemiRandomSRSModule {
fn next(&mut self, set: &mut DeckData) -> usize {
let index = self.last_weights.sample(&mut self.generator);
if self.history.contains(&index) {
return self.next(set);
}
if self.history.len() == self.history_len(set) {
self.history.remove(0);
}
self.history.push(index);
index
}
fn open(&mut self, _: WordOpenMode, index: usize, word: CardStatistics) {
let new_weight = (100.0 / word.calculated_score()).powf(2.0);
self.last_weights
.update_weights(&[(index, &new_weight)])
.unwrap();
}
fn init(&mut self, set: &mut DeckData) {
self.initialized = true;
let weights = set
.set
.iter()
.map(|s| (100.0 / s.calculated_score()).powf(2.0) * 2.0)
.collect::<Vec<f32>>();
self.last_weights = WeightedIndex::new(weights).unwrap();
}
}
impl SemiRandomSRSModule {
fn history_len(&self, set: &DeckData) -> usize {
((set.len() as f32 * MAX_HISTORY_LEN_PART) as usize).clamp(1, MAX_HISTORY_LEN)
}
}
#[derive(Clone)]
struct WorstWordsSRSModule {
initialized: bool,
pool_size: usize,
rounds_remaining: u8,
pool: Vec<usize>,
queue: Vec<usize>,
rounds_count: u8,
}
impl SRSModule for WorstWordsSRSModule {
fn next(&mut self, set: &mut DeckData) -> usize {
if self.rounds_remaining == 0 {
self.fill_pool(set);
self.rounds_remaining = self.rounds_count;
}
if self.queue.is_empty() {
self.queue.append(&mut self.pool.clone());
self.rounds_remaining -= 1;
}
self.queue.pop().unwrap()
}
fn open(&mut self, _: WordOpenMode, _: usize, _: CardStatistics) {}
fn init(&mut self, _: &mut DeckData) {
self.initialized = true;
}
}
impl WorstWordsSRSModule {
fn new() -> WorstWordsSRSModule {
WorstWordsSRSModule {
initialized: false,
pool_size: 15,
rounds_count: 2,
rounds_remaining: 0,
pool: vec![],
queue: vec![],
}
}
fn fill_pool(&mut self, set: &DeckData) {
let mut sorted = set
.set
.clone()
.into_iter()
.zip(0..set.set.len())
.collect::<Vec<_>>();
sorted.sort_by_key(|c| c.0.score);
let mut worst = sorted[0..self.pool_size]
.iter()
.map(|(_, index)| *index)
.collect::<Vec<usize>>();
worst.shuffle(&mut rand::rng());
self.pool = worst;
}
}
#[derive(Clone)]
pub struct DeckSettings {
pub id: Id,
pub name: String,
pub forward: String,
pub backward: String,
pub filter: String,
pub count: Option<usize>,
pub worst_words_list: Option<Vec<WordData>>,
pub open_mode: OrderMode,
}
impl DeckSettings {
pub(crate) fn with_name(name: String) -> DeckSettings {
DeckSettings {
id: 0.into(),
name,
forward: "".to_string(),
backward: "".to_string(),
filter: "true".to_string(),
count: None,
worst_words_list: None,
open_mode: OrderMode::Default,
}
}
pub(crate) fn check_filter(&self) -> bool {
let engine = Engine::new();
let ast = engine.compile(&self.filter);
ast.is_ok()
}
pub fn get_word_list(&self, state: &AppState) -> Vec<usize> {
let time = Instant::now();
let mut list = vec![];
let engine = Engine::new();
let ast = engine.compile(&self.filter);
if ast.is_err() {
return list;
}
let ast = ast.unwrap();
let groups = &state.word_groups;
list = state
.dictionary
.par_iter()
.enumerate()
.filter(|(_, word)| {
let mut more = rhai::Map::new();
for iced in &word.additional {
more.insert(iced.0.clone().into(), iced.1.clone().into());
}
let mut scope = Scope::new();
scope
.push_constant("id", word.id.0)
.push_constant("key", word.key.clone())
.push_constant("value", word.value.clone())
.push_constant("tags", word.tags.clone())
.push_constant("more", more)
.push_constant(
"group",
groups
.iter()
.find(|g| g.id == word.group_id)
.cloned()
.unwrap()
.name,
);
let result = engine.eval_ast_with_scope::<bool>(&mut scope, &ast);
result.is_ok() && result.unwrap()
})
.map(|(index, _)| index)
.collect();
println!("Collecting available words is {:?}", time.elapsed());
list
}
pub fn require_speech(&self) -> bool {
self.forward == "speech" || self.backward == "speech"
}
}
impl Into<Id> for u32{
fn into(self) -> Id {
Id(self)
}
}
// impl From<u32> for Id{
// fn from(id: u32) -> Id{
// Id(id)
// }
// }
impl Into<u32> for Id{
fn into(self) -> u32 {
self.0
}
}
impl Display for Id {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0)
}
}
impl FromSql for Id{
fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
match value {
ValueRef::Integer(i) => Ok(Id(i as u32)),
_ => Err(FromSqlError::InvalidType),
}
}
}
impl ToSql for Id{
fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
Ok(ToSqlOutput::from(self.0 as i64))
}
}