fix: health endpoint crashing with embedded cache usage

compute percentile on analytics records

.env_sample

@@ -1,2 +1,2 @@
 export POSTGRES_DATABASE_URL="postgres://postgres:password@localhost:5432/postgres"
-export MARIA_DATABASE_URL="mysql://maria:password@localhost:3306/maria"
+export MARIA_DATABASE_URL="mysql://root:password@localhost:3306/maria"

.github/workflows/linux.yml

@@ -129,12 +129,12 @@ jobs:
         if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'mCaptcha/mCaptcha'
         run: make docker-publish
 
-          #      - name: publish bins
+      - name: publish bins
-          #        if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'mCaptcha/mCaptcha'
+        if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'mCaptcha/mCaptcha'
-          #        run: ./scripts/publish.sh publish master latest $DUMBSERVE_PASSWORD
+        run: ./scripts/publish.sh publish master latest $DUMBSERVE_PASSWORD
-          #        env:
+        env:
-          #          DUMBSERVE_PASSWORD: ${{ secrets.DUMBSERVE_PASSWORD }}
+          DUMBSERVE_PASSWORD: ${{ secrets.DUMBSERVE_PASSWORD }}
-          #          GPG_PASSWORD: ${{ secrets.GPG_PASSWORD }}
+          GPG_PASSWORD: ${{ secrets.GPG_PASSWORD }}
 
       - name: generate documentation
         if: matrix.version == 'stable' && (github.repository == 'mCaptcha/mCaptcha')

.nvmrc

@@ -1 +1 @@
-18
+20

db/db-core/src/lib.rs

@@ -292,6 +292,32 @@ pub trait MCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
 
     /// Get all psuedo IDs
     async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult<Vec<String>>;
+
+    /// Track maximum nonce received against captcha levels
+    async fn update_max_nonce_for_level(
+        &self,
+        captcha_key: &str,
+        difficulty_factor: u32,
+        latest_nonce: u32,
+    ) -> DBResult<()>;
+
+    /// Get maximum nonce tracked so far for captcha levels
+    async fn get_max_nonce_for_level(
+        &self,
+        captcha_key: &str,
+        difficulty_factor: u32,
+    ) -> DBResult<u32>;
+
+    /// Get number of analytics entries that are under a certain duration
+    async fn stats_get_num_logs_under_time(&self, duration: u32) -> DBResult<usize>;
+
+    /// Get the entry at a location in the list of analytics entires under a certain time limit
+    /// and sorted in ascending order
+    async fn stats_get_entry_at_location_for_time_limit_asc(
+        &self,
+        duration: u32,
+        location: u32,
+    ) -> DBResult<Option<usize>>;
 }
 
 #[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)]

db/db-core/src/tests.rs

@@ -7,6 +7,29 @@
 use crate::errors::*;
 use crate::prelude::*;
 
+/// easy traffic pattern
+pub const TRAFFIC_PATTERN: TrafficPattern = TrafficPattern {
+    avg_traffic: 500,
+    peak_sustainable_traffic: 5_000,
+    broke_my_site_traffic: Some(10_000),
+};
+
+/// levels for complex captcha config
+pub const LEVELS: [Level; 3] = [
+    Level {
+        difficulty_factor: 1,
+        visitor_threshold: 1,
+    },
+    Level {
+        difficulty_factor: 2,
+        visitor_threshold: 2,
+    },
+    Level {
+        difficulty_factor: 3,
+        visitor_threshold: 3,
+    },
+];
+
 /// test all database functions
 pub async fn database_works<'a, T: MCDatabase>(
     db: &T,
@@ -250,7 +273,6 @@ pub async fn database_works<'a, T: MCDatabase>(
     db.record_confirm(c.key).await.unwrap();
 
     // analytics start
-
     db.analytics_create_psuedo_id_if_not_exists(c.key)
         .await
         .unwrap();
@@ -282,11 +304,31 @@ pub async fn database_works<'a, T: MCDatabase>(
     );
 
     let analytics = CreatePerformanceAnalytics {
-        time: 0,
+        time: 1,
-        difficulty_factor: 0,
+        difficulty_factor: 1,
         worker_type: "wasm".into(),
     };
+
+    assert_eq!(
+        db.stats_get_num_logs_under_time(analytics.time)
+            .await
+            .unwrap(),
+        0
+    );
+
     db.analysis_save(c.key, &analytics).await.unwrap();
+    assert_eq!(
+        db.stats_get_num_logs_under_time(analytics.time)
+            .await
+            .unwrap(),
+        1
+    );
+    assert_eq!(
+        db.stats_get_num_logs_under_time(analytics.time - 1)
+            .await
+            .unwrap(),
+        0
+    );
     let limit = 50;
     let mut offset = 0;
     let a = db.analytics_fetch(c.key, limit, offset).await.unwrap();
@@ -305,11 +347,82 @@ pub async fn database_works<'a, T: MCDatabase>(
         .unwrap();
     assert_eq!(db.analytics_fetch(c.key, 1000, 0).await.unwrap().len(), 0);
     assert!(!db.analytics_captcha_is_published(c.key).await.unwrap());
+
+    let rest_analytics = [
+        CreatePerformanceAnalytics {
+            time: 2,
+            difficulty_factor: 2,
+            worker_type: "wasm".into(),
+        },
+        CreatePerformanceAnalytics {
+            time: 3,
+            difficulty_factor: 3,
+            worker_type: "wasm".into(),
+        },
+        CreatePerformanceAnalytics {
+            time: 4,
+            difficulty_factor: 4,
+            worker_type: "wasm".into(),
+        },
+        CreatePerformanceAnalytics {
+            time: 5,
+            difficulty_factor: 5,
+            worker_type: "wasm".into(),
+        },
+    ];
+    for a in rest_analytics.iter() {
+        db.analysis_save(c.key, &a).await.unwrap();
+    }
+    assert!(db
+        .stats_get_entry_at_location_for_time_limit_asc(1, 2)
+        .await
+        .unwrap()
+        .is_none());
+    assert_eq!(
+        db.stats_get_entry_at_location_for_time_limit_asc(2, 1)
+            .await
+            .unwrap(),
+        Some(2)
+    );
+    assert_eq!(
+        db.stats_get_entry_at_location_for_time_limit_asc(3, 2)
+            .await
+            .unwrap(),
+        Some(3)
+    );
+
     db.analytics_delete_all_records_for_campaign(c.key)
         .await
         .unwrap();
     // analytics end
 
+    // nonce tracking start
+    assert_eq!(
+        db.get_max_nonce_for_level(c.key, l[0].difficulty_factor)
+            .await
+            .unwrap(),
+        0
+    );
+    db.update_max_nonce_for_level(c.key, l[0].difficulty_factor, 1000)
+        .await
+        .unwrap();
+    assert_eq!(
+        db.get_max_nonce_for_level(c.key, l[0].difficulty_factor)
+            .await
+            .unwrap(),
+        1000
+    );
+    db.update_max_nonce_for_level(c.key, l[0].difficulty_factor, 10_000)
+        .await
+        .unwrap();
+    assert_eq!(
+        db.get_max_nonce_for_level(c.key, l[0].difficulty_factor)
+            .await
+            .unwrap(),
+        10_000
+    );
+    // nonce tracking end
+
     assert_eq!(db.fetch_solve(p.username, c.key).await.unwrap().len(), 1);
     assert_eq!(
         db.fetch_config_fetched(p.username, c.key)

db/db-sqlx-maria/.sqlx/query-216478d53870d7785cd0be43f030883ab79eaafb558d9197d09aea3adbd7b0bc.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "INSERT INTO\n                    mcaptcha_track_nonce (level_id, nonce)\n                VALUES  ((\n                    SELECT\n                        level_id\n                    FROM\n                        mcaptcha_levels\n                    WHERE\n                        config_id = (SELECT config_id FROM mcaptcha_config WHERE captcha_key = ?)\n                    AND\n                        difficulty_factor = ?\n                    AND\n                        visitor_threshold = ?\n                    ), ?);",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 4
+    },
+    "nullable": []
+  },
+  "hash": "216478d53870d7785cd0be43f030883ab79eaafb558d9197d09aea3adbd7b0bc"
+}

db/db-sqlx-maria/.sqlx/query-349ba17ff197aca7ee9fbd43e227d181c27ae04702fd6bdb6ddc32aab3bcb1ea.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "UPDATE mcaptcha_track_nonce SET nonce = ?\n                WHERE level_id =  (\n                    SELECT\n                        level_id\n                    FROM\n                        mcaptcha_levels\n                    WHERE\n                        config_id = (SELECT config_id FROM mcaptcha_config WHERE captcha_key = ?)\n                    AND\n                        difficulty_factor = ?\n                    )\n                AND nonce <= ?;",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 4
+    },
+    "nullable": []
+  },
+  "hash": "349ba17ff197aca7ee9fbd43e227d181c27ae04702fd6bdb6ddc32aab3bcb1ea"
+}

db/db-sqlx-maria/.sqlx/query-9bae79667a8cc631541879321e72a40f20cf812584aaf44418089bc7a51e07c4.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "MySQL",
+  "query": "SELECT\n                COUNT(difficulty_factor) AS count\n            FROM\n                mcaptcha_pow_analytics\n            WHERE time <= ?;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": {
+          "type": "LongLong",
+          "flags": "NOT_NULL | BINARY",
+          "char_set": 63,
+          "max_size": 21
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 1
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "9bae79667a8cc631541879321e72a40f20cf812584aaf44418089bc7a51e07c4"
+}

db/db-sqlx-maria/.sqlx/query-9def82dcec9c8d477824182bb2f71044cc264cf2073ab4f60a0000b435ed0f0b.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "MySQL",
+  "query": "INSERT INTO\n                    mcaptcha_track_nonce (level_id, nonce)\n                VALUES  ((\n                    SELECT\n                        level_id\n                    FROM\n                        mcaptcha_levels\n                    WHERE\n                        config_id = (SELECT config_id FROM mcaptcha_config WHERE captcha_key =?)\n                    AND\n                        difficulty_factor = ?\n                    ), ?);",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 3
+    },
+    "nullable": []
+  },
+  "hash": "9def82dcec9c8d477824182bb2f71044cc264cf2073ab4f60a0000b435ed0f0b"
+}

db/db-sqlx-maria/.sqlx/query-b739ec4cfab1ec60947106c8112e931510c3a50a1606facdde0c0ebb540d5beb.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "MySQL",
+  "query": "SELECT nonce FROM mcaptcha_track_nonce\n                WHERE level_id =  (\n                    SELECT\n                        level_id\n                    FROM\n                        mcaptcha_levels\n                    WHERE\n                        config_id = (SELECT config_id FROM mcaptcha_config WHERE captcha_key = ?)\n                    AND\n                        difficulty_factor = ?\n                    );",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "nonce",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL",
+          "char_set": 63,
+          "max_size": 11
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "b739ec4cfab1ec60947106c8112e931510c3a50a1606facdde0c0ebb540d5beb"
+}

db/db-sqlx-maria/.sqlx/query-c4d6ad934e38218931e74ae1c31c6712cbadb40f31bb12e160c9d333c7e3835c.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "MySQL",
+  "query": "SELECT\n            difficulty_factor\n        FROM\n            mcaptcha_pow_analytics\n        WHERE\n            time <= ?\n        ORDER BY difficulty_factor ASC LIMIT 1 OFFSET ?;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "difficulty_factor",
+        "type_info": {
+          "type": "Long",
+          "flags": "NOT_NULL | NO_DEFAULT_VALUE",
+          "char_set": 63,
+          "max_size": 11
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "c4d6ad934e38218931e74ae1c31c6712cbadb40f31bb12e160c9d333c7e3835c"
+}

db/db-sqlx-maria/migrations/20231029000927_mcaptcha_track_nonce.sql

@@ -0,0 +1,12 @@
+-- Add migration script here
+CREATE TABLE IF NOT EXISTS mcaptcha_track_nonce (
+	level_id INTEGER NOT NULL,
+	nonce INTEGER NOT NULL DEFAULT 0,
+	ID INT auto_increment,
+	PRIMARY KEY(ID),
+	CONSTRAINT `fk_mcaptcha_track_nonce_level_id`
+		FOREIGN KEY (level_id)
+		REFERENCES mcaptcha_levels (level_id)
+		ON DELETE CASCADE
+		ON UPDATE CASCADE
+);

db/db-sqlx-maria/src/lib.rs

@@ -433,6 +433,39 @@ impl MCDatabase for Database {
             futs.push(fut);
         }
 
+        try_join_all(futs)
+            .await
+            .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        let mut futs = Vec::with_capacity(levels.len());
+
+        for level in levels.iter() {
+            let difficulty_factor = level.difficulty_factor as i32;
+            let visitor_threshold = level.visitor_threshold as i32;
+            let fut = sqlx::query!(
+                "INSERT INTO
+                    mcaptcha_track_nonce (level_id, nonce)
+                VALUES  ((
+                    SELECT
+                        level_id
+                    FROM
+                        mcaptcha_levels
+                    WHERE
+                        config_id = (SELECT config_id FROM mcaptcha_config WHERE captcha_key = ?)
+                    AND
+                        difficulty_factor = ?
+                    AND
+                        visitor_threshold = ?
+                    ), ?);",
+                &captcha_key,
+                difficulty_factor,
+                visitor_threshold,
+                0,
+            )
+            .execute(&self.pool);
+            futs.push(fut);
+        }
+
         try_join_all(futs)
             .await
             .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
@@ -1087,6 +1120,160 @@ impl MCDatabase for Database {
 
         Ok(res.drain(0..).map(|r| r.psuedo_id).collect())
     }
+
+    /// Track maximum nonce received against captcha levels
+    async fn update_max_nonce_for_level(
+        &self,
+        captcha_key: &str,
+        difficulty_factor: u32,
+        latest_nonce: u32,
+    ) -> DBResult<()> {
+        let latest_nonce = latest_nonce as i64;
+        sqlx::query!(
+                "UPDATE mcaptcha_track_nonce SET nonce = ?
+                WHERE level_id =  (
+                    SELECT
+                        level_id
+                    FROM
+                        mcaptcha_levels
+                    WHERE
+                        config_id = (SELECT config_id FROM mcaptcha_config WHERE captcha_key = ?)
+                    AND
+                        difficulty_factor = ?
+                    )
+                AND nonce <= ?;",
+                latest_nonce,
+                &captcha_key,
+                difficulty_factor as i64,
+                latest_nonce
+            )
+            .execute(&self.pool).await
+        .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        Ok(())
+    }
+
+    /// Get maximum nonce tracked so far for captcha levels
+    async fn get_max_nonce_for_level(
+        &self,
+        captcha_key: &str,
+        difficulty_factor: u32,
+    ) -> DBResult<u32> {
+        struct X {
+            nonce: i32,
+        }
+
+        async fn inner_get_max_nonce(
+            pool: &MySqlPool,
+            captcha_key: &str,
+            difficulty_factor: u32,
+        ) -> DBResult<X> {
+            sqlx::query_as!(
+                X,
+                "SELECT nonce FROM mcaptcha_track_nonce
+                WHERE level_id =  (
+                    SELECT
+                        level_id
+                    FROM
+                        mcaptcha_levels
+                    WHERE
+                        config_id = (SELECT config_id FROM mcaptcha_config WHERE captcha_key = ?)
+                    AND
+                        difficulty_factor = ?
+                    );",
+                &captcha_key,
+                difficulty_factor as i32,
+            )
+                .fetch_one(pool).await
+                .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))
+        }
+
+        let res = inner_get_max_nonce(&self.pool, captcha_key, difficulty_factor).await;
+        if let Err(DBError::CaptchaNotFound) = res {
+            sqlx::query!(
+                "INSERT INTO
+                    mcaptcha_track_nonce (level_id, nonce)
+                VALUES  ((
+                    SELECT
+                        level_id
+                    FROM
+                        mcaptcha_levels
+                    WHERE
+                        config_id = (SELECT config_id FROM mcaptcha_config WHERE captcha_key =?)
+                    AND
+                        difficulty_factor = ?
+                    ), ?);",
+                &captcha_key,
+                difficulty_factor as i32,
+                0,
+            )
+            .execute(&self.pool)
+            .await
+                .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+            let res =
+                inner_get_max_nonce(&self.pool, captcha_key, difficulty_factor).await?;
+            Ok(res.nonce as u32)
+        } else {
+            let res = res?;
+            Ok(res.nonce as u32)
+        }
+    }
+
+    /// Get number of analytics entries that are under a certain duration
+    async fn stats_get_num_logs_under_time(&self, duration: u32) -> DBResult<usize> {
+        struct Count {
+            count: Option<i64>,
+        }
+
+        //"SELECT COUNT(*) FROM (SELECT difficulty_factor FROM mcaptcha_pow_analytics WHERE time <= ?) as count",
+        let count = sqlx::query_as!(
+            Count,
+            "SELECT
+                COUNT(difficulty_factor) AS count
+            FROM
+                mcaptcha_pow_analytics
+            WHERE time <= ?;",
+            duration as i32,
+        )
+        .fetch_one(&self.pool)
+        .await
+        .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        Ok(count.count.unwrap_or_else(|| 0) as usize)
+    }
+
+    /// Get the entry at a location in the list of analytics entires under a certain time limited
+    /// and sorted in ascending order
+    async fn stats_get_entry_at_location_for_time_limit_asc(
+        &self,
+        duration: u32,
+        location: u32,
+    ) -> DBResult<Option<usize>> {
+        struct Difficulty {
+            difficulty_factor: Option<i32>,
+        }
+
+        match sqlx::query_as!(
+            Difficulty,
+            "SELECT
+            difficulty_factor
+        FROM
+            mcaptcha_pow_analytics
+        WHERE
+            time <= ?
+        ORDER BY difficulty_factor ASC LIMIT 1 OFFSET ?;",
+            duration as i32,
+            location as i64 - 1,
+        )
+        .fetch_one(&self.pool)
+        .await
+        {
+            Ok(res) => Ok(Some(res.difficulty_factor.unwrap() as usize)),
+            Err(sqlx::Error::RowNotFound) => Ok(None),
+            Err(e) => Err(map_row_not_found_err(e, DBError::CaptchaNotFound)),
+        }
+    }
 }
 
 #[derive(Clone)]

db/db-sqlx-maria/src/tests.rs

@@ -5,9 +5,11 @@
 
 #![cfg(test)]
 
-use sqlx::mysql::MySqlPoolOptions;
 use std::env;
 
+use sqlx::{migrate::MigrateDatabase, mysql::MySqlPoolOptions};
+use url::Url;
+
 use crate::*;
 
 use db_core::tests::*;
@@ -26,28 +28,6 @@ async fn everyting_works() {
     const HEADING: &str = "testing notifications get db mariadb";
     const MESSAGE: &str = "testing notifications get message db mariadb";
 
-    // easy traffic pattern
-    const TRAFFIC_PATTERN: TrafficPattern = TrafficPattern {
-        avg_traffic: 500,
-        peak_sustainable_traffic: 5_000,
-        broke_my_site_traffic: Some(10_000),
-    };
-
-    const LEVELS: [Level; 3] = [
-        Level {
-            difficulty_factor: 1,
-            visitor_threshold: 1,
-        },
-        Level {
-            difficulty_factor: 2,
-            visitor_threshold: 2,
-        },
-        Level {
-            difficulty_factor: 3,
-            visitor_threshold: 3,
-        },
-    ];
-
     const ADD_NOTIFICATION: AddNotification = AddNotification {
         from: NAME,
         to: NAME,
@@ -56,10 +36,20 @@ async fn everyting_works() {
     };
 
     let url = env::var("MARIA_DATABASE_URL").unwrap();
+
+    let mut parsed = Url::parse(&url).unwrap();
+    parsed.set_path("db_maria_test");
+    let url = parsed.to_string();
+
+    if sqlx::MySql::database_exists(&url).await.unwrap() {
+        sqlx::MySql::drop_database(&url).await.unwrap();
+    }
+    sqlx::MySql::create_database(&url).await.unwrap();
+
     let pool_options = MySqlPoolOptions::new().max_connections(2);
     let connection_options = ConnectionOptions::Fresh(Fresh {
         pool_options,
-        url,
+        url: url.clone(),
         disable_logging: false,
     });
     let db = connection_options.connect().await.unwrap();
@@ -78,4 +68,6 @@ async fn everyting_works() {
         description: CAPTCHA_DESCRIPTION,
     };
     database_works(&db, &p, &c, &LEVELS, &TRAFFIC_PATTERN, &ADD_NOTIFICATION).await;
+    drop(db);
+    sqlx::MySql::drop_database(&url).await.unwrap();
 }

db/db-sqlx-postgres/.sqlx/query-133ee23ab5ac7c664a86b6edfaa8da79281b6d1f5ba33c642a6ea1b0682fe0b0.json

@@ -0,0 +1,17 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO\n                    mcaptcha_track_nonce (level_id, nonce)\n                VALUES  ((\n                    SELECT\n                        level_id\n                    FROM\n                        mcaptcha_levels\n                    WHERE\n                        config_id = (SELECT config_id FROM mcaptcha_config WHERE key = ($1))\n                    AND\n                        difficulty_factor = $2\n                    AND\n                        visitor_threshold = $3\n                    ), $4);",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int4",
+        "Int4",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "133ee23ab5ac7c664a86b6edfaa8da79281b6d1f5ba33c642a6ea1b0682fe0b0"
+}

db/db-sqlx-postgres/.sqlx/query-96f1f1e45144d5add6c4ba4cd2df8eda6043bc8cd6952787f92a687fef778a6e.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT nonce FROM mcaptcha_track_nonce\n                WHERE level_id =  (\n                    SELECT\n                        level_id\n                    FROM\n                        mcaptcha_levels\n                    WHERE\n                        config_id = (SELECT config_id FROM mcaptcha_config WHERE key = ($1))\n                    AND\n                        difficulty_factor = $2\n                    );",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "nonce",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int4"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "96f1f1e45144d5add6c4ba4cd2df8eda6043bc8cd6952787f92a687fef778a6e"
+}

db/db-sqlx-postgres/.sqlx/query-c08c1dd4bfcb6cbd0359c79cc3be79526a012b006ce9deb80bceb4e1a04c835d.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(difficulty_factor) FROM mcaptcha_pow_analytics WHERE time <= $1;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4"
+      ]
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "c08c1dd4bfcb6cbd0359c79cc3be79526a012b006ce9deb80bceb4e1a04c835d"
+}

db/db-sqlx-postgres/.sqlx/query-c67aec0c3d5786fb495b6ed60fa106437d8e5034d3a40bf8face2ca7c12f2694.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT\n            difficulty_factor\n        FROM\n            mcaptcha_pow_analytics\n        WHERE\n            time <= $1\n        ORDER BY difficulty_factor ASC LIMIT 1 OFFSET $2;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "difficulty_factor",
+        "type_info": "Int4"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int4",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "c67aec0c3d5786fb495b6ed60fa106437d8e5034d3a40bf8face2ca7c12f2694"
+}

db/db-sqlx-postgres/.sqlx/query-e0088cf77c1c3a0184f35d1899a6168023fba021adf281cf1c8f9e8ccfe3a03e.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "INSERT INTO\n                    mcaptcha_track_nonce (level_id, nonce)\n                VALUES  ((\n                    SELECT\n                        level_id\n                    FROM\n                        mcaptcha_levels\n                    WHERE\n                        config_id = (SELECT config_id FROM mcaptcha_config WHERE key = ($1))\n                    AND\n                        difficulty_factor = $2\n                    ), $3);",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int4",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e0088cf77c1c3a0184f35d1899a6168023fba021adf281cf1c8f9e8ccfe3a03e"
+}

db/db-sqlx-postgres/.sqlx/query-e33ee14cf76cd09d9a157b8784a3fe25b89eaca105aa30e479d31b756cd5c88b.json

@@ -0,0 +1,16 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "UPDATE mcaptcha_track_nonce SET nonce = $3\n                WHERE level_id =  (\n                    SELECT\n                        level_id\n                    FROM\n                        mcaptcha_levels\n                    WHERE\n                        config_id = (SELECT config_id FROM mcaptcha_config WHERE key = ($1))\n                    AND\n                        difficulty_factor = $2\n                    )\n                AND nonce <= $3;",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Text",
+        "Int4",
+        "Int4"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "e33ee14cf76cd09d9a157b8784a3fe25b89eaca105aa30e479d31b756cd5c88b"
+}

db/db-sqlx-postgres/migrations/20231028235346_mcaptcha_track_nonce.sql

@@ -0,0 +1,6 @@
+-- Add migration script here
+CREATE TABLE IF NOT EXISTS mcaptcha_track_nonce (
+	nonce INTEGER NOT NULL DEFAULT 0,
+	level_id INTEGER references mcaptcha_levels(level_id)  ON DELETE CASCADE,
+	ID SERIAL PRIMARY KEY NOT NULL
+);

db/db-sqlx-postgres/src/lib.rs

@@ -445,6 +445,38 @@ impl MCDatabase for Database {
             futs.push(fut);
         }
 
+        try_join_all(futs)
+            .await
+            .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        let mut futs = Vec::with_capacity(levels.len());
+        for level in levels.iter() {
+            let difficulty_factor = level.difficulty_factor as i32;
+            let visitor_threshold = level.visitor_threshold as i32;
+            let fut = sqlx::query!(
+                "INSERT INTO
+                    mcaptcha_track_nonce (level_id, nonce)
+                VALUES  ((
+                    SELECT
+                        level_id
+                    FROM
+                        mcaptcha_levels
+                    WHERE
+                        config_id = (SELECT config_id FROM mcaptcha_config WHERE key = ($1))
+                    AND
+                        difficulty_factor = $2
+                    AND
+                        visitor_threshold = $3
+                    ), $4);",
+                &captcha_key,
+                difficulty_factor,
+                visitor_threshold,
+                0,
+            )
+            .execute(&self.pool);
+            futs.push(fut);
+        }
+
         try_join_all(futs)
             .await
             .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
@@ -1097,6 +1129,154 @@ impl MCDatabase for Database {
 
         Ok(res.drain(0..).map(|r| r.psuedo_id).collect())
     }
+
+    /// Track maximum nonce received against captcha levels
+    async fn update_max_nonce_for_level(
+        &self,
+        captcha_key: &str,
+        difficulty_factor: u32,
+        latest_nonce: u32,
+    ) -> DBResult<()> {
+        sqlx::query!(
+                "UPDATE mcaptcha_track_nonce SET nonce = $3
+                WHERE level_id =  (
+                    SELECT
+                        level_id
+                    FROM
+                        mcaptcha_levels
+                    WHERE
+                        config_id = (SELECT config_id FROM mcaptcha_config WHERE key = ($1))
+                    AND
+                        difficulty_factor = $2
+                    )
+                AND nonce <= $3;",
+                &captcha_key,
+                difficulty_factor as i32,
+                latest_nonce as i32,
+            )
+            .execute(&self.pool).await
+        .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        Ok(())
+    }
+
+    /// Get maximum nonce tracked so far for captcha levels
+    async fn get_max_nonce_for_level(
+        &self,
+        captcha_key: &str,
+        difficulty_factor: u32,
+    ) -> DBResult<u32> {
+        struct X {
+            nonce: i32,
+        }
+
+        async fn inner_get_max_nonce(
+            pool: &PgPool,
+            captcha_key: &str,
+            difficulty_factor: u32,
+        ) -> DBResult<X> {
+            sqlx::query_as!(
+                X,
+                "SELECT nonce FROM mcaptcha_track_nonce
+                WHERE level_id =  (
+                    SELECT
+                        level_id
+                    FROM
+                        mcaptcha_levels
+                    WHERE
+                        config_id = (SELECT config_id FROM mcaptcha_config WHERE key = ($1))
+                    AND
+                        difficulty_factor = $2
+                    );",
+                &captcha_key,
+                difficulty_factor as i32,
+            )
+        .fetch_one(pool)
+                .await
+                .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))
+        }
+
+        let res = inner_get_max_nonce(&self.pool, captcha_key, difficulty_factor).await;
+        if let Err(DBError::CaptchaNotFound) = res {
+            sqlx::query!(
+                "INSERT INTO
+                    mcaptcha_track_nonce (level_id, nonce)
+                VALUES  ((
+                    SELECT
+                        level_id
+                    FROM
+                        mcaptcha_levels
+                    WHERE
+                        config_id = (SELECT config_id FROM mcaptcha_config WHERE key = ($1))
+                    AND
+                        difficulty_factor = $2
+                    ), $3);",
+                &captcha_key,
+                difficulty_factor as i32,
+                0,
+            )
+            .execute(&self.pool)
+            .await
+                .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+            let res =
+                inner_get_max_nonce(&self.pool, captcha_key, difficulty_factor).await?;
+            Ok(res.nonce as u32)
+        } else {
+            let res = res?;
+            Ok(res.nonce as u32)
+        }
+    }
+
+    /// Get number of analytics entries that are under a certain duration
+    async fn stats_get_num_logs_under_time(&self, duration: u32) -> DBResult<usize> {
+        struct Count {
+            count: Option<i64>,
+        }
+
+        let count = sqlx::query_as!(
+        Count,
+        "SELECT COUNT(difficulty_factor) FROM mcaptcha_pow_analytics WHERE time <= $1;",
+        duration as i32,
+    )
+        .fetch_one(&self.pool)
+        .await
+        .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        Ok(count.count.unwrap_or_else(|| 0) as usize)
+    }
+
+    /// Get the entry at a location in the list of analytics entires under a certain time limit
+    /// and sorted in ascending order
+    async fn stats_get_entry_at_location_for_time_limit_asc(
+        &self,
+        duration: u32,
+        location: u32,
+    ) -> DBResult<Option<usize>> {
+        struct Difficulty {
+            difficulty_factor: Option<i32>,
+        }
+
+        match sqlx::query_as!(
+            Difficulty,
+            "SELECT
+            difficulty_factor
+        FROM
+            mcaptcha_pow_analytics
+        WHERE
+            time <= $1
+        ORDER BY difficulty_factor ASC LIMIT 1 OFFSET $2;",
+            duration as i32,
+            location as i64 - 1,
+        )
+        .fetch_one(&self.pool)
+        .await
+        {
+            Ok(res) => Ok(Some(res.difficulty_factor.unwrap() as usize)),
+            Err(sqlx::Error::RowNotFound) => Ok(None),
+            Err(e) => Err(map_row_not_found_err(e, DBError::CaptchaNotFound)),
+        }
+    }
 }
 
 #[derive(Clone)]

db/db-sqlx-postgres/src/tests.rs

@@ -5,9 +5,12 @@
 
 #![cfg(test)]
 
-use sqlx::postgres::PgPoolOptions;
 use std::env;
 
+use sqlx::migrate::MigrateDatabase;
+use sqlx::postgres::PgPoolOptions;
+use url::Url;
+
 use crate::*;
 
 use db_core::tests::*;
@@ -26,28 +29,6 @@ async fn everyting_works() {
     const HEADING: &str = "testing notifications get db postgres";
     const MESSAGE: &str = "testing notifications get message db postgres";
 
-    // easy traffic pattern
-    const TRAFFIC_PATTERN: TrafficPattern = TrafficPattern {
-        avg_traffic: 500,
-        peak_sustainable_traffic: 5_000,
-        broke_my_site_traffic: Some(10_000),
-    };
-
-    const LEVELS: [Level; 3] = [
-        Level {
-            difficulty_factor: 1,
-            visitor_threshold: 1,
-        },
-        Level {
-            difficulty_factor: 2,
-            visitor_threshold: 2,
-        },
-        Level {
-            difficulty_factor: 3,
-            visitor_threshold: 3,
-        },
-    ];
-
     const ADD_NOTIFICATION: AddNotification = AddNotification {
         from: NAME,
         to: NAME,
@@ -56,10 +37,20 @@ async fn everyting_works() {
     };
 
     let url = env::var("POSTGRES_DATABASE_URL").unwrap();
+
+    let mut parsed = Url::parse(&url).unwrap();
+    parsed.set_path("db_postgres_test");
+    let url = parsed.to_string();
+
+    if sqlx::Postgres::database_exists(&url).await.unwrap() {
+        sqlx::Postgres::drop_database(&url).await.unwrap();
+    }
+    sqlx::Postgres::create_database(&url).await.unwrap();
+
     let pool_options = PgPoolOptions::new().max_connections(2);
     let connection_options = ConnectionOptions::Fresh(Fresh {
         pool_options,
-        url,
+        url: url.clone(),
         disable_logging: false,
     });
     let db = connection_options.connect().await.unwrap();
@@ -78,4 +69,6 @@ async fn everyting_works() {
         description: CAPTCHA_DESCRIPTION,
     };
     database_works(&db, &p, &c, &LEVELS, &TRAFFIC_PATTERN, &ADD_NOTIFICATION).await;
+    drop(db);
+    sqlx::Postgres::drop_database(&url).await.unwrap();
 }

package.json

@@ -37,8 +37,8 @@
     "webpack-dev-server": "^4.15.1"
   },
   "dependencies": {
-    "@mcaptcha/pow-wasm": "^0.1.0-rc1",
+    "@mcaptcha/pow_sha256-polyfill": "^0.1.0-rc2",
-    "@mcaptcha/pow_sha256-polyfill": "^0.1.0-rc1",
+    "@mcaptcha/vanilla-glue": "^0.1.0-rc1",
-    "@mcaptcha/vanilla-glue": "^0.1.0-rc1"
+    "@mcaptcha/pow-wasm": "^0.1.0-rc2"
   }
 }

scripts/publish.sh

@@ -17,7 +17,7 @@ DUMBSERVE_PASSWORD=$4
 DUMBSERVE_HOST="https://$DUMBSERVE_USERNAME:$DUMBSERVE_PASSWORD@dl.mcaptcha.org"
 
 NAME=mcaptcha
-KEY=0CBABF3084E84E867A76709750BE39D10ECE01FB
+KEY=73DAC973A9ADBB9ADCB5CDC4595A08135BA9FF73
 
 TMP_DIR=$(mktemp -d)
 FILENAME="$NAME-$2-linux-amd64"

src/api/v1/meta.rs

@@ -62,6 +62,7 @@ impl Health {
 #[my_codegen::get(path = "crate::V1_API_ROUTES.meta.health")]
 async fn health(data: AppData) -> impl Responder {
     let mut resp_builder = HealthBuilder::default();
+    resp_builder.redis(None);
 
     resp_builder.db(data.db.ping().await);
 

src/api/v1/mod.rs

@@ -14,6 +14,7 @@ pub mod meta;
 pub mod notifications;
 pub mod pow;
 mod routes;
+pub mod stats;
 pub mod survey;
 
 pub use routes::ROUTES;
@@ -26,6 +27,7 @@ pub fn services(cfg: &mut ServiceConfig) {
     mcaptcha::services(cfg);
     notifications::services(cfg);
     survey::services(cfg);
+    stats::services(cfg);
 }
 
 #[derive(Deserialize)]

src/api/v1/pow/get_config.rs

@@ -5,6 +5,7 @@
 
 //use actix::prelude::*;
 use actix_web::{web, HttpResponse, Responder};
+use libmcaptcha::pow::PoWConfig;
 use libmcaptcha::{
     defense::LevelBuilder, master::messages::AddSiteBuilder, DefenseBuilder,
     MCaptchaBuilder,
@@ -21,7 +22,13 @@ pub struct GetConfigPayload {
     pub key: String,
 }
 
-// API keys are mcaptcha actor names
+#[derive(Clone, Serialize, Deserialize, Debug)]
+pub struct ApiPoWConfig {
+    pub string: String,
+    pub difficulty_factor: u32,
+    pub salt: String,
+    pub max_recorded_nonce: u32,
+}
 
 /// get PoW configuration for an mcaptcha key
 #[my_codegen::post(path = "V1_API_ROUTES.pow.get_config()")]
@@ -35,52 +42,34 @@ pub async fn get_config(
     }
     let payload = payload.into_inner();
 
-    match data.captcha.get_pow(payload.key.clone()).await {
+    let config: ServiceResult<PoWConfig> =
-        Ok(Some(config)) => {
+        match data.captcha.get_pow(payload.key.clone()).await {
-            data.stats.record_fetch(&data, &payload.key).await?;
+            Ok(Some(config)) => Ok(config),
-            Ok(HttpResponse::Ok().json(config))
+            Ok(None) => {
-        }
+                init_mcaptcha(&data, &payload.key).await?;
-        Ok(None) => {
+                let config = data
-            init_mcaptcha(&data, &payload.key).await?;
+                    .captcha
-            let config = data
+                    .get_pow(payload.key.clone())
-                .captcha
+                    .await
-                .get_pow(payload.key.clone())
+                    .expect("mcaptcha should be initialized and ready to go");
-                .await
+                Ok(config.unwrap())
-                .expect("mcaptcha should be initialized and ready to go");
+            }
-            // background it. would require data::Data to be static
+            Err(e) => Err(e.into()),
-            // to satidfy lifetime
+        };
-            data.stats.record_fetch(&data, &payload.key).await?;
+    let config = config?;
-            Ok(HttpResponse::Ok().json(config))
+    let max_nonce = data
-        }
+        .db
-        Err(e) => Err(e.into()),
+        .get_max_nonce_for_level(&payload.key, config.difficulty_factor)
-    }
+        .await?;
+    data.stats.record_fetch(&data, &payload.key).await?;
 
-    //    match res.exists {
+    let config = ApiPoWConfig {
-    //        Some(true) => {
+        string: config.string,
-    //            match data.captcha.get_pow(payload.key.clone()).await {
+        difficulty_factor: config.difficulty_factor,
-    //                Ok(Some(config)) => {
+        salt: config.salt,
-    //                    record_fetch(&payload.key, &data.db).await;
+        max_recorded_nonce: max_nonce,
-    //                    Ok(HttpResponse::Ok().json(config))
+    };
-    //                }
+    Ok(HttpResponse::Ok().json(config))
-    //                Ok(None) => {
-    //                    init_mcaptcha(&data, &payload.key).await?;
-    //                    let config = data
-    //                        .captcha
-    //                        .get_pow(payload.key.clone())
-    //                        .await
-    //                        .expect("mcaptcha should be initialized and ready to go");
-    //                    // background it. would require data::Data to be static
-    //                    // to satidfy lifetime
-    //                    record_fetch(&payload.key, &data.db).await;
-    //                    Ok(HttpResponse::Ok().json(config))
-    //                }
-    //                Err(e) => Err(e.into()),
-    //            }
-    //        }
-    //
-    //        Some(false) => Err(ServiceError::TokenNotFound),
-    //        None => Err(ServiceError::TokenNotFound),
-    //    }
 }
 /// Call this when [MCaptcha][libmcaptcha::MCaptcha] is not in master.
 ///

src/api/v1/pow/verify_pow.rs

@@ -65,6 +65,7 @@ pub async fn verify_pow(
     let payload = payload.into_inner();
     let worker_type = payload.worker_type.clone();
     let time = payload.time;
+    let nonce = payload.nonce;
     let (res, difficulty_factor) = data.captcha.verify_pow(payload.into(), ip).await?;
     data.stats.record_solve(&data, &key).await?;
     if let (Some(time), Some(worker_type)) = (time, worker_type) {
@@ -75,6 +76,9 @@ pub async fn verify_pow(
         };
         data.db.analysis_save(&key, &analytics).await?;
     }
+    data.db
+        .update_max_nonce_for_level(&key, difficulty_factor, nonce as u32)
+        .await?;
     let payload = ValidationToken { token: res };
     Ok(HttpResponse::Ok().json(payload))
 }

src/api/v1/routes.rs

@@ -11,6 +11,7 @@ use super::mcaptcha::routes::Captcha;
 use super::meta::routes::Meta;
 use super::notifications::routes::Notifications;
 use super::pow::routes::PoW;
+use super::stats::routes::Stats;
 use super::survey::routes::Survey;
 
 pub const ROUTES: Routes = Routes::new();
@@ -23,6 +24,7 @@ pub struct Routes {
     pub pow: PoW,
     pub survey: Survey,
     pub notifications: Notifications,
+    pub stats: Stats,
 }
 
 impl Routes {
@@ -35,6 +37,7 @@ impl Routes {
             pow: PoW::new(),
             notifications: Notifications::new(),
             survey: Survey::new(),
+            stats: Stats::new(),
         }
     }
 }

src/api/v1/stats.rs

@@ -0,0 +1,252 @@
+// Copyright (C) 2021  Aravinth Manivannan <realaravinth@batsense.net>
+// SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+use actix_web::{web, HttpResponse, Responder};
+use derive_builder::Builder;
+use serde::{Deserialize, Serialize};
+
+use crate::errors::*;
+use crate::AppData;
+
+#[derive(Clone, Debug, Deserialize, Builder, Serialize)]
+pub struct BuildDetails {
+    pub version: &'static str,
+    pub git_commit_hash: &'static str,
+}
+
+pub mod routes {
+    use serde::{Deserialize, Serialize};
+
+    #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+    pub struct Stats {
+        pub percentile_benches: &'static str,
+    }
+
+    impl Stats {
+        pub const fn new() -> Self {
+            Self {
+                percentile_benches: "/api/v1/stats/analytics/percentile",
+            }
+        }
+    }
+}
+
+/// Get difficulty factor with max time limit for percentile of stats
+#[my_codegen::post(path = "crate::V1_API_ROUTES.stats.percentile_benches")]
+async fn percentile_benches(
+    data: AppData,
+    payload: web::Json<PercentileReq>,
+) -> ServiceResult<impl Responder> {
+    let count = data.db.stats_get_num_logs_under_time(payload.time).await?;
+
+    if count == 0 {
+        return Ok(HttpResponse::Ok().json(PercentileResp {
+            difficulty_factor: None,
+        }));
+    }
+
+    if count < 2 {
+        return Ok(HttpResponse::Ok().json(PercentileResp {
+            difficulty_factor: None,
+        }));
+    }
+
+    let location = ((count - 1) as f64 * (payload.percentile / 100.00)) + 1.00;
+    let fraction = location - location.floor();
+
+    if fraction > 0.00 {
+        if let (Some(base), Some(ceiling)) = (
+            data.db
+                .stats_get_entry_at_location_for_time_limit_asc(
+                    payload.time,
+                    location.floor() as u32,
+                )
+                .await?,
+            data.db
+                .stats_get_entry_at_location_for_time_limit_asc(
+                    payload.time,
+                    location.floor() as u32 + 1,
+                )
+                .await?,
+        ) {
+            let res = base as u32 + ((ceiling - base) as f64 * fraction).floor() as u32;
+
+            return Ok(HttpResponse::Ok().json(PercentileResp {
+                difficulty_factor: Some(res),
+            }));
+        }
+    } else {
+        if let Some(base) = data
+            .db
+            .stats_get_entry_at_location_for_time_limit_asc(
+                payload.time,
+                location.floor() as u32,
+            )
+            .await?
+        {
+            let res = base as u32;
+
+            return Ok(HttpResponse::Ok().json(PercentileResp {
+                difficulty_factor: Some(res),
+            }));
+        }
+    };
+    Ok(HttpResponse::Ok().json(PercentileResp {
+        difficulty_factor: None,
+    }))
+}
+
+#[derive(Clone, Debug, Deserialize, Builder, Serialize)]
+/// Health check return datatype
+pub struct PercentileReq {
+    time: u32,
+    percentile: f64,
+}
+
+#[derive(Clone, Debug, Deserialize, Builder, Serialize)]
+/// Health check return datatype
+pub struct PercentileResp {
+    difficulty_factor: Option<u32>,
+}
+
+pub fn services(cfg: &mut web::ServiceConfig) {
+    cfg.service(percentile_benches);
+}
+
+#[cfg(test)]
+mod tests {
+    use actix_web::{http::StatusCode, test, App};
+
+    use super::*;
+    use crate::api::v1::services;
+    use crate::*;
+
+    #[actix_rt::test]
+    async fn stats_bench_work_pg() {
+        let data = crate::tests::pg::get_data().await;
+        stats_bench_work(data).await;
+    }
+
+    #[actix_rt::test]
+    async fn stats_bench_work_maria() {
+        let data = crate::tests::maria::get_data().await;
+        stats_bench_work(data).await;
+    }
+
+    async fn stats_bench_work(data: ArcData) {
+        use crate::tests::*;
+
+        const NAME: &str = "benchstatsuesr";
+        const EMAIL: &str = "benchstatsuesr@testadminuser.com";
+        const PASSWORD: &str = "longpassword2";
+
+        const DEVICE_USER_PROVIDED: &str = "foo";
+        const DEVICE_SOFTWARE_RECOGNISED: &str = "Foobar.v2";
+        const THREADS: i32 = 4;
+
+        let data = &data;
+        {
+            delete_user(&data, NAME).await;
+        }
+
+        register_and_signin(data, NAME, EMAIL, PASSWORD).await;
+        // create captcha
+        let (_, _signin_resp, key) = add_levels_util(data, NAME, PASSWORD).await;
+        let app = get_app!(data).await;
+
+        let page = 1;
+        let tmp_id = uuid::Uuid::new_v4();
+        let download_rotue = V1_API_ROUTES
+            .survey
+            .get_download_route(&tmp_id.to_string(), page);
+
+        let download_req = test::call_service(
+            &app,
+            test::TestRequest::get().uri(&download_rotue).to_request(),
+        )
+        .await;
+        assert_eq!(download_req.status(), StatusCode::NOT_FOUND);
+
+        data.db
+            .analytics_create_psuedo_id_if_not_exists(&key.key)
+            .await
+            .unwrap();
+
+        let psuedo_id = data
+            .db
+            .analytics_get_psuedo_id_from_capmaign_id(&key.key)
+            .await
+            .unwrap();
+
+        for i in 1..6 {
+            println!("[{i}] Saving analytics");
+            let analytics = db_core::CreatePerformanceAnalytics {
+                time: i,
+                difficulty_factor: i,
+                worker_type: "wasm".into(),
+            };
+            data.db.analysis_save(&key.key, &analytics).await.unwrap();
+        }
+
+        let msg = PercentileReq {
+            time: 1,
+            percentile: 99.00,
+        };
+        let resp = test::call_service(
+            &app,
+            post_request!(&msg, V1_API_ROUTES.stats.percentile_benches).to_request(),
+        )
+        .await;
+        assert_eq!(resp.status(), StatusCode::OK);
+        let resp: PercentileResp = test::read_body_json(resp).await;
+
+        assert!(resp.difficulty_factor.is_none());
+
+        let msg = PercentileReq {
+            time: 1,
+            percentile: 100.00,
+        };
+
+        let resp = test::call_service(
+            &app,
+            post_request!(&msg, V1_API_ROUTES.stats.percentile_benches).to_request(),
+        )
+        .await;
+        assert_eq!(resp.status(), StatusCode::OK);
+        let resp: PercentileResp = test::read_body_json(resp).await;
+
+        assert!(resp.difficulty_factor.is_none());
+
+        let msg = PercentileReq {
+            time: 2,
+            percentile: 100.00,
+        };
+
+        let resp = test::call_service(
+            &app,
+            post_request!(&msg, V1_API_ROUTES.stats.percentile_benches).to_request(),
+        )
+        .await;
+        assert_eq!(resp.status(), StatusCode::OK);
+        let resp: PercentileResp = test::read_body_json(resp).await;
+
+        assert_eq!(resp.difficulty_factor.unwrap(), 2);
+
+        let msg = PercentileReq {
+            time: 5,
+            percentile: 90.00,
+        };
+
+        let resp = test::call_service(
+            &app,
+            post_request!(&msg, V1_API_ROUTES.stats.percentile_benches).to_request(),
+        )
+        .await;
+        assert_eq!(resp.status(), StatusCode::OK);
+        let resp: PercentileResp = test::read_body_json(resp).await;
+
+        assert_eq!(resp.difficulty_factor.unwrap(), 4);
+        delete_user(&data, NAME).await;
+    }
+}

src/tests/mod.rs

@@ -29,6 +29,9 @@ pub fn get_settings() -> Settings {
 pub mod pg {
     use std::env;
 
+    use sqlx::migrate::MigrateDatabase;
+
+    use crate::api::v1::mcaptcha::get_random;
     use crate::data::Data;
     use crate::settings::*;
     use crate::survey::SecretsStore;
@@ -38,6 +41,16 @@ pub mod pg {
 
     pub async fn get_data() -> ArcData {
         let url = env::var("POSTGRES_DATABASE_URL").unwrap();
+
+        let mut parsed = url::Url::parse(&url).unwrap();
+        parsed.set_path(&get_random(16));
+        let url = parsed.to_string();
+
+        if sqlx::Postgres::database_exists(&url).await.unwrap() {
+            sqlx::Postgres::drop_database(&url).await.unwrap();
+        }
+        sqlx::Postgres::create_database(&url).await.unwrap();
+
         let mut settings = get_settings();
         settings.captcha.runners = Some(1);
         settings.database.url = url.clone();
@@ -50,6 +63,9 @@ pub mod pg {
 pub mod maria {
     use std::env;
 
+    use sqlx::migrate::MigrateDatabase;
+
+    use crate::api::v1::mcaptcha::get_random;
     use crate::data::Data;
     use crate::settings::*;
     use crate::survey::SecretsStore;
@@ -59,6 +75,16 @@ pub mod maria {
 
     pub async fn get_data() -> ArcData {
         let url = env::var("MARIA_DATABASE_URL").unwrap();
+
+        let mut parsed = url::Url::parse(&url).unwrap();
+        parsed.set_path(&get_random(16));
+        let url = parsed.to_string();
+
+        if sqlx::MySql::database_exists(&url).await.unwrap() {
+            sqlx::MySql::drop_database(&url).await.unwrap();
+        }
+        sqlx::MySql::create_database(&url).await.unwrap();
+
         let mut settings = get_settings();
         settings.captcha.runners = Some(1);
         settings.database.url = url.clone();

templates/panel/sitekey/add/advance/form.html

@@ -52,7 +52,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later
   <. } .>
 
     <label class="sitekey-form__label" for="publish_benchmarks">
-		Anonymously publish CAPTCHA performance statistics to help other webmasters
+        Anonymously publish CAPTCHA performance statistics to help other webmasters. <a href="https://mcaptcha.org/blog/introducing-mcaptcha-net">Please see here for more info</a>.
 	  <input
 		class="sitekey-form__input"
 		type="checkbox"

templates/panel/sitekey/add/novice/form.html

@@ -76,7 +76,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later
 
 
     <label class="sitekey-form__label" for="publish_benchmarks">
-		Anonymously publish CAPTCHA performance statistics to help other webmasters
+        Anonymously publish CAPTCHA performance statistics to help other webmasters. <a href="https://mcaptcha.org/blog/introducing-mcaptcha-net">Please see here for more info</a>.
 	  <input
 		class="sitekey-form__input"
 		type="checkbox"

templates/widget/index.html

@@ -6,51 +6,54 @@ SPDX-License-Identifier: MIT OR Apache-2.0
 
 <. include!("../components/headers/widget-headers.html"); .>
   <body>
-    <form class="widget__contaienr">
+    <main class="widget__container">
-		<noscript>
+      <form class="widget__inner-container">
-		  <div class="widget__noscript-container">
+          <noscript>
-			<span class="widget__noscript-warning">
+            <div class="widget__noscript-container">
-			  Please enable JavaScript to receive mCaptcha challenge
+              <span class="widget__noscript-warning">
-			</span>
+                Please enable JavaScript to receive mCaptcha challenge
-			<a class="widget__source-code" href="https://github.com/mCaptcha">
+              </span>
-			  Read our source code
+              <a class="widget__source-code" href="https://github.com/mCaptcha">
-			</a>
+                Read our source code
-		 </div>
+              </a>
-      </noscript>
+           </div>
-      <label class="widget__verification-container" for="widget__verification-checkbox">
+        </noscript>
-        <input 
+        <label class="widget__verification-container" for="widget__verification-checkbox">
-			id="widget__verification-checkbox"
+          <input 
-			class="widget__verification-checkbox"
+              id="widget__verification-checkbox"
-		type="checkbox" />
+              class="widget__verification-checkbox"
-		<span id="widget__verification-text--before">I'm not a robot</span>
+          type="checkbox" />
-		<span id="widget__verification-text--during">Processing...</span>
+          <span id="widget__verification-text--before">I'm not a robot</span>
-		<span id="widget__verification-text--after">Verified!</span>
+          <span id="widget__verification-text--during">Processing...</span>
-		<span id="widget__verification-text--error">Something went wrong</span>
+          <span id="widget__verification-text--after">Verified!</span>
-      </label>
+          <span id="widget__verification-text--error">Something went wrong</span>
-      <div class="widget__mcaptcha-details">
+        </label>
-        <a href="<.= crate::PKG_HOMEPAGE .>" 
+        <div class="widget__mcaptcha-details">
-			class="widget__mcaptcha-logo-container"
+          <a href="<.= crate::PKG_HOMEPAGE .>" 
-			target="_blank"
+              class="widget__mcaptcha-logo-container"
-		 >
+              target="_blank"
-          <img
+           >
-            class="widget__mcaptcha-logo"
+            <img
-			src="<.= crate::FILES.get("./static/cache/img/icon-trans.png").unwrap().>"
+              class="widget__mcaptcha-logo"
-            alt="mCaptcha logo"
+              src="<.= crate::FILES.get("./static/cache/img/icon-trans.png").unwrap().>"
-          />
+              alt="mCaptcha logo"
-          <p class="widget__mcaptcha-brand-name">mCaptcha</p>
+            />
-        </a>
+            <p class="widget__mcaptcha-brand-name">mCaptcha</p>
-        <div class="widget__mcaptcha-info-container">
-		  <a class="widget__mcaptcha-info-link"
-			target="_blank"
-			href="<.= crate::PKG_HOMEPAGE .><.= crate::PAGES.privacy .>">
-            Privacy
-          </a>
-		  <a class="widget__mcaptcha-info-link"
-		  	target="_blank"
-			href="<.= crate::PKG_HOMEPAGE .><.= crate::PAGES.security .>">
-            Terms
           </a>
+          <div class="widget__mcaptcha-info-container">
+            <a class="widget__mcaptcha-info-link"
+              target="_blank"
+              href="<.= crate::PKG_HOMEPAGE .><.= crate::PAGES.privacy .>">
+              Privacy
+            </a>
+            <a class="widget__mcaptcha-info-link"
+              target="_blank"
+              href="<.= crate::PKG_HOMEPAGE .><.= crate::PAGES.security .>">
+              Terms
+            </a>
+          </div>
         </div>
-      </div>
+      </form>
-    </form>
+      <div class="progress__bar"><div class="progress__fill"></div></div>
+    </main>
 <.include!("./footer.html"); .>

templates/widget/index.ts

@@ -3,7 +3,7 @@
 //
 // SPDX-License-Identifier: MIT OR Apache-2.0
 
-import { Work, ServiceWorkerWork } from "./types";
+import { Work, ServiceWorkerMessage } from "./types";
 import fetchPoWConfig from "./fetchPoWConfig";
 import sendWork from "./sendWork";
 import sendToParent from "./sendToParent";
@@ -24,6 +24,9 @@ export const registerVerificationEventHandler = (): void => {
 };
 
 export const solveCaptchaRunner = async (e: Event): Promise<void> => {
+  const PROGRESS_FILL = <HTMLElement>document.querySelector(".progress__fill");
+  let width = 0;
+
   if (LOCK) {
     e.preventDefault();
     return;
@@ -32,6 +35,8 @@ export const solveCaptchaRunner = async (e: Event): Promise<void> => {
   try {
     LOCK = true;
     if (CONST.btn().checked == false) {
+      width = 0;
+      PROGRESS_FILL.style.width = `${width}%`;
       CONST.messageText().before();
       LOCK = false;
       return;
@@ -43,32 +48,49 @@ export const solveCaptchaRunner = async (e: Event): Promise<void> => {
     CONST.messageText().during();
     // 1. get config
     const config = await fetchPoWConfig();
+    const max_recorded_nonce = config.max_recorded_nonce;
     // 2. prove work
     worker.postMessage(config);
 
     worker.onmessage = async (event: MessageEvent) => {
-      const resp: ServiceWorkerWork = event.data;
+      const resp: ServiceWorkerMessage = event.data;
-      console.log(
-        `Proof generated. Difficuly: ${config.difficulty_factor} Duration: ${resp.work.time}`
-      );
 
-      const proof: Work = {
+      if (resp.type === "work") {
-        key: CONST.sitekey(),
+        width = 80;
-        string: config.string,
+        PROGRESS_FILL.style.width = `${width}%`;
-        nonce: resp.work.nonce,
+        console.log(
-        result: resp.work.result,
+          `Proof generated. Difficuly: ${config.difficulty_factor} Duration: ${resp.value.work.time}`
-        time: Math.trunc(resp.work.time),
+        );
-        worker_type: resp.work.worker_type,
-      };
 
-      // 3. submit work
+        const proof: Work = {
-      const token = await sendWork(proof);
+          key: CONST.sitekey(),
-      // 4. send token
+          string: config.string,
-      sendToParent(token);
+          nonce: resp.value.work.nonce,
-      // 5. mark checkbox checked
+          result: resp.value.work.result,
-      CONST.btn().checked = true;
+          time: Math.trunc(resp.value.work.time),
-      CONST.messageText().after();
+          worker_type: resp.value.work.worker_type,
-      LOCK = false;
+        };
+
+        width = 90;
+        PROGRESS_FILL.style.width = `${width}%`;
+        // 3. submit work
+        const token = await sendWork(proof);
+        // 4. send token
+        sendToParent(token);
+        // 5. mark checkbox checked
+        CONST.btn().checked = true;
+        width = 100;
+        PROGRESS_FILL.style.width = `${width}%`;
+        CONST.messageText().after();
+        LOCK = false;
+      }
+      if (resp.type === "progress") {
+        if (width < 80) {
+          width = (resp.nonce / max_recorded_nonce) * 100;
+          PROGRESS_FILL.style.width = `${width}%`;
+        }
+        console.log(`received nonce ${resp.nonce}`);
+      }
     };
   } catch (e) {
     CONST.messageText().error();

templates/widget/main.scss

@@ -7,106 +7,138 @@
 
 @import "../reset";
 
-.widget__contaienr {
+body {
-  align-items: center;
+	display: flex;
-  box-sizing: border-box;
+	flex-direction: column;
-  display: flex;
+	width: 100%;
-  height: 100%;
+}
+
+.widget__container {
+	align-items: center;
+	box-sizing: border-box;
+	display: flex;
+	flex-direction: column;
+	height: 100%;
+	margin: auto 0;
+}
+
+.widget__inner-container {
+	align-items: center;
+	box-sizing: border-box;
+	display: flex;
+	height: 100%;
+	width: 100%;
 }
 
 .widget__noscript-container {
-  display: flex;
+	display: flex;
-  font-size: 0.7rem;
+	font-size: 0.7rem;
-  line-height: 20px;
+	line-height: 20px;
-  flex-direction: column;
+	flex-direction: column;
-  padding: 5px;
+	padding: 5px;
-  box-sizing: border-box;
+	box-sizing: border-box;
-  height: 100%;
+	height: 100%;
-  margin: auto;
+	margin: auto;
 }
 
 .widget__noscript-warning {
-  display: block;
+	display: block;
-  margin: auto;
+	margin: auto;
-  flex: 2;
+	flex: 2;
-  width: 100%;
+	width: 100%;
-  margin: auto;
+	margin: auto;
 }
 
 .widget__source-code {
-  display: block;
+	display: block;
-  flex: 1;
+	flex: 1;
 }
 
 .widget__verification-container {
-  align-items: center;
+	align-items: center;
-  display: none;
+	display: none;
-  line-height: 30px;
+	line-height: 30px;
-  font-size: 1rem;
+	font-size: 1rem;
 }
 
 .widget__verification-checkbox {
-  width: 30px;
+	width: 30px;
-  height: 30px;
+	height: 30px;
-  margin: 0 10px;
+	margin: 0 10px;
 }
 
 #widget__verification-text--during {
-  display: none;
+	display: none;
 }
 
 #widget__verification-text--after {
-  display: none;
+	display: none;
-  color: green;
+	color: green;
 }
 
 #widget__verification-text--error {
-  display: none;
+	display: none;
-  color: red;
+	color: red;
 }
 
 .widget__verification-checkbox:checked ~ #widget__verification-text--before {
-  display: none;
+	display: none;
 }
 
 .widget__verification-checkbox:checked ~ #widget__verification-text--during {
-  display: none;
+	display: none;
 }
 
 .widget__verification-checkbox:checked ~ #widget__verification-text--error {
-  display: none;
+	display: none;
 }
 
 .widget__verification-checkbox:checked ~ #widget__verification-text--after {
-  display: block;
+	display: block;
 }
 
 .widget__mcaptcha-details {
-  display: flex;
+	display: flex;
-  flex-direction: column;
+	flex-direction: column;
-  margin-left: auto;
+	margin-left: auto;
-  margin-right: 10px;
+	margin-right: 10px;
 }
 
 .widget__mcaptcha-brand-name {
-  font-size: 0.7rem;
+	font-size: 0.7rem;
-  font-weight: 600;
+	font-weight: 600;
-  margin: auto;
+	margin: auto;
-  text-align: center;
+	text-align: center;
 }
 
 .widget__mcaptcha-logo {
-  display: block;
+	display: block;
-  width: 35px;
+	width: 35px;
-  margin: auto;
+	margin: auto;
 }
 
 .widget__mcaptcha-info-container {
-  display: flex;
+	display: flex;
-  margin: auto;
+	margin: auto;
 }
 
 .widget__mcaptcha-info-link {
-  font-size: 0.5rem;
+	font-size: 0.5rem;
-  margin: 2px;
+	margin: 2px;
+}
+
+/* progress bar courtesy of https://codepen.io/Bizzy-Coding/pen/poOymVJ?editors=1111 */
+.progress__bar {
+	position: relative;
+	height: 5px;
+	width: 100%;
+	background: #fff;
+	border-radius: 15px;
+}
+
+.progress__fill {
+	background: #65a2e0;
+	border-radius: 15px;
+	height: 100%;
+	width: 0%;
 }

templates/widget/prove.ts

@@ -3,7 +3,7 @@
 //
 // SPDX-License-Identifier: MIT OR Apache-2.0
 
-import { gen_pow } from "@mcaptcha/pow-wasm";
+import { stepped_gen_pow } from "@mcaptcha/pow-wasm";
 import * as p from "@mcaptcha/pow_sha256-polyfill";
 import { WasmWork, PoWConfig, SubmitWork } from "./types";
 
@@ -12,19 +12,25 @@ import { WasmWork, PoWConfig, SubmitWork } from "./types";
  * @param {PoWConfig} config - the proof-of-work configuration using which
  * work needs to be computed
  * */
-const prove = async (config: PoWConfig): Promise<SubmitWork> => {
+const prove = async (
+  config: PoWConfig,
+  progress: (nonce: number) => void
+): Promise<SubmitWork> => {
   const WASM = "wasm";
   const JS = "js";
+  const STEPS = 5000;
   if (WasmSupported) {
     let proof: WasmWork = null;
     let res: SubmitWork = null;
     let time: number = null;
 
     const t0 = performance.now();
-    const proofString = gen_pow(
+    const proofString = stepped_gen_pow(
       config.salt,
       config.string,
-      config.difficulty_factor
+      config.difficulty_factor,
+      STEPS,
+      progress
     );
     const t1 = performance.now();
     time = t1 - t0;
@@ -47,10 +53,12 @@ const prove = async (config: PoWConfig): Promise<SubmitWork> => {
 
     const t0 = performance.now();
 
-    proof = await p.generate_work(
+    proof = await p.stepped_generate_work(
       config.salt,
       config.string,
-      config.difficulty_factor
+      config.difficulty_factor,
+      STEPS,
+      progress
     );
     const t1 = performance.now();
     time = t1 - t0;

templates/widget/service-worker.ts

@@ -6,17 +6,31 @@
 import log from "../logger";
 
 import prove from "./prove";
-import { PoWConfig, ServiceWorkerWork } from "./types";
+import { PoWConfig, ServiceWorkerMessage, ServiceWorkerWork } from "./types";
 
 log.log("worker registered");
 onmessage = async (e) => {
   console.debug("message received at worker");
   const config: PoWConfig = e.data;
 
-  const work = await prove(config);
+  const progressCallback = (nonce: number) => {
-  const res: ServiceWorkerWork = {
+    const res: ServiceWorkerMessage = {
+      type: "progress",
+      nonce: nonce,
+    };
+
+    postMessage(res);
+  };
+
+  const work = await prove(config, progressCallback);
+  const w: ServiceWorkerWork = {
     work,
   };
 
+  const res: ServiceWorkerMessage = {
+    type: "work",
+    value: w,
+  };
+
   postMessage(res);
 };

templates/widget/types.ts

@@ -32,8 +32,13 @@ export type PoWConfig = {
   string: string;
   difficulty_factor: number;
   salt: string;
+  max_recorded_nonce: number;
 };
 
 export type Token = {
   token: string;
 };
+
+export type ServiceWorkerMessage =
+  | { type: "work"; value: ServiceWorkerWork }
+  | { type: "progress"; nonce: number };

yarn.lock

@@ -631,15 +631,15 @@
   resolved "https://registry.yarnpkg.com/@mcaptcha/core-glue/-/core-glue-0.1.0-rc1.tgz#76d665a3fc537062061e12e274f969ac3e053685"
   integrity sha512-P4SgUioJDR38QpnP9sPY72NyaYex8MXD6RbzrfKra+ngamT26XjqVZEHBiZU2RT7u0SsWhuko4N1ntNOghsgpg==
 
-"@mcaptcha/pow-wasm@^0.1.0-rc1":
+"@mcaptcha/pow-wasm@^0.1.0-rc2":
-  version "0.1.0-rc1"
+  version "0.1.0-rc2"
-  resolved "https://registry.yarnpkg.com/@mcaptcha/pow-wasm/-/pow-wasm-0.1.0-rc1.tgz#eef8409e0c74e9c7261587bdebd80a8c4af92f9e"
+  resolved "https://registry.yarnpkg.com/@mcaptcha/pow-wasm/-/pow-wasm-0.1.0-rc2.tgz#c7aaa678325600a178b11a702e2aeb9f8143e605"
-  integrity sha512-7+PGKoe1StFRsa9TEqztzK4/obbdY4OfavFX+geTk8b3K26D+eHPyimJ9BPlpI1VZl8ujR3CnbfbnQSRaqS7ZQ==
+  integrity sha512-2G0nQ2GQWECRcE5kzfULDsQ032s6/PDzE1rncMdQAR1Mu2YQfFZHgnX4zLJmQnjKIhy9meIjXvatVSyIllrbtg==
 
-"@mcaptcha/pow_sha256-polyfill@^0.1.0-rc1":
+"@mcaptcha/pow_sha256-polyfill@^0.1.0-rc2":
-  version "0.1.0-rc1"
+  version "0.1.0-rc2"
-  resolved "https://registry.yarnpkg.com/@mcaptcha/pow_sha256-polyfill/-/pow_sha256-polyfill-0.1.0-rc1.tgz#dfeee88f5f6fd99aeae65dbcff6fbb09fe8a1696"
+  resolved "https://registry.yarnpkg.com/@mcaptcha/pow_sha256-polyfill/-/pow_sha256-polyfill-0.1.0-rc2.tgz#253320e7a6666e395ef9dfb123d1102066d72b87"
-  integrity sha512-OFA4W3/vh8ORUnifbm8c/8eP22CbiXr4Un6/l4fMyqLj1aoQLMGAiuqab0trGqBnY0DU2bwTMyxflx26/cWgIw==
+  integrity sha512-ERIbxIo+ZnQKtti/T4FLmcY0neuc5R05L97qYc62Hm++i+3dx/W6A8oC4V9U0XKCPYnHZFoZozAZlbsGXjrsVQ==
 
 "@mcaptcha/vanilla-glue@^0.1.0-rc1":
   version "0.1.0-rc1"