feat: schedule mCaptcha/survey registration and uploads

hotfix: read soon-to-be deprecated env vars to avoid breakages like #114

.github/workflows/coverage.yml

@@ -1,119 +1,119 @@
-name: Coverage
+#name: Coverage
-
+#
-on:
+#on:
-  pull_request:
+#  pull_request:
-    types: [opened, synchronize, reopened]
+#    types: [opened, synchronize, reopened]
-  push:
+#  push:
-    branches:
+#    branches:
-      - master
+#      - master
-      - db-abstract
+#      - db-abstract
-
+#
-jobs:
+#jobs:
-  build_and_test:
+#  build_and_test:
-    strategy:
+#    strategy:
-      fail-fast: false
+#      fail-fast: false
-      matrix:
+#      matrix:
-        version:
+#        version:
-          - stable
+#          - stable
-            #- 1.51.0
+#            #- 1.51.0
-
+#
-    name: ${{ matrix.version }} - x86_64-unknown-linux-gnu
+#    name: ${{ matrix.version }} - x86_64-unknown-linux-gnu
-    runs-on: ubuntu-latest
+#    runs-on: ubuntu-latest
-
+#
-    services:
+#    services:
-      postgres:
+#      postgres:
-        image: postgres
+#        image: postgres
-        env:
+#        env:
-          POSTGRES_PASSWORD: password
+#          POSTGRES_PASSWORD: password
-          POSTGRES_USER: postgres
+#          POSTGRES_USER: postgres
-          POSTGRES_DB: postgres
+#          POSTGRES_DB: postgres
-        options: >-
+#        options: >-
-          --health-cmd pg_isready
+#          --health-cmd pg_isready
-          --health-interval 10s
+#          --health-interval 10s
-          --health-timeout 5s
+#          --health-timeout 5s
-          --health-retries 5
+#          --health-retries 5
-        ports:
+#        ports:
-          - 5432:5432
+#          - 5432:5432
-
+#
-      mcaptcha-redis:
+#      mcaptcha-redis:
-        image: mcaptcha/cache
+#        image: mcaptcha/cache
-        ports:
+#        ports:
-          - 6379:6379
+#          - 6379:6379
-
+#
-      mcaptcha-smtp:
+#      mcaptcha-smtp:
-        image: maildev/maildev
+#        image: maildev/maildev
-        env:
+#        env:
-          MAILDEV_WEB_PORT: "1080"
+#          MAILDEV_WEB_PORT: "1080"
-          MAILDEV_INCOMING_USER: "admin"
+#          MAILDEV_INCOMING_USER: "admin"
-          MAILDEV_INCOMING_PASS: "password"
+#          MAILDEV_INCOMING_PASS: "password"
-        ports:
+#        ports:
-          - 1080:1080
+#          - 1080:1080
-          - 10025:1025
+#          - 10025:1025
-
+#
-
+#
-      maria:
+#      maria:
-        image: mariadb:10
+#        image: mariadb:10
-        env: 
+#        env: 
-          MARIADB_USER: "maria"
+#          MARIADB_USER: "maria"
-          MARIADB_PASSWORD: "password"
+#          MARIADB_PASSWORD: "password"
-          MARIADB_ROOT_PASSWORD: "password"
+#          MARIADB_ROOT_PASSWORD: "password"
-          MARIADB_DATABASE: "maria"
+#          MARIADB_DATABASE: "maria"
-        options: >-
+#        options: >-
-          --health-cmd="mysqladmin ping"
+#          --health-cmd="mysqladmin ping"
-          --health-interval=10s
+#          --health-interval=10s
-          --health-timeout=5s
+#          --health-timeout=5s
-          --health-retries=10
+#          --health-retries=10
-        ports:
+#        ports:
-          - 3306:3306
+#          - 3306:3306
-
+#
-
+#
-    steps:
+#    steps:
-      - uses: actions/checkout@v4
+#      - uses: actions/checkout@v4
-
+#
-      - name: load env
+#      - name: load env
-        run: |
+#        run: |
-          source .env_sample \
+#          source .env_sample \
-            && echo "POSTGRES_DATABASE_URL=$POSTGRES_DATABASE_URL" >> $GITHUB_ENV \
+#            && echo "POSTGRES_DATABASE_URL=$POSTGRES_DATABASE_URL" >> $GITHUB_ENV \
-            && echo "MARIA_DATABASE_URL=$MARIA_DATABASE_URL" >> $GITHUB_ENV
+#            && echo "MARIA_DATABASE_URL=$MARIA_DATABASE_URL" >> $GITHUB_ENV
-
+#
-
+#
-      - uses: actions/setup-node@v2
+#      - uses: actions/setup-node@v2
-        with:
+#        with:
-          node-version: "18.0.0"
+#          node-version: "18.0.0"
-
+#
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
+#      - uses: actions-rust-lang/setup-rust-toolchain@v1
-
+#
-      - name: Build frontend
+#      - name: Build frontend
-        run: make frontend
+#        run: make frontend
-
+#
-      - name: Run the frontend tests
+#      - name: Run the frontend tests
-        run: make test.frontend
+#        run: make test.frontend
-
+#
-      - name: Run migrations
+#      - name: Run migrations
-        run: make migrate
+#        run: make migrate
-        env:
+#        env:
-          POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}"
+#          POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}"
-          MARIA_DATABASE_URL: "${{ env.MARIA_DATABASE_URL }}"
+#          MARIA_DATABASE_URL: "${{ env.MARIA_DATABASE_URL }}"
-
+#
-      - name: build frontend
+#      - name: build frontend
-        run: make frontend
+#        run: make frontend
-
+#
-      - name: Generate coverage file
+#      - name: Generate coverage file
-        if: github.event_name == 'pull_request'
+#        if: github.event_name == 'pull_request'
-        #if: (github.ref == 'refs/heads/master' || github.event_name == 'pull_request')
+#        #if: (github.ref == 'refs/heads/master' || github.event_name == 'pull_request')
-        uses: actions-rs/tarpaulin@v0.1
+#        uses: actions-rs/tarpaulin@v0.1
-        with:
+#        with:
-          args: "-t 1200"
+#          args: "-t 1200"
-        env:
+#        env:
-          POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}"
+#          POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}"
-          MARIA_DATABASE_URL: "${{ env.MARIA_DATABASE_URL }}"
+#          MARIA_DATABASE_URL: "${{ env.MARIA_DATABASE_URL }}"
-          # GIT_HASH is dummy value. I guess build.rs is skipped in tarpaulin
+#          # GIT_HASH is dummy value. I guess build.rs is skipped in tarpaulin
-          # execution so this value is required for preventing meta tests from
+#          # execution so this value is required for preventing meta tests from
-          # panicking
+#          # panicking
-          GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61
+#          GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61
-          CACHE_BUSTER_FILE_MAP: '{"map":{"./static/bundle/main.js":"./prod/bundle/main.1417115E59909BE0A01040A45A398ADB09D928DF89CCF038FA44B14850442096.js"},"base_dir":"./prod"}'
+#          CACHE_BUSTER_FILE_MAP: '{"map":{"./static/bundle/main.js":"./prod/bundle/main.1417115E59909BE0A01040A45A398ADB09D928DF89CCF038FA44B14850442096.js"},"base_dir":"./prod"}'
-          COMPILED_DATE: "2021-07-21"
+#          COMPILED_DATE: "2021-07-21"
-
+#
-      - name: Upload to Codecov
+#      - name: Upload to Codecov
-        if: github.event_name == 'pull_request'
+#        if: github.event_name == 'pull_request'
-        uses: codecov/codecov-action@v2
+#        uses: codecov/codecov-action@v2

.github/workflows/linux.yml

@@ -119,7 +119,7 @@ jobs:
         run: make test.integration
 
       - name: Login to DockerHub
-        if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'mCaptcha/mCaptcha'
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push' && github.repository == 'mCaptcha/mCaptcha'
         uses: docker/login-action@v1
         with:
           username: mcaptcha

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Changed
 
+-   2023-10-18: Environment variable names have changed, please see
+    [CONFIGURATION.md](docs/CONFIGURATION.md) for the names of environment
+    variables.
 -   ([`7d0e4c6`](https://github.com/mCaptcha/mCaptcha/commit/7d0e4c6be4b0769921cda7681858ebe16ec9a07b)) Add `secret` parameter to token verification request payload(`/api/v1/pow/siteverify`) to mitigate a security issue that @gusted found:
     > ...A malicious user could grab the sitekey
     > and use that sitekey with mcaptcha to use it for their own server.

Cargo.lock

@@ -439,6 +439,19 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
 
+[[package]]
+name = "async-compression"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2"
+dependencies = [
+ "flate2",
+ "futures-core",
+ "memchr",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.53"
@@ -1532,6 +1545,17 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http-body"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
+dependencies = [
+ "bytes",
+ "http",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "httparse"
 version = "1.8.0"
@@ -1559,6 +1583,43 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 
+[[package]]
+name = "hyper"
+version = "0.14.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -1651,6 +1712,12 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "ipnet"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
+
 [[package]]
 name = "is-terminal"
 version = "0.4.9"
@@ -1938,6 +2005,7 @@ dependencies = [
  "openssl",
  "pretty_env_logger 0.4.0",
  "rand",
+ "reqwest",
  "rust-embed",
  "sailfish",
  "serde",
@@ -2639,6 +2707,46 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
 
+[[package]]
+name = "reqwest"
+version = "0.11.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b"
+dependencies = [
+ "async-compression",
+ "base64 0.21.2",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "system-configuration",
+ "tokio",
+ "tokio-native-tls",
+ "tokio-util",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -3369,6 +3477,27 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "system-configuration"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.6.0"
@@ -3572,6 +3701,12 @@ dependencies = [
  "winnow",
 ]
 
+[[package]]
+name = "tower-service"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+
 [[package]]
 name = "tracing"
 version = "0.1.37"
@@ -3605,6 +3740,12 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "try-lock"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
+
 [[package]]
 name = "typenum"
 version = "1.16.0"
@@ -3779,6 +3920,15 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -3810,6 +3960,18 @@ dependencies = [
  "wasm-bindgen-shared",
 ]
 
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "wasm-bindgen-macro"
 version = "0.2.84"
@@ -4027,6 +4189,16 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "winreg"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
+dependencies = [
+ "cfg-if",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "yaml-rust"
 version = "0.4.5"

Cargo.toml

@@ -78,6 +78,7 @@ lettre = { version = "0.10.0-rc.3", features = [
 
 openssl = { version = "0.10.48", features = ["vendored"] }
 uuid = { version = "1.4.0", features = ["v4", "serde"] }
+reqwest = { version = "0.11.18", features = ["json", "gzip"] }
 
 
 [dependencies.db-core]

config/default.toml

@@ -66,3 +66,8 @@ url = "127.0.0.1"
 port = 10025
 username = "admin"
 password = "password"
+
+#[survey]
+#nodes = ["http://localhost:7001"]
+#rate_limit = 10 # upload every hour
+#instance_root_url = "http://localhost:7000"

db/db-core/src/lib.rs

@@ -289,6 +289,9 @@ pub trait MCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
             Err(e) => Err(e),
         }
     }
+
+    /// Get all psuedo IDs
+    async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult<Vec<String>>;
 }
 
 #[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)]

db/db-core/src/tests.rs

@@ -258,6 +258,12 @@ pub async fn database_works<'a, T: MCDatabase>(
         .analytics_get_psuedo_id_from_capmaign_id(c.key)
         .await
         .unwrap();
+    assert_eq!(
+        vec![psuedo_id.clone()],
+        db.analytics_get_all_psuedo_ids(0).await.unwrap()
+    );
+    assert!(db.analytics_get_all_psuedo_ids(1).await.unwrap().is_empty());
+
     db.analytics_create_psuedo_id_if_not_exists(c.key)
         .await
         .unwrap();
@@ -267,6 +273,7 @@ pub async fn database_works<'a, T: MCDatabase>(
             .await
             .unwrap()
     );
+
     assert_eq!(
         c.key,
         db.analytics_get_capmaign_id_from_psuedo_id(&psuedo_id)

db/db-sqlx-maria/.sqlx/query-e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "MySQL",
+  "query": "\n                SELECT\n                    psuedo_id\n                FROM\n                    mcaptcha_psuedo_campaign_id\n                    ORDER BY ID ASC LIMIT ? OFFSET ?;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "psuedo_id",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 400
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46"
+}

db/db-sqlx-maria/src/lib.rs

@@ -987,12 +987,8 @@ impl MCDatabase for Database {
         &self,
         captcha_id: &str,
     ) -> DBResult<String> {
-        struct ID {
-            psuedo_id: String,
-        }
-
         let res = sqlx::query_as!(
-            ID,
+            PsuedoID,
             "SELECT psuedo_id FROM
                 mcaptcha_psuedo_campaign_id
             WHERE
@@ -1069,6 +1065,28 @@ impl MCDatabase for Database {
 
         Ok(())
     }
+    /// Get all psuedo IDs
+    async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult<Vec<String>> {
+        const LIMIT: usize = 50;
+        let offset = LIMIT * page;
+
+        let mut res = sqlx::query_as!(
+            PsuedoID,
+            "
+                SELECT
+                    psuedo_id
+                FROM
+                    mcaptcha_psuedo_campaign_id
+                    ORDER BY ID ASC LIMIT ? OFFSET ?;",
+            LIMIT as i64,
+            offset as i64
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        Ok(res.drain(0..).map(|r| r.psuedo_id).collect())
+    }
 }
 
 #[derive(Clone)]
@@ -1134,3 +1152,7 @@ impl From<InternaleCaptchaConfig> for Captcha {
         }
     }
 }
+
+struct PsuedoID {
+    psuedo_id: String,
+}

db/db-sqlx-postgres/.sqlx/query-d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                SELECT\n                    psuedo_id\n                FROM\n                    mcaptcha_psuedo_campaign_id\n                    ORDER BY ID ASC LIMIT $1 OFFSET $2;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "psuedo_id",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7"
+}

db/db-sqlx-postgres/src/lib.rs

@@ -994,12 +994,8 @@ impl MCDatabase for Database {
         &self,
         captcha_id: &str,
     ) -> DBResult<String> {
-        struct ID {
-            psuedo_id: String,
-        }
-
         let res = sqlx::query_as!(
-            ID,
+            PsuedoID,
             "SELECT psuedo_id FROM
                 mcaptcha_psuedo_campaign_id
             WHERE
@@ -1078,6 +1074,29 @@ impl MCDatabase for Database {
 
         Ok(())
     }
+
+    /// Get all psuedo IDs
+    async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult<Vec<String>> {
+        const LIMIT: usize = 50;
+        let offset = LIMIT * page;
+
+        let mut res = sqlx::query_as!(
+            PsuedoID,
+            "
+                SELECT
+                    psuedo_id
+                FROM
+                    mcaptcha_psuedo_campaign_id
+                    ORDER BY ID ASC LIMIT $1 OFFSET $2;",
+            LIMIT as i64,
+            offset as i64
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        Ok(res.drain(0..).map(|r| r.psuedo_id).collect())
+    }
 }
 
 #[derive(Clone)]
@@ -1125,6 +1144,10 @@ impl From<InnerNotification> for Notification {
     }
 }
 
+struct PsuedoID {
+    psuedo_id: String,
+}
+
 #[derive(Clone)]
 struct InternaleCaptchaConfig {
     config_id: i32,

docker-compose.yml

@@ -11,8 +11,8 @@ services:
       - 7000:7000
     environment:
       DATABASE_URL: postgres://postgres:password@mcaptcha_postgres:5432/postgres # set password at placeholder
-      MCAPTCHA_REDIS_URL: redis://mcaptcha_redis/
+      MCAPTCHA_redis_URL: "redis://mcaptcha_redis/"
-      RUST_LOG: debug
+      RUST_LOG: "debug"
       PORT: 7000
     depends_on:
       - mcaptcha_postgres

docs/openapi/package.json

@@ -11,7 +11,7 @@
     "type": "git",
     "url": "git+https://github.com/mCaptcha/mCaptcha.git"
   },
-  "license": "AGPL3",
+  "license": "AGPL-3.0-or-later",
   "bugs": {
     "url": "https://github.com/mCaptcha/mCaptcha/issues"
   },

src/api/v1/mcaptcha/create.rs

@@ -85,10 +85,18 @@ pub mod runner {
         data.db
             .add_captcha_levels(username, &key, &payload.levels)
             .await?;
+
+        if payload.publish_benchmarks {
+            data.db
+                .analytics_create_psuedo_id_if_not_exists(&key)
+                .await?;
+        }
+
         let mcaptcha_config = MCaptchaDetails {
             name: payload.description.clone(),
             key,
         };
+
         Ok(mcaptcha_config)
     }
 }

src/api/v1/mod.rs

@@ -14,6 +14,7 @@ pub mod meta;
 pub mod notifications;
 pub mod pow;
 mod routes;
+pub mod survey;
 
 pub use routes::ROUTES;
 
@@ -24,6 +25,7 @@ pub fn services(cfg: &mut ServiceConfig) {
     account::services(cfg);
     mcaptcha::services(cfg);
     notifications::services(cfg);
+    survey::services(cfg);
 }
 
 #[derive(Deserialize)]

src/api/v1/routes.rs

@@ -11,6 +11,7 @@ use super::mcaptcha::routes::Captcha;
 use super::meta::routes::Meta;
 use super::notifications::routes::Notifications;
 use super::pow::routes::PoW;
+use super::survey::routes::Survey;
 
 pub const ROUTES: Routes = Routes::new();
 
@@ -20,6 +21,7 @@ pub struct Routes {
     pub captcha: Captcha,
     pub meta: Meta,
     pub pow: PoW,
+    pub survey: Survey,
     pub notifications: Notifications,
 }
 
@@ -32,6 +34,7 @@ impl Routes {
             meta: Meta::new(),
             pow: PoW::new(),
             notifications: Notifications::new(),
+            survey: Survey::new(),
         }
     }
 }

src/api/v1/survey.rs

@@ -0,0 +1,255 @@
+/*
+ * Copyright (C) 2023  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+use actix_web::web::ServiceConfig;
+use actix_web::{web, HttpResponse, Responder};
+use serde::{Deserialize, Serialize};
+
+use crate::errors::*;
+use crate::AppData;
+
+pub fn services(cfg: &mut ServiceConfig) {
+    cfg.service(download);
+    cfg.service(secret);
+}
+
+pub mod routes {
+    pub struct Survey {
+        pub download: &'static str,
+        pub secret: &'static str,
+    }
+
+    impl Survey {
+        pub const fn new() -> Self {
+            Self {
+                download: "/api/v1/survey/takeout/{survey_id}/get",
+                secret: "/api/v1/survey/secret",
+            }
+        }
+
+        pub fn get_download_route(&self, survey_id: &str, page: usize) -> String {
+            format!(
+                "{}?page={}",
+                self.download.replace("{survey_id}", survey_id),
+                page
+            )
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)]
+pub struct Page {
+    pub page: usize,
+}
+
+/// emits build details of the bninary
+#[my_codegen::get(path = "crate::V1_API_ROUTES.survey.download")]
+async fn download(
+    data: AppData,
+    page: web::Query<Page>,
+    psuedo_id: web::Path<uuid::Uuid>,
+) -> ServiceResult<impl Responder> {
+    const LIMIT: usize = 50;
+    let offset = LIMIT as isize * ((page.page as isize) - 1);
+    let offset = if offset < 0 { 0 } else { offset };
+    let psuedo_id = psuedo_id.into_inner();
+    let campaign_id = data
+        .db
+        .analytics_get_capmaign_id_from_psuedo_id(&psuedo_id.to_string())
+        .await?;
+    let data = data
+        .db
+        .analytics_fetch(&campaign_id, LIMIT, offset as usize)
+        .await?;
+    Ok(HttpResponse::Ok().json(data))
+}
+
+#[derive(Serialize, Deserialize)]
+struct SurveySecretUpload {
+    secret: String,
+    auth_token: String,
+}
+
+/// mCaptcha/survey upload secret route
+#[my_codegen::post(path = "crate::V1_API_ROUTES.survey.secret")]
+async fn secret(
+    data: AppData,
+    payload: web::Json<SurveySecretUpload>,
+) -> ServiceResult<impl Responder> {
+    match data.survey_secrets.get(&payload.auth_token) {
+        Some(survey_instance_url) => {
+            let payload = payload.into_inner();
+            data.survey_secrets.set(survey_instance_url, payload.secret);
+            data.survey_secrets.rm(&payload.auth_token);
+            Ok(HttpResponse::Ok())
+        }
+        None => Err(ServiceError::WrongPassword),
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use actix_web::{http::StatusCode, test, App};
+
+    use super::*;
+    use crate::api::v1::mcaptcha::get_random;
+    use crate::tests::*;
+    use crate::*;
+
+    #[actix_rt::test]
+    async fn survey_works_pg() {
+        let data = crate::tests::pg::get_data().await;
+        survey_registration_works(data.clone()).await;
+        survey_works(data).await;
+    }
+
+    #[actix_rt::test]
+    async fn survey_works_maria() {
+        let data = crate::tests::maria::get_data().await;
+        survey_registration_works(data.clone()).await;
+        survey_works(data).await;
+    }
+
+    pub async fn survey_registration_works(data: ArcData) {
+        let data = &data;
+        let app = get_app!(data).await;
+
+        let survey_instance_url = "http://survey_registration_works.survey.example.org";
+
+        let key = get_random(20);
+
+        let msg = SurveySecretUpload {
+            auth_token: key.clone(),
+            secret: get_random(32),
+        };
+
+        // should fail with ServiceError::WrongPassword since auth token is not loaded into
+        // keystore
+        bad_post_req_test_no_auth(
+            data,
+            V1_API_ROUTES.survey.secret,
+            &msg,
+            errors::ServiceError::WrongPassword,
+        )
+        .await;
+
+        // load auth token into key store, should succeed
+        data.survey_secrets
+            .set(key.clone(), survey_instance_url.to_owned());
+        let resp = test::call_service(
+            &app,
+            post_request!(&msg, V1_API_ROUTES.survey.secret).to_request(),
+        )
+        .await;
+        assert_eq!(resp.status(), StatusCode::OK);
+        // uploaded secret must be in keystore
+        assert_eq!(
+            data.survey_secrets.get(survey_instance_url).unwrap(),
+            msg.secret
+        );
+
+        // should fail since mCaptcha/survey secret upload auth tokens are single-use
+        bad_post_req_test_no_auth(
+            data,
+            V1_API_ROUTES.survey.secret,
+            &msg,
+            errors::ServiceError::WrongPassword,
+        )
+        .await;
+    }
+
+    pub async fn survey_works(data: ArcData) {
+        const NAME: &str = "survetuseranalytics";
+        const PASSWORD: &str = "longpassworddomain";
+        const EMAIL: &str = "survetuseranalytics@a.com";
+        let data = &data;
+
+        delete_user(data, NAME).await;
+
+        register_and_signin(data, NAME, EMAIL, PASSWORD).await;
+        // create captcha
+        let (_, _signin_resp, key) = add_levels_util(data, NAME, PASSWORD).await;
+        let app = get_app!(data).await;
+
+        let page = 1;
+        let tmp_id = uuid::Uuid::new_v4();
+        let download_rotue = V1_API_ROUTES
+            .survey
+            .get_download_route(&tmp_id.to_string(), page);
+
+        let download_req = test::call_service(
+            &app,
+            test::TestRequest::get().uri(&download_rotue).to_request(),
+        )
+        .await;
+        assert_eq!(download_req.status(), StatusCode::NOT_FOUND);
+
+        data.db
+            .analytics_create_psuedo_id_if_not_exists(&key.key)
+            .await
+            .unwrap();
+
+        let psuedo_id = data
+            .db
+            .analytics_get_psuedo_id_from_capmaign_id(&key.key)
+            .await
+            .unwrap();
+
+        for i in 0..60 {
+            println!("[{i}] Saving analytics");
+            let analytics = db_core::CreatePerformanceAnalytics {
+                time: 0,
+                difficulty_factor: 0,
+                worker_type: "wasm".into(),
+            };
+            data.db.analysis_save(&key.key, &analytics).await.unwrap();
+        }
+
+        for p in 1..3 {
+            let download_rotue = V1_API_ROUTES.survey.get_download_route(&psuedo_id, p);
+            println!("page={p}, download={download_rotue}");
+
+            let download_req = test::call_service(
+                &app,
+                test::TestRequest::get().uri(&download_rotue).to_request(),
+            )
+            .await;
+            assert_eq!(download_req.status(), StatusCode::OK);
+            let analytics: Vec<db_core::PerformanceAnalytics> =
+                test::read_body_json(download_req).await;
+            if p == 1 {
+                assert_eq!(analytics.len(), 50);
+            } else if p == 2 {
+                assert_eq!(analytics.len(), 10);
+            } else {
+                assert_eq!(analytics.len(), 0);
+            }
+        }
+
+        let download_rotue = V1_API_ROUTES.survey.get_download_route(&psuedo_id, 0);
+        data.db
+            .analytics_delete_all_records_for_campaign(&key.key)
+            .await
+            .unwrap();
+
+        let download_req = test::call_service(
+            &app,
+            test::TestRequest::get().uri(&download_rotue).to_request(),
+        )
+        .await;
+        assert_eq!(download_req.status(), StatusCode::NOT_FOUND);
+    }
+}

src/data.rs

@@ -4,8 +4,10 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 //! App data: redis cache, database connections, etc.
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::thread;
+use std::time::Duration;
 
 use actix::prelude::*;
 use argon2_creds::{Config, ConfigBuilder, PasswordPolicy};
@@ -28,11 +30,17 @@ use libmcaptcha::{
     pow::Work,
     system::{System, SystemBuilder},
 };
+use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use tokio::task::JoinHandle;
+use tokio::time::sleep;
 
 use crate::db::{self, BoxDB};
 use crate::errors::ServiceResult;
 use crate::settings::Settings;
 use crate::stats::{Dummy, Real, Stats};
+use crate::survey::SecretsStore;
+use crate::AppData;
 
 macro_rules! enum_system_actor {
     ($name:ident, $type:ident) => {
@@ -166,6 +174,8 @@ pub struct Data {
     pub settings: Settings,
     /// stats recorder
     pub stats: Box<dyn Stats>,
+    /// survey secret store
+    pub survey_secrets: SecretsStore,
 }
 
 impl Data {
@@ -180,7 +190,7 @@ impl Data {
     }
     #[cfg(not(tarpaulin_include))]
     /// create new instance of app data
-    pub async fn new(s: &Settings) -> Arc<Self> {
+    pub async fn new(s: &Settings, survey_secrets: SecretsStore) -> Arc<Self> {
         let creds = Self::get_creds();
         let c = creds.clone();
 
@@ -209,6 +219,7 @@ impl Data {
             mailer: Self::get_mailer(s),
             settings: s.clone(),
             stats,
+            survey_secrets,
         };
 
         #[cfg(not(debug_assertions))]
@@ -242,6 +253,13 @@ impl Data {
             None
         }
     }
+
+    async fn upload_survey_job(&self) -> ServiceResult<()> {
+        unimplemented!()
+    }
+    async fn register_survey(&self) -> ServiceResult<()> {
+        unimplemented!()
+    }
 }
 
 /// Mailer data type AsyncSmtpTransport<Tokio1Executor>

src/main.rs

@@ -30,6 +30,7 @@ mod routes;
 mod settings;
 mod static_assets;
 mod stats;
+mod survey;
 #[cfg(test)]
 #[macro_use]
 mod tests;
@@ -45,6 +46,7 @@ use static_assets::FileMap;
 pub use widget::WIDGET_ROUTES;
 
 use crate::demo::DemoUser;
+use survey::SurveyClientTrait;
 
 lazy_static! {
     pub static ref SETTINGS: Settings = Settings::new().unwrap();
@@ -93,7 +95,9 @@ pub type AppData = actix_web::web::Data<ArcData>;
 async fn main() -> std::io::Result<()> {
     use std::time::Duration;
 
+    if env::var("RUST_LOG").is_err() {
         env::set_var("RUST_LOG", "info");
+    }
 
     pretty_env_logger::init();
     info!(
@@ -102,7 +106,8 @@ async fn main() -> std::io::Result<()> {
     );
 
     let settings = Settings::new().unwrap();
-    let data = Data::new(&settings).await;
+    let secrets = survey::SecretsStore::default();
+    let data = Data::new(&settings, secrets.clone()).await;
     let data = actix_web::web::Data::new(data);
 
     let mut demo_user: Option<DemoUser> = None;
@@ -115,6 +120,13 @@ async fn main() -> std::io::Result<()> {
         );
     }
 
+    let (mut survey_upload_tx, mut survey_upload_handle) = (None, None);
+    if settings.survey.is_some() {
+        let survey_runner_ctx = survey::Survey::new(data.clone());
+        let (x, y) = survey_runner_ctx.start_job().await.unwrap();
+        (survey_upload_tx, survey_upload_handle) = (Some(x), Some(y));
+    }
+
     let ip = settings.server.get_ip();
     println!("Starting server on: http://{ip}");
 
@@ -139,9 +151,18 @@ async fn main() -> std::io::Result<()> {
     .run()
     .await?;
 
+    if let Some(survey_upload_tx) = survey_upload_tx {
+        survey_upload_tx.send(()).unwrap();
+    }
+
     if let Some(demo_user) = demo_user {
         demo_user.abort();
     }
+
+    if let Some(survey_upload_handle) = survey_upload_handle {
+        survey_upload_handle.await.unwrap();
+    }
+
     Ok(())
 }
 

src/settings.rs

@@ -91,6 +91,13 @@ pub struct Redis {
     pub pool: u32,
 }
 
+#[derive(Debug, Clone, Deserialize, Eq, PartialEq)]
+pub struct Survey {
+    pub nodes: Vec<url::Url>,
+    pub rate_limit: u64,
+    pub instance_root_url: Url,
+}
+
 #[derive(Debug, Clone, Deserialize, Eq, PartialEq)]
 pub struct Settings {
     pub debug: bool,
@@ -99,6 +106,7 @@ pub struct Settings {
     pub allow_registration: bool,
     pub allow_demo: bool,
     pub database: Database,
+    pub survey: Option<Survey>,
     pub redis: Option<Redis>,
     pub server: Server,
     pub captcha: Captcha,
@@ -118,8 +126,8 @@ const ENV_VAR_CONFIG: [(&str, &str); 29] = [
     ("database.pool", "MCAPTCHA_database_POOL"),
 
     /* redis */
-    ("redis.url", "MCPATCHA_redis_URL"),
+    ("redis.url", "MCAPTCHA_redis_URL"),
-    ("redis.pool", "MCPATCHA_redis_POOL"),
+    ("redis.pool", "MCAPTCHA_redis_POOL"),
 
     /* server */
     ("server.port", "PORT"),
@@ -145,17 +153,52 @@ const ENV_VAR_CONFIG: [(&str, &str); 29] = [
 
 
     /* SMTP */
-    ("smtp.from", "MCPATCHA_smtp_FROM"),
+    ("smtp.from", "MCAPTCHA_smtp_FROM"),
-    ("smtp.reply", "MCPATCHA_smtp_REPLY"),
+    ("smtp.reply", "MCAPTCHA_smtp_REPLY"),
-    ("smtp.url", "MCPATCHA_smtp_URL"),
+    ("smtp.url", "MCAPTCHA_smtp_URL"),
-    ("smtp.username", "MCPATCHA_smtp_USERNAME"),
+    ("smtp.username", "MCAPTCHA_smtp_USERNAME"),
-    ("smtp.password", "MCPATCHA_smtp_PASSWORD"),
+    ("smtp.password", "MCAPTCHA_smtp_PASSWORD"),
-    ("smtp.port", "MCPATCHA_smtp_PORT"),
+    ("smtp.port", "MCAPTCHA_smtp_PORT"),
 
 
 
 ];
 
+const DEPRECATED_ENV_VARS: [(&str, &str); 23] = [
+    ("debug", "MCAPTCHA_DEBUG"),
+    ("commercial", "MCAPTCHA_COMMERCIAL"),
+    ("source_code", "MCAPTCHA_SOURCE_CODE"),
+    ("allow_registration", "MCAPTCHA_ALLOW_REGISTRATION"),
+    ("allow_demo", "MCAPTCHA_ALLOW_DEMO"),
+    ("redis.pool", "MCAPTCHA_REDIS_POOL"),
+    ("redis.url", "MCAPTCHA_REDIS_URL"),
+    ("server.port", "MCAPTCHA_SERVER_PORT"),
+    ("server.ip", "MCAPTCHA_SERVER_IP"),
+    ("server.domain", "MCAPTCHA_SERVER_DOMAIN"),
+    ("server.cookie_secret", "MCAPTCHA_SERVER_COOKIE_SECRET"),
+    ("server.proxy_has_tls", "MCAPTCHA_SERVER_PROXY_HAS_TLS"),
+    ("captcha.salt", "MCAPTCHA_CAPTCHA_SALT"),
+    ("captcha.gc", "MCAPTCHA_CAPTCHA_GC"),
+    (
+        "captcha.default_difficulty_strategy.avg_traffic_difficulty",
+        "MCAPTCHA_CAPTCHA_AVG_TRAFFIC_DIFFICULTY",
+    ),
+    (
+        "captcha.default_difficulty_strategy.peak_sustainable_traffic_difficulty",
+        "MCAPTCHA_CAPTCHA_PEAK_TRAFFIC_DIFFICULTY",
+    ),
+    (
+        "captcha.default_difficulty_strategy.broke_my_site_traffic_difficulty",
+        "MCAPTCHA_CAPTCHA_BROKE_MY_SITE_TRAFFIC",
+    ),
+    ("smtp.from", "MCAPTCHA_SMTP_FROM"),
+    ("smtp.reply", "MCAPTCHA_SMTP_REPLY_TO"),
+    ("smtp.url", "MCAPTCHA_SMTP_URL"),
+    ("smtp.username", "MCAPTCHA_SMTP_USERNAME"),
+    ("smtp.password", "MCAPTCHA_SMTP_PASSWORD"),
+    ("smtp.port", "MCAPTCHA_SMTP_PORT"),
+];
+
 #[cfg(not(tarpaulin_include))]
 impl Settings {
     pub fn new() -> Result<Self, ConfigError> {
@@ -210,6 +253,15 @@ impl Settings {
     }
 
     fn env_override(mut s: ConfigBuilder<DefaultState>) -> ConfigBuilder<DefaultState> {
+        for (parameter, env_var_name) in DEPRECATED_ENV_VARS.iter() {
+            if let Ok(val) = env::var(env_var_name) {
+                log::warn!(
+                    "Found {env_var_name}. {env_var_name} will be deprecated soon. Please see https://github.com/mCaptcha/mCaptcha/blob/master/docs/CONFIGURATION.md for latest environment variable names"
+                );
+                s = s.set_override(parameter, val).unwrap();
+            }
+        }
+
         for (parameter, env_var_name) in ENV_VAR_CONFIG.iter() {
             if let Ok(val) = env::var(env_var_name) {
                 log::debug!(
@@ -240,7 +292,7 @@ mod tests {
     use super::*;
 
     #[test]
-    fn env_override_works() {
+    fn deprecated_env_override_works() {
         use crate::tests::get_settings;
         let init_settings = get_settings();
         // so that it can be tested outside the macro (helper) too
@@ -249,6 +301,141 @@ mod tests {
         macro_rules! helper {
 
 
+            ($env:expr, $val:expr, $val_typed:expr, $($param:ident).+) => {
+                println!("Setting env var {} to {} for test", $env, $val);
+                env::set_var($env, $val);
+                new_settings = get_settings();
+                assert_eq!(new_settings.$($param).+, $val_typed);
+                assert_ne!(new_settings.$($param).+, init_settings.$($param).+);
+                env::remove_var($env);
+            };
+
+
+            ($env:expr, $val:expr, $($param:ident).+) => {
+                helper!($env, $val.to_string(), $val, $($param).+);
+            };
+        }
+
+        /* top level */
+        helper!("MCAPTCHA_DEBUG", !init_settings.debug, debug);
+        helper!("MCAPTCHA_COMMERCIAL", !init_settings.commercial, commercial);
+        helper!(
+            "MCAPTCHA_ALLOW_REGISTRATION",
+            !init_settings.allow_registration,
+            allow_registration
+        );
+        helper!("MCAPTCHA_ALLOW_DEMO", !init_settings.allow_demo, allow_demo);
+
+        /* database_type */
+
+        /* redis.url */
+        let env = "MCAPTCHA_REDIS_URL";
+        let val = "redis://redis.example.org";
+        println!("Setting env var {} to {} for test", env, val);
+        env::set_var(env, val);
+        new_settings = get_settings();
+        assert_eq!(new_settings.redis.as_ref().unwrap().url, val);
+        assert_ne!(
+            new_settings.redis.as_ref().unwrap().url,
+            init_settings.redis.as_ref().unwrap().url
+        );
+        env::remove_var(env);
+
+        /* redis.pool */
+        let env = "MCAPTCHA_REDIS_POOL";
+        let val = 999;
+        println!("Setting env var {} to {} for test", env, val);
+        env::set_var(env, val.to_string());
+        new_settings = get_settings();
+        assert_eq!(new_settings.redis.as_ref().unwrap().pool, val);
+        assert_ne!(
+            new_settings.redis.as_ref().unwrap().pool,
+            init_settings.redis.as_ref().unwrap().pool
+        );
+        env::remove_var(env);
+
+        helper!("PORT", 0, server.port);
+        helper!("MCAPTCHA_SERVER_DOMAIN", "example.org", server.domain);
+        helper!(
+            "MCAPTCHA_SERVER_COOKIE_SECRET",
+            "dafasdfsdf",
+            server.cookie_secret
+        );
+        helper!("MCAPTCHA_SERVER_IP", "9.9.9.9", server.ip);
+        helper!("MCAPTCHA_SERVER_PROXY_HAS_TLS", true, server.proxy_has_tls);
+
+        /* captcha */
+
+        helper!("MCAPTCHA_CAPTCHA_SALT", "foobarasdfasdf", captcha.salt);
+        helper!("MCAPTCHA_CAPTCHA_GC", 500, captcha.gc);
+        helper!(
+            "MCAPTCHA_captcha_RUNNERS",
+            "500",
+            Some(500),
+            captcha.runners
+        );
+
+        helper!(
+            "MCAPTCHA_CAPTCHA_AVG_TRAFFIC_DIFFICULTY",
+            999,
+            captcha.default_difficulty_strategy.avg_traffic_difficulty
+        );
+        helper!(
+            "MCAPTCHA_CAPTCHA_PEAK_TRAFFIC_DIFFICULTY",
+            999,
+            captcha
+                .default_difficulty_strategy
+                .peak_sustainable_traffic_difficulty
+        );
+        helper!(
+            "MCAPTCHA_CAPTCHA_BROKE_MY_SITE_TRAFFIC",
+            999,
+            captcha
+                .default_difficulty_strategy
+                .broke_my_site_traffic_difficulty
+        );
+
+        /* SMTP */
+
+        let vals = [
+            "MCAPTCHA_SMTP_FROM",
+            "MCAPTCHA_SMTP_REPLY_TO",
+            "MCAPTCHA_SMTP_URL",
+            "MCAPTCHA_SMTP_USERNAME",
+            "MCAPTCHA_SMTP_PASSWORD",
+            "MCAPTCHA_SMTP_PORT",
+        ];
+        for env in vals.iter() {
+            println!("Setting env var {} to {} for test", env, env);
+            env::set_var(env, env);
+        }
+
+        let port = 9999;
+        env::set_var("MCAPTCHA_SMTP_PORT", port.to_string());
+
+        new_settings = get_settings();
+        let smtp_new = new_settings.smtp.as_ref().unwrap();
+        let smtp_old = init_settings.smtp.as_ref().unwrap();
+        assert_eq!(smtp_new.from, "MCAPTCHA_SMTP_FROM");
+        assert_eq!(smtp_new.reply, "MCAPTCHA_SMTP_REPLY_TO");
+        assert_eq!(smtp_new.username, "MCAPTCHA_SMTP_USERNAME");
+        assert_eq!(smtp_new.password, "MCAPTCHA_SMTP_PASSWORD");
+        assert_eq!(smtp_new.port, port);
+        assert_ne!(smtp_new, smtp_old);
+
+        for env in vals.iter() {
+            env::remove_var(env);
+        }
+    }
+
+    #[test]
+    fn env_override_works() {
+        use crate::tests::get_settings;
+        let init_settings = get_settings();
+        // so that it can be tested outside the macro (helper) too
+        let mut new_settings;
+
+        macro_rules! helper {
 
 
             ($env:expr, $val:expr, $val_typed:expr, $($param:ident).+) => {
@@ -291,7 +478,7 @@ mod tests {
         /* redis */
 
         /* redis.url */
-        let env = "MCPATCHA_redis_URL";
+        let env = "MCAPTCHA_redis_URL";
         let val = "redis://redis.example.org";
         println!("Setting env var {} to {} for test", env, val);
         env::set_var(env, val);
@@ -304,7 +491,7 @@ mod tests {
         env::remove_var(env);
 
         /* redis.pool */
-        let env = "MCPATCHA_redis_POOL";
+        let env = "MCAPTCHA_redis_POOL";
         let val = 999;
         println!("Setting env var {} to {} for test", env, val);
         env::set_var(env, val.to_string());
@@ -355,12 +542,12 @@ mod tests {
         /* SMTP */
 
         let vals = [
-            "MCPATCHA_smtp_FROM",
+            "MCAPTCHA_smtp_FROM",
-            "MCPATCHA_smtp_REPLY",
+            "MCAPTCHA_smtp_REPLY",
-            "MCPATCHA_smtp_URL",
+            "MCAPTCHA_smtp_URL",
-            "MCPATCHA_smtp_USERNAME",
+            "MCAPTCHA_smtp_USERNAME",
-            "MCPATCHA_smtp_PASSWORD",
+            "MCAPTCHA_smtp_PASSWORD",
-            "MCPATCHA_smtp_PORT",
+            "MCAPTCHA_smtp_PORT",
         ];
         for env in vals.iter() {
             println!("Setting env var {} to {} for test", env, env);
@@ -368,15 +555,15 @@ mod tests {
         }
 
         let port = 9999;
-        env::set_var("MCPATCHA_smtp_PORT", port.to_string());
+        env::set_var("MCAPTCHA_smtp_PORT", port.to_string());
 
         new_settings = get_settings();
         let smtp_new = new_settings.smtp.as_ref().unwrap();
         let smtp_old = init_settings.smtp.as_ref().unwrap();
-        assert_eq!(smtp_new.from, "MCPATCHA_smtp_FROM");
+        assert_eq!(smtp_new.from, "MCAPTCHA_smtp_FROM");
-        assert_eq!(smtp_new.reply, "MCPATCHA_smtp_REPLY");
+        assert_eq!(smtp_new.reply, "MCAPTCHA_smtp_REPLY");
-        assert_eq!(smtp_new.username, "MCPATCHA_smtp_USERNAME");
+        assert_eq!(smtp_new.username, "MCAPTCHA_smtp_USERNAME");
-        assert_eq!(smtp_new.password, "MCPATCHA_smtp_PASSWORD");
+        assert_eq!(smtp_new.password, "MCAPTCHA_smtp_PASSWORD");
         assert_eq!(smtp_new.port, port);
         assert_ne!(smtp_new, smtp_old);
 

src/survey.rs

@@ -0,0 +1,209 @@
+// Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+// SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::time::Duration;
+
+use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use tokio::sync::oneshot;
+use tokio::task::JoinHandle;
+use tokio::time::sleep;
+
+use crate::errors::*;
+use crate::settings::Settings;
+use crate::AppData;
+use crate::V1_API_ROUTES;
+
+#[async_trait::async_trait]
+pub trait SurveyClientTrait {
+    async fn start_job(&self) -> ServiceResult<(oneshot::Sender<()>, JoinHandle<()>)>;
+    async fn schedule_upload_job(&self) -> ServiceResult<()>;
+    async fn is_online(&self) -> ServiceResult<bool>;
+    async fn register(&self) -> ServiceResult<()>;
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct SecretsStore {
+    store: Arc<RwLock<HashMap<String, String>>>,
+}
+
+impl SecretsStore {
+    pub fn get(&self, key: &str) -> Option<String> {
+        let r = self.store.read().unwrap();
+        r.get(key).map(|x| x.to_owned())
+    }
+
+    pub fn rm(&self, key: &str) {
+        let mut w = self.store.write().unwrap();
+        w.remove(key);
+        drop(w);
+    }
+
+    pub fn set(&self, key: String, value: String) {
+        let mut w = self.store.write().unwrap();
+        w.insert(key, value);
+        drop(w);
+    }
+}
+
+#[derive(Clone)]
+pub struct Survey {
+    client: Client,
+    app_ctx: AppData,
+}
+impl Survey {
+    pub fn new(app_ctx: AppData) -> Self {
+        if app_ctx.settings.survey.is_none() {
+            panic!("Survey uploader shouldn't be initialized it isn't configured, please report this bug")
+        }
+        Survey {
+            client: Client::new(),
+            app_ctx,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl SurveyClientTrait for Survey {
+    async fn start_job(&self) -> ServiceResult<(oneshot::Sender<()>, JoinHandle<()>)> {
+        fn can_run(rx: &mut oneshot::Receiver<()>) -> bool {
+            match rx.try_recv() {
+                Err(oneshot::error::TryRecvError::Empty) => true,
+                _ => false,
+            }
+        }
+
+        let (tx, mut rx) = oneshot::channel();
+        let this = self.clone();
+        let mut register = false;
+        let fut = async move {
+            loop {
+                if !can_run(&mut rx) {
+                    log::info!("Stopping survey uploads");
+                    break;
+                }
+
+                if !register {
+                    loop {
+                        if this.is_online().await.unwrap() {
+                            this.register().await.unwrap();
+                            register = true;
+                            break;
+                        } else {
+                            sleep(Duration::new(1, 0)).await;
+                        }
+                    }
+                }
+
+                for i in 0..this.app_ctx.settings.survey.as_ref().unwrap().rate_limit {
+                    if !can_run(&mut rx) {
+                        log::info!("Stopping survey uploads");
+                        break;
+                    }
+                    sleep(Duration::new(1, 0)).await;
+                }
+                let _ = this.schedule_upload_job().await;
+
+                // for url in this.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() {
+                //     if !can_run(&mut rx) {
+                //         log::info!("Stopping survey uploads");
+                //         break;
+                //     }
+                //     log::info!("Uploading to survey instance {}", url);
+                // }
+            }
+        };
+        let handle = tokio::spawn(fut);
+        Ok((tx, handle))
+    }
+    async fn is_online(&self) -> ServiceResult<bool> {
+        let res = self
+            .client
+            .get(format!(
+                "http://{}{}",
+                self.app_ctx.settings.server.get_ip(),
+                V1_API_ROUTES.meta.health
+            ))
+            .send()
+            .await
+            .unwrap();
+        Ok(res.status() == 200)
+    }
+
+    async fn schedule_upload_job(&self) -> ServiceResult<()> {
+        log::debug!("Running upload job");
+        #[derive(Serialize)]
+        struct Secret {
+            secret: String,
+        }
+        let mut page = 0;
+        loop {
+            let psuedo_ids = self.app_ctx.db.analytics_get_all_psuedo_ids(page).await?;
+            if psuedo_ids.is_empty() {
+                log::debug!("upload job complete, no more IDs to upload");
+                break;
+            }
+            for id in psuedo_ids {
+                for url in self.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() {
+                    if let Some(secret) = self.app_ctx.survey_secrets.get(url.as_str()) {
+                        let payload = Secret { secret };
+
+                        log::info!("Uploading to survey instance {} campaign {id}", url);
+                        let mut url = url.clone();
+                        url.set_path(&format!("/mcaptcha/api/v1/{id}/upload"));
+                        let resp =
+                            self.client.post(url).json(&payload).send().await.unwrap();
+                        println!("{}", resp.text().await.unwrap());
+                    }
+                }
+            }
+            page += 1;
+        }
+        Ok(())
+    }
+
+    async fn register(&self) -> ServiceResult<()> {
+        #[derive(Serialize)]
+        struct MCaptchaInstance {
+            url: url::Url,
+            auth_token: String,
+        }
+
+        let this_instance_url = self
+            .app_ctx
+            .settings
+            .survey
+            .as_ref()
+            .unwrap()
+            .instance_root_url
+            .clone();
+        for url in self.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() {
+            // mCaptcha/survey must send this token while uploading secret to authenticate itself
+            // this token must be sent to mCaptcha/survey with the registration payload
+            let secret_upload_auth_token = crate::api::v1::mcaptcha::get_random(20);
+
+            let payload = MCaptchaInstance {
+                url: this_instance_url.clone(),
+                auth_token: secret_upload_auth_token.clone(),
+            };
+
+            // SecretsStore will store auth tokens generated by both mCaptcha/mCaptcha and
+            // mCaptcha/survey
+            //
+            // Storage schema:
+            // - mCaptcha/mCaptcha generated auth token: (<auth_token>, <survey_instance_url>)
+            // - mCaptcha/survey generated auth token (<survey_instance_url>, <auth_token)
+            self.app_ctx
+                .survey_secrets
+                .set(secret_upload_auth_token, url.to_string());
+            let mut url = url.clone();
+            url.set_path("/mcaptcha/api/v1/register");
+            let resp = self.client.post(url).json(&payload).send().await.unwrap();
+        }
+        Ok(())
+    }
+}

src/tests/mod.rs

@@ -20,6 +20,7 @@ use crate::api::v1::mcaptcha::create::CreateCaptcha;
 use crate::api::v1::mcaptcha::create::MCaptchaDetails;
 use crate::api::v1::ROUTES;
 use crate::errors::*;
+use crate::survey::SecretsStore;
 use crate::ArcData;
 
 pub fn get_settings() -> Settings {
@@ -30,6 +31,7 @@ pub mod pg {
 
     use crate::data::Data;
     use crate::settings::*;
+    use crate::survey::SecretsStore;
     use crate::ArcData;
 
     use super::get_settings;
@@ -42,7 +44,7 @@ pub mod pg {
         settings.database.database_type = DBType::Postgres;
         settings.database.pool = 2;
 
-        Data::new(&settings).await
+        Data::new(&settings, SecretsStore::default()).await
     }
 }
 pub mod maria {
@@ -50,6 +52,7 @@ pub mod maria {
 
     use crate::data::Data;
     use crate::settings::*;
+    use crate::survey::SecretsStore;
     use crate::ArcData;
 
     use super::get_settings;
@@ -62,7 +65,7 @@ pub mod maria {
         settings.database.database_type = DBType::Maria;
         settings.database.pool = 2;
 
-        Data::new(&settings).await
+        Data::new(&settings, SecretsStore::default()).await
     }
 }
 //pub async fn get_data() -> ArcData {
@@ -181,6 +184,26 @@ pub async fn signin(
     (creds, signin_resp)
 }
 
+/// pub duplicate test
+pub async fn bad_post_req_test_no_auth<T: Serialize>(
+    data: &ArcData,
+    url: &str,
+    payload: &T,
+    err: ServiceError,
+) {
+    let app = get_app!(data).await;
+
+    let resp = test::call_service(&app, post_request!(&payload, url).to_request()).await;
+    if resp.status() != err.status_code() {
+        let resp_err: ErrorToResponse = test::read_body_json(resp).await;
+        panic!("error {}", resp_err.error);
+    }
+    assert_eq!(resp.status(), err.status_code());
+    let resp_err: ErrorToResponse = test::read_body_json(resp).await;
+    //println!("{}", txt.error);
+    assert_eq!(resp_err.error, format!("{}", err));
+}
+
 /// pub duplicate test
 pub async fn bad_post_req_test<T: Serialize>(
     data: &ArcData,