feat: schedule mCaptcha/survey registration and uploads

hotfix: read soon-to-be deprecated env vars to avoid breakages like #114

.github/workflows/coverage.yml

@@ -1,119 +1,119 @@
-name: Coverage
+#name: Coverage
-
+#
-on:
+#on:
-  pull_request:
+#  pull_request:
-    types: [opened, synchronize, reopened]
+#    types: [opened, synchronize, reopened]
-  push:
+#  push:
-    branches:
+#    branches:
-      - master
+#      - master
-      - db-abstract
+#      - db-abstract
-
+#
-jobs:
+#jobs:
-  build_and_test:
+#  build_and_test:
-    strategy:
+#    strategy:
-      fail-fast: false
+#      fail-fast: false
-      matrix:
+#      matrix:
-        version:
+#        version:
-          - stable
+#          - stable
-            #- 1.51.0
+#            #- 1.51.0
-
+#
-    name: ${{ matrix.version }} - x86_64-unknown-linux-gnu
+#    name: ${{ matrix.version }} - x86_64-unknown-linux-gnu
-    runs-on: ubuntu-latest
+#    runs-on: ubuntu-latest
-
+#
-    services:
+#    services:
-      postgres:
+#      postgres:
-        image: postgres
+#        image: postgres
-        env:
+#        env:
-          POSTGRES_PASSWORD: password
+#          POSTGRES_PASSWORD: password
-          POSTGRES_USER: postgres
+#          POSTGRES_USER: postgres
-          POSTGRES_DB: postgres
+#          POSTGRES_DB: postgres
-        options: >-
+#        options: >-
-          --health-cmd pg_isready
+#          --health-cmd pg_isready
-          --health-interval 10s
+#          --health-interval 10s
-          --health-timeout 5s
+#          --health-timeout 5s
-          --health-retries 5
+#          --health-retries 5
-        ports:
+#        ports:
-          - 5432:5432
+#          - 5432:5432
-
+#
-      mcaptcha-redis:
+#      mcaptcha-redis:
-        image: mcaptcha/cache
+#        image: mcaptcha/cache
-        ports:
+#        ports:
-          - 6379:6379
+#          - 6379:6379
-
+#
-      mcaptcha-smtp:
+#      mcaptcha-smtp:
-        image: maildev/maildev
+#        image: maildev/maildev
-        env:
+#        env:
-          MAILDEV_WEB_PORT: "1080"
+#          MAILDEV_WEB_PORT: "1080"
-          MAILDEV_INCOMING_USER: "admin"
+#          MAILDEV_INCOMING_USER: "admin"
-          MAILDEV_INCOMING_PASS: "password"
+#          MAILDEV_INCOMING_PASS: "password"
-        ports:
+#        ports:
-          - 1080:1080
+#          - 1080:1080
-          - 10025:1025
+#          - 10025:1025
-
+#
-
+#
-      maria:
+#      maria:
-        image: mariadb:10
+#        image: mariadb:10
-        env: 
+#        env: 
-          MARIADB_USER: "maria"
+#          MARIADB_USER: "maria"
-          MARIADB_PASSWORD: "password"
+#          MARIADB_PASSWORD: "password"
-          MARIADB_ROOT_PASSWORD: "password"
+#          MARIADB_ROOT_PASSWORD: "password"
-          MARIADB_DATABASE: "maria"
+#          MARIADB_DATABASE: "maria"
-        options: >-
+#        options: >-
-          --health-cmd="mysqladmin ping"
+#          --health-cmd="mysqladmin ping"
-          --health-interval=10s
+#          --health-interval=10s
-          --health-timeout=5s
+#          --health-timeout=5s
-          --health-retries=10
+#          --health-retries=10
-        ports:
+#        ports:
-          - 3306:3306
+#          - 3306:3306
-
+#
-
+#
-    steps:
+#    steps:
-      - uses: actions/checkout@v4
+#      - uses: actions/checkout@v4
-
+#
-      - name: load env
+#      - name: load env
-        run: |
+#        run: |
-          source .env_sample \
+#          source .env_sample \
-            && echo "POSTGRES_DATABASE_URL=$POSTGRES_DATABASE_URL" >> $GITHUB_ENV \
+#            && echo "POSTGRES_DATABASE_URL=$POSTGRES_DATABASE_URL" >> $GITHUB_ENV \
-            && echo "MARIA_DATABASE_URL=$MARIA_DATABASE_URL" >> $GITHUB_ENV
+#            && echo "MARIA_DATABASE_URL=$MARIA_DATABASE_URL" >> $GITHUB_ENV
-
+#
-
+#
-      - uses: actions/setup-node@v2
+#      - uses: actions/setup-node@v2
-        with:
+#        with:
-          node-version: "18.0.0"
+#          node-version: "18.0.0"
-
+#
-      - uses: actions-rust-lang/setup-rust-toolchain@v1
+#      - uses: actions-rust-lang/setup-rust-toolchain@v1
-
+#
-      - name: Build frontend
+#      - name: Build frontend
-        run: make frontend
+#        run: make frontend
-
+#
-      - name: Run the frontend tests
+#      - name: Run the frontend tests
-        run: make test.frontend
+#        run: make test.frontend
-
+#
-      - name: Run migrations
+#      - name: Run migrations
-        run: make migrate
+#        run: make migrate
-        env:
+#        env:
-          POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}"
+#          POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}"
-          MARIA_DATABASE_URL: "${{ env.MARIA_DATABASE_URL }}"
+#          MARIA_DATABASE_URL: "${{ env.MARIA_DATABASE_URL }}"
-
+#
-      - name: build frontend
+#      - name: build frontend
-        run: make frontend
+#        run: make frontend
-
+#
-      - name: Generate coverage file
+#      - name: Generate coverage file
-        if: github.event_name == 'pull_request'
+#        if: github.event_name == 'pull_request'
-        #if: (github.ref == 'refs/heads/master' || github.event_name == 'pull_request')
+#        #if: (github.ref == 'refs/heads/master' || github.event_name == 'pull_request')
-        uses: actions-rs/tarpaulin@v0.1
+#        uses: actions-rs/tarpaulin@v0.1
-        with:
+#        with:
-          args: "-t 1200"
+#          args: "-t 1200"
-        env:
+#        env:
-          POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}"
+#          POSTGRES_DATABASE_URL: "${{ env.POSTGRES_DATABASE_URL }}"
-          MARIA_DATABASE_URL: "${{ env.MARIA_DATABASE_URL }}"
+#          MARIA_DATABASE_URL: "${{ env.MARIA_DATABASE_URL }}"
-          # GIT_HASH is dummy value. I guess build.rs is skipped in tarpaulin
+#          # GIT_HASH is dummy value. I guess build.rs is skipped in tarpaulin
-          # execution so this value is required for preventing meta tests from
+#          # execution so this value is required for preventing meta tests from
-          # panicking
+#          # panicking
-          GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61
+#          GIT_HASH: 8e77345f1597e40c2e266cb4e6dee74888918a61
-          CACHE_BUSTER_FILE_MAP: '{"map":{"./static/bundle/main.js":"./prod/bundle/main.1417115E59909BE0A01040A45A398ADB09D928DF89CCF038FA44B14850442096.js"},"base_dir":"./prod"}'
+#          CACHE_BUSTER_FILE_MAP: '{"map":{"./static/bundle/main.js":"./prod/bundle/main.1417115E59909BE0A01040A45A398ADB09D928DF89CCF038FA44B14850442096.js"},"base_dir":"./prod"}'
-          COMPILED_DATE: "2021-07-21"
+#          COMPILED_DATE: "2021-07-21"
-
+#
-      - name: Upload to Codecov
+#      - name: Upload to Codecov
-        if: github.event_name == 'pull_request'
+#        if: github.event_name == 'pull_request'
-        uses: codecov/codecov-action@v2
+#        uses: codecov/codecov-action@v2

.github/workflows/linux.yml

@@ -119,7 +119,7 @@ jobs:
         run: make test.integration
 
       - name: Login to DockerHub
-        if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'mCaptcha/mCaptcha'
+        if: github.ref == 'refs/heads/master' && github.event_name == 'push' && github.repository == 'mCaptcha/mCaptcha'
         uses: docker/login-action@v1
         with:
           username: mcaptcha
@@ -129,12 +129,12 @@ jobs:
         if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'mCaptcha/mCaptcha'
         run: make docker-publish
 
-      - name: publish bins
+          #      - name: publish bins
-        if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'mCaptcha/mCaptcha'
+          #        if: (github.ref == 'refs/heads/master' || github.event_name == 'push') && github.repository == 'mCaptcha/mCaptcha'
-        run: ./scripts/publish.sh publish master latest $DUMBSERVE_PASSWORD
+          #        run: ./scripts/publish.sh publish master latest $DUMBSERVE_PASSWORD
-        env:
+          #        env:
-          DUMBSERVE_PASSWORD: ${{ secrets.DUMBSERVE_PASSWORD }}
+          #          DUMBSERVE_PASSWORD: ${{ secrets.DUMBSERVE_PASSWORD }}
-          GPG_PASSWORD: ${{ secrets.GPG_PASSWORD }}
+          #          GPG_PASSWORD: ${{ secrets.GPG_PASSWORD }}
 
       - name: generate documentation
         if: matrix.version == 'stable' && (github.repository == 'mCaptcha/mCaptcha')

.gitignore

@@ -16,3 +16,5 @@ assets
 yarn-error.log
 src/cache_buster_data.json.license
 **/**/target
+src/libcachebust_data.json
+utils/cache-bust/src/libcachebust_data.json

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ### Changed
 
+-   2023-10-18: Environment variable names have changed, please see
+    [CONFIGURATION.md](docs/CONFIGURATION.md) for the names of environment
+    variables.
 -   ([`7d0e4c6`](https://github.com/mCaptcha/mCaptcha/commit/7d0e4c6be4b0769921cda7681858ebe16ec9a07b)) Add `secret` parameter to token verification request payload(`/api/v1/pow/siteverify`) to mitigate a security issue that @gusted found:
     > ...A malicious user could grab the sitekey
     > and use that sitekey with mcaptcha to use it for their own server.

Cargo.lock

@@ -101,7 +101,7 @@ dependencies = [
  "percent-encoding",
  "pin-project-lite",
  "rand",
- "sha1 0.10.5",
+ "sha1",
  "smallvec",
  "tokio",
  "tokio-util",
@@ -439,6 +439,19 @@ version = "0.7.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711"
 
+[[package]]
+name = "async-compression"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2"
+dependencies = [
+ "flate2",
+ "futures-core",
+ "memchr",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "async-trait"
 version = "0.1.53"
@@ -641,21 +654,6 @@ dependencies = [
  "bytes",
 ]
 
-[[package]]
-name = "cache-buster"
-version = "0.2.0"
-source = "git+https://github.com/realaravinth/cache-buster#7ca4545722fb99be30698a5e72c7d982a70fa11f"
-dependencies = [
- "data-encoding",
- "derive_builder 0.11.1",
- "mime",
- "mime_guess",
- "serde",
- "serde_json",
- "sha2",
- "walkdir",
-]
-
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -1125,7 +1123,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36"
 dependencies = [
  "atty",
- "humantime",
+ "humantime 1.3.0",
+ "log",
+ "regex",
+ "termcolor",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0"
+dependencies = [
+ "humantime 2.1.0",
+ "is-terminal",
  "log",
  "regex",
  "termcolor",
@@ -1534,6 +1545,17 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http-body"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
+dependencies = [
+ "bytes",
+ "http",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "httparse"
 version = "1.8.0"
@@ -1555,6 +1577,49 @@ dependencies = [
  "quick-error",
 ]
 
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
+[[package]]
+name = "hyper"
+version = "0.14.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -1647,6 +1712,23 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "ipnet"
+version = "2.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6"
+
+[[package]]
+name = "is-terminal"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b"
+dependencies = [
+ "hermit-abi 0.3.1",
+ "rustix 0.38.13",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "itertools"
 version = "0.11.0"
@@ -1744,6 +1826,22 @@ version = "0.2.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
 
+[[package]]
+name = "libcachebust"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e65010b256a3c85a90e88212d3e360d2ac15643d087878f09837ba6b2c83c120"
+dependencies = [
+ "data-encoding",
+ "derive_builder 0.12.0",
+ "mime",
+ "mime_guess",
+ "serde",
+ "serde_json",
+ "sha2",
+ "walkdir",
+]
+
 [[package]]
 name = "libm"
 version = "0.2.7"
@@ -1752,17 +1850,18 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4"
 
 [[package]]
 name = "libmcaptcha"
-version = "0.2.3"
+version = "0.2.4"
-source = "git+https://github.com/mCaptcha/libmcaptcha?tag=0.2.3#6bc0d11f8839d18c4d930cac0c0b86505ed1b2f6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3bd24de3df9ea98d75d929dc4df5d637832213d3deb61d2cfa5c9055ccdaea6b"
 dependencies = [
  "actix",
  "crossbeam-channel",
- "derive_builder 0.11.1",
+ "derive_builder 0.12.0",
  "derive_more",
  "log",
+ "mcaptcha_pow_sha256",
  "num_cpus",
- "pow_sha256",
+ "pretty_env_logger 0.5.0",
- "pretty_env_logger",
  "rand",
  "redis",
  "serde",
@@ -1793,6 +1892,12 @@ version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
+
 [[package]]
 name = "local-channel"
 version = "0.1.3"
@@ -1881,7 +1986,6 @@ dependencies = [
  "argon2-creds",
  "async-trait",
  "awc",
- "cache-buster",
  "config",
  "db-core",
  "db-sqlx-maria",
@@ -1891,6 +1995,7 @@ dependencies = [
  "futures",
  "lazy_static",
  "lettre",
+ "libcachebust",
  "libmcaptcha",
  "log",
  "mcaptcha_pow_sha256",
@@ -1898,8 +2003,9 @@ dependencies = [
  "mime_guess",
  "num_cpus",
  "openssl",
- "pretty_env_logger",
+ "pretty_env_logger 0.4.0",
  "rand",
+ "reqwest",
  "rust-embed",
  "sailfish",
  "serde",
@@ -2341,6 +2447,26 @@ dependencies = [
  "siphasher",
 ]
 
+[[package]]
+name = "pin-project"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.38",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.10"
@@ -2392,18 +2518,6 @@ dependencies = [
  "universal-hash",
 ]
 
-[[package]]
-name = "pow_sha256"
-version = "0.3.1"
-source = "git+https://github.com/mcaptcha/pow_sha256#148f1cb70d19114d1340661a77b2b679e95715f6"
-dependencies = [
- "bincode",
- "derive_builder 0.11.1",
- "num",
- "serde",
- "sha2",
-]
-
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -2422,7 +2536,17 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d"
 dependencies = [
- "env_logger",
+ "env_logger 0.7.1",
+ "log",
+]
+
+[[package]]
+name = "pretty_env_logger"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "865724d4dbe39d9f3dd3b52b88d859d66bcb2d6a0acfd5ea68a65fb66d4bdc1c"
+dependencies = [
+ "env_logger 0.10.0",
  "log",
 ]
 
@@ -2523,9 +2647,9 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.21.7"
+version = "0.23.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "152f3863635cbb76b73bc247845781098302c6c9ad2060e1a9a7de56840346b6"
+checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -2540,8 +2664,10 @@ dependencies = [
  "r2d2",
  "rand",
  "ryu",
- "sha1 0.6.1",
+ "sha1_smol",
+ "socket2",
  "tokio",
+ "tokio-retry",
  "tokio-util",
  "url",
 ]
@@ -2581,6 +2707,46 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
 
+[[package]]
+name = "reqwest"
+version = "0.11.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b"
+dependencies = [
+ "async-compression",
+ "base64 0.21.2",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "system-configuration",
+ "tokio",
+ "tokio-native-tls",
+ "tokio-util",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -2710,7 +2876,20 @@ dependencies = [
  "errno",
  "io-lifetimes",
  "libc",
- "linux-raw-sys",
+ "linux-raw-sys 0.3.8",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "rustix"
+version = "0.38.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662"
+dependencies = [
+ "bitflags 2.4.1",
+ "errno",
+ "libc",
+ "linux-raw-sys 0.4.10",
  "windows-sys 0.48.0",
 ]
 
@@ -2912,15 +3091,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "sha1"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770"
-dependencies = [
- "sha1_smol",
-]
-
 [[package]]
 name = "sha1"
 version = "0.10.5"
@@ -3163,7 +3333,7 @@ dependencies = [
  "rand",
  "rsa",
  "serde",
- "sha1 0.10.5",
+ "sha1",
  "sha2",
  "smallvec",
  "sqlx-core",
@@ -3203,7 +3373,7 @@ dependencies = [
  "rand",
  "serde",
  "serde_json",
- "sha1 0.10.5",
+ "sha1",
  "sha2",
  "smallvec",
  "sqlx-core",
@@ -3307,6 +3477,27 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "system-configuration"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "tempfile"
 version = "3.6.0"
@@ -3317,7 +3508,7 @@ dependencies = [
  "cfg-if",
  "fastrand",
  "redox_syscall 0.3.5",
- "rustix",
+ "rustix 0.37.21",
  "windows-sys 0.48.0",
 ]
 
@@ -3431,6 +3622,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-retry"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
+dependencies = [
+ "pin-project",
+ "rand",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-stream"
 version = "0.1.14"
@@ -3499,6 +3701,12 @@ dependencies = [
  "winnow",
 ]
 
+[[package]]
+name = "tower-service"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+
 [[package]]
 name = "tracing"
 version = "0.1.37"
@@ -3532,6 +3740,12 @@ dependencies = [
  "once_cell",
 ]
 
+[[package]]
+name = "try-lock"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
+
 [[package]]
 name = "typenum"
 version = "1.16.0"
@@ -3706,6 +3920,15 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
 [[package]]
 name = "wasi"
 version = "0.11.0+wasi-snapshot-preview1"
@@ -3737,6 +3960,18 @@ dependencies = [
  "wasm-bindgen-shared",
 ]
 
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "wasm-bindgen-macro"
 version = "0.2.84"
@@ -3954,6 +4189,16 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "winreg"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
+dependencies = [
+ "cfg-if",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "yaml-rust"
 version = "0.4.5"

Cargo.toml

@@ -33,7 +33,7 @@ actix-service = "2.0.0"
 async-trait = "0.1.51"
 mime_guess = "2.0.3"
 rust-embed = "6.4.0"
-cache-buster = { git = "https://github.com/realaravinth/cache-buster" }
+libcachebust = "0.3.0"
 
 futures = "0.3.15"
 tokio = { version = "1.14", features = ["sync"]}
@@ -59,9 +59,7 @@ log = "0.4"
 lazy_static = "1.4"
 
 
-libmcaptcha = { version = "0.2.3", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"], tag ="0.2.3" }
+libmcaptcha = "0.2.4"
-#libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"] }
-#libmcaptcha = { path = "../libmcaptcha", features = ["full"]}
 
 rand = "0.8"
 
@@ -80,6 +78,7 @@ lettre = { version = "0.10.0-rc.3", features = [
 
 openssl = { version = "0.10.48", features = ["vendored"] }
 uuid = { version = "1.4.0", features = ["v4", "serde"] }
+reqwest = { version = "0.11.18", features = ["json", "gzip"] }
 
 
 [dependencies.db-core]

Makefile

@@ -4,7 +4,7 @@
 
 BUNDLE = static/cache/bundle
 OPENAPI = docs/openapi
-CLEAN_UP = $(BUNDLE) src/cache_buster_data.json assets
+CLEAN_UP = $(BUNDLE) src/libcachebust_data.json assets
 
 define deploy_dependencies ## deploy dependencies
 	@-docker create --name ${db}  \

config/default.toml

@@ -66,3 +66,8 @@ url = "127.0.0.1"
 port = 10025
 username = "admin"
 password = "password"
+
+#[survey]
+#nodes = ["http://localhost:7001"]
+#rate_limit = 10 # upload every hour
+#instance_root_url = "http://localhost:7000"

db/db-core/Cargo.toml

@@ -13,8 +13,7 @@ async-trait = "0.1.51"
 thiserror = "1.0.30"
 serde = { version = "1", features = ["derive"]}
 url = { version  = "2.2.2", features = ["serde"] }
-libmcaptcha = {  version = "0.2.3", git = "https://github.com/mCaptcha/libmcaptcha", features = ["minimal"], default-features = false, tag = "0.2.3"}
+libmcaptcha = "0.2.4"
-#libmcaptcha = { branch = "master", git = "https://github.com/mCaptcha/libmcaptcha", features = ["full"] }
 
 [features]
 default = []

db/db-core/src/lib.rs

@@ -289,6 +289,9 @@ pub trait MCDatabase: std::marker::Send + std::marker::Sync + CloneSPDatabase {
             Err(e) => Err(e),
         }
     }
+
+    /// Get all psuedo IDs
+    async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult<Vec<String>>;
 }
 
 #[derive(Debug, Clone, Default, Deserialize, Serialize, PartialEq)]

db/db-core/src/tests.rs

@@ -258,6 +258,12 @@ pub async fn database_works<'a, T: MCDatabase>(
         .analytics_get_psuedo_id_from_capmaign_id(c.key)
         .await
         .unwrap();
+    assert_eq!(
+        vec![psuedo_id.clone()],
+        db.analytics_get_all_psuedo_ids(0).await.unwrap()
+    );
+    assert!(db.analytics_get_all_psuedo_ids(1).await.unwrap().is_empty());
+
     db.analytics_create_psuedo_id_if_not_exists(c.key)
         .await
         .unwrap();
@@ -267,6 +273,7 @@ pub async fn database_works<'a, T: MCDatabase>(
             .await
             .unwrap()
     );
+
     assert_eq!(
         c.key,
         db.analytics_get_capmaign_id_from_psuedo_id(&psuedo_id)

db/db-sqlx-maria/.sqlx/query-e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46.json

@@ -0,0 +1,25 @@
+{
+  "db_name": "MySQL",
+  "query": "\n                SELECT\n                    psuedo_id\n                FROM\n                    mcaptcha_psuedo_campaign_id\n                    ORDER BY ID ASC LIMIT ? OFFSET ?;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "psuedo_id",
+        "type_info": {
+          "type": "VarString",
+          "flags": "NOT_NULL | UNIQUE_KEY | NO_DEFAULT_VALUE",
+          "char_set": 224,
+          "max_size": 400
+        }
+      }
+    ],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "e2c30dafa790b388a193ad8785c0a7d88d8e7a7558775e238fe009f478003e46"
+}

db/db-sqlx-maria/src/lib.rs

@@ -987,12 +987,8 @@ impl MCDatabase for Database {
         &self,
         captcha_id: &str,
     ) -> DBResult<String> {
-        struct ID {
-            psuedo_id: String,
-        }
-
         let res = sqlx::query_as!(
-            ID,
+            PsuedoID,
             "SELECT psuedo_id FROM
                 mcaptcha_psuedo_campaign_id
             WHERE
@@ -1069,6 +1065,28 @@ impl MCDatabase for Database {
 
         Ok(())
     }
+    /// Get all psuedo IDs
+    async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult<Vec<String>> {
+        const LIMIT: usize = 50;
+        let offset = LIMIT * page;
+
+        let mut res = sqlx::query_as!(
+            PsuedoID,
+            "
+                SELECT
+                    psuedo_id
+                FROM
+                    mcaptcha_psuedo_campaign_id
+                    ORDER BY ID ASC LIMIT ? OFFSET ?;",
+            LIMIT as i64,
+            offset as i64
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        Ok(res.drain(0..).map(|r| r.psuedo_id).collect())
+    }
 }
 
 #[derive(Clone)]
@@ -1134,3 +1152,7 @@ impl From<InternaleCaptchaConfig> for Captcha {
         }
     }
 }
+
+struct PsuedoID {
+    psuedo_id: String,
+}

db/db-sqlx-postgres/.sqlx/query-d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7.json

@@ -0,0 +1,23 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                SELECT\n                    psuedo_id\n                FROM\n                    mcaptcha_psuedo_campaign_id\n                    ORDER BY ID ASC LIMIT $1 OFFSET $2;",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "psuedo_id",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8",
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "d6b89b032e3a65bb5739dde8901a0d6363939bdd87739b4292dd1d88e03ce6f7"
+}

db/db-sqlx-postgres/src/lib.rs

@@ -994,12 +994,8 @@ impl MCDatabase for Database {
         &self,
         captcha_id: &str,
     ) -> DBResult<String> {
-        struct ID {
-            psuedo_id: String,
-        }
-
         let res = sqlx::query_as!(
-            ID,
+            PsuedoID,
             "SELECT psuedo_id FROM
                 mcaptcha_psuedo_campaign_id
             WHERE
@@ -1078,6 +1074,29 @@ impl MCDatabase for Database {
 
         Ok(())
     }
+
+    /// Get all psuedo IDs
+    async fn analytics_get_all_psuedo_ids(&self, page: usize) -> DBResult<Vec<String>> {
+        const LIMIT: usize = 50;
+        let offset = LIMIT * page;
+
+        let mut res = sqlx::query_as!(
+            PsuedoID,
+            "
+                SELECT
+                    psuedo_id
+                FROM
+                    mcaptcha_psuedo_campaign_id
+                    ORDER BY ID ASC LIMIT $1 OFFSET $2;",
+            LIMIT as i64,
+            offset as i64
+        )
+        .fetch_all(&self.pool)
+        .await
+        .map_err(|e| map_row_not_found_err(e, DBError::CaptchaNotFound))?;
+
+        Ok(res.drain(0..).map(|r| r.psuedo_id).collect())
+    }
 }
 
 #[derive(Clone)]
@@ -1125,6 +1144,10 @@ impl From<InnerNotification> for Notification {
     }
 }
 
+struct PsuedoID {
+    psuedo_id: String,
+}
+
 #[derive(Clone)]
 struct InternaleCaptchaConfig {
     config_id: i32,

docker-compose.yml

@@ -11,8 +11,8 @@ services:
       - 7000:7000
     environment:
       DATABASE_URL: postgres://postgres:password@mcaptcha_postgres:5432/postgres # set password at placeholder
-      MCAPTCHA_REDIS_URL: redis://mcaptcha_redis/
+      MCAPTCHA_redis_URL: "redis://mcaptcha_redis/"
-      RUST_LOG: debug
+      RUST_LOG: "debug"
       PORT: 7000
     depends_on:
       - mcaptcha_postgres

docs/openapi/package.json

@@ -11,7 +11,7 @@
     "type": "git",
     "url": "git+https://github.com/mCaptcha/mCaptcha.git"
   },
-  "license": "AGPL3",
+  "license": "AGPL-3.0-or-later",
   "bugs": {
     "url": "https://github.com/mCaptcha/mCaptcha/issues"
   },

src/api/v1/mcaptcha/create.rs

@@ -85,10 +85,18 @@ pub mod runner {
         data.db
             .add_captcha_levels(username, &key, &payload.levels)
             .await?;
+
+        if payload.publish_benchmarks {
+            data.db
+                .analytics_create_psuedo_id_if_not_exists(&key)
+                .await?;
+        }
+
         let mcaptcha_config = MCaptchaDetails {
             name: payload.description.clone(),
             key,
         };
+
         Ok(mcaptcha_config)
     }
 }

src/api/v1/mod.rs

@@ -14,6 +14,7 @@ pub mod meta;
 pub mod notifications;
 pub mod pow;
 mod routes;
+pub mod survey;
 
 pub use routes::ROUTES;
 
@@ -24,6 +25,7 @@ pub fn services(cfg: &mut ServiceConfig) {
     account::services(cfg);
     mcaptcha::services(cfg);
     notifications::services(cfg);
+    survey::services(cfg);
 }
 
 #[derive(Deserialize)]

src/api/v1/routes.rs

@@ -11,6 +11,7 @@ use super::mcaptcha::routes::Captcha;
 use super::meta::routes::Meta;
 use super::notifications::routes::Notifications;
 use super::pow::routes::PoW;
+use super::survey::routes::Survey;
 
 pub const ROUTES: Routes = Routes::new();
 
@@ -20,6 +21,7 @@ pub struct Routes {
     pub captcha: Captcha,
     pub meta: Meta,
     pub pow: PoW,
+    pub survey: Survey,
     pub notifications: Notifications,
 }
 
@@ -32,6 +34,7 @@ impl Routes {
             meta: Meta::new(),
             pow: PoW::new(),
             notifications: Notifications::new(),
+            survey: Survey::new(),
         }
     }
 }

src/api/v1/survey.rs

@@ -0,0 +1,255 @@
+/*
+ * Copyright (C) 2023  Aravinth Manivannan <realaravinth@batsense.net>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+use actix_web::web::ServiceConfig;
+use actix_web::{web, HttpResponse, Responder};
+use serde::{Deserialize, Serialize};
+
+use crate::errors::*;
+use crate::AppData;
+
+pub fn services(cfg: &mut ServiceConfig) {
+    cfg.service(download);
+    cfg.service(secret);
+}
+
+pub mod routes {
+    pub struct Survey {
+        pub download: &'static str,
+        pub secret: &'static str,
+    }
+
+    impl Survey {
+        pub const fn new() -> Self {
+            Self {
+                download: "/api/v1/survey/takeout/{survey_id}/get",
+                secret: "/api/v1/survey/secret",
+            }
+        }
+
+        pub fn get_download_route(&self, survey_id: &str, page: usize) -> String {
+            format!(
+                "{}?page={}",
+                self.download.replace("{survey_id}", survey_id),
+                page
+            )
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)]
+pub struct Page {
+    pub page: usize,
+}
+
+/// emits build details of the bninary
+#[my_codegen::get(path = "crate::V1_API_ROUTES.survey.download")]
+async fn download(
+    data: AppData,
+    page: web::Query<Page>,
+    psuedo_id: web::Path<uuid::Uuid>,
+) -> ServiceResult<impl Responder> {
+    const LIMIT: usize = 50;
+    let offset = LIMIT as isize * ((page.page as isize) - 1);
+    let offset = if offset < 0 { 0 } else { offset };
+    let psuedo_id = psuedo_id.into_inner();
+    let campaign_id = data
+        .db
+        .analytics_get_capmaign_id_from_psuedo_id(&psuedo_id.to_string())
+        .await?;
+    let data = data
+        .db
+        .analytics_fetch(&campaign_id, LIMIT, offset as usize)
+        .await?;
+    Ok(HttpResponse::Ok().json(data))
+}
+
+#[derive(Serialize, Deserialize)]
+struct SurveySecretUpload {
+    secret: String,
+    auth_token: String,
+}
+
+/// mCaptcha/survey upload secret route
+#[my_codegen::post(path = "crate::V1_API_ROUTES.survey.secret")]
+async fn secret(
+    data: AppData,
+    payload: web::Json<SurveySecretUpload>,
+) -> ServiceResult<impl Responder> {
+    match data.survey_secrets.get(&payload.auth_token) {
+        Some(survey_instance_url) => {
+            let payload = payload.into_inner();
+            data.survey_secrets.set(survey_instance_url, payload.secret);
+            data.survey_secrets.rm(&payload.auth_token);
+            Ok(HttpResponse::Ok())
+        }
+        None => Err(ServiceError::WrongPassword),
+    }
+}
+
+#[cfg(test)]
+pub mod tests {
+    use actix_web::{http::StatusCode, test, App};
+
+    use super::*;
+    use crate::api::v1::mcaptcha::get_random;
+    use crate::tests::*;
+    use crate::*;
+
+    #[actix_rt::test]
+    async fn survey_works_pg() {
+        let data = crate::tests::pg::get_data().await;
+        survey_registration_works(data.clone()).await;
+        survey_works(data).await;
+    }
+
+    #[actix_rt::test]
+    async fn survey_works_maria() {
+        let data = crate::tests::maria::get_data().await;
+        survey_registration_works(data.clone()).await;
+        survey_works(data).await;
+    }
+
+    pub async fn survey_registration_works(data: ArcData) {
+        let data = &data;
+        let app = get_app!(data).await;
+
+        let survey_instance_url = "http://survey_registration_works.survey.example.org";
+
+        let key = get_random(20);
+
+        let msg = SurveySecretUpload {
+            auth_token: key.clone(),
+            secret: get_random(32),
+        };
+
+        // should fail with ServiceError::WrongPassword since auth token is not loaded into
+        // keystore
+        bad_post_req_test_no_auth(
+            data,
+            V1_API_ROUTES.survey.secret,
+            &msg,
+            errors::ServiceError::WrongPassword,
+        )
+        .await;
+
+        // load auth token into key store, should succeed
+        data.survey_secrets
+            .set(key.clone(), survey_instance_url.to_owned());
+        let resp = test::call_service(
+            &app,
+            post_request!(&msg, V1_API_ROUTES.survey.secret).to_request(),
+        )
+        .await;
+        assert_eq!(resp.status(), StatusCode::OK);
+        // uploaded secret must be in keystore
+        assert_eq!(
+            data.survey_secrets.get(survey_instance_url).unwrap(),
+            msg.secret
+        );
+
+        // should fail since mCaptcha/survey secret upload auth tokens are single-use
+        bad_post_req_test_no_auth(
+            data,
+            V1_API_ROUTES.survey.secret,
+            &msg,
+            errors::ServiceError::WrongPassword,
+        )
+        .await;
+    }
+
+    pub async fn survey_works(data: ArcData) {
+        const NAME: &str = "survetuseranalytics";
+        const PASSWORD: &str = "longpassworddomain";
+        const EMAIL: &str = "survetuseranalytics@a.com";
+        let data = &data;
+
+        delete_user(data, NAME).await;
+
+        register_and_signin(data, NAME, EMAIL, PASSWORD).await;
+        // create captcha
+        let (_, _signin_resp, key) = add_levels_util(data, NAME, PASSWORD).await;
+        let app = get_app!(data).await;
+
+        let page = 1;
+        let tmp_id = uuid::Uuid::new_v4();
+        let download_rotue = V1_API_ROUTES
+            .survey
+            .get_download_route(&tmp_id.to_string(), page);
+
+        let download_req = test::call_service(
+            &app,
+            test::TestRequest::get().uri(&download_rotue).to_request(),
+        )
+        .await;
+        assert_eq!(download_req.status(), StatusCode::NOT_FOUND);
+
+        data.db
+            .analytics_create_psuedo_id_if_not_exists(&key.key)
+            .await
+            .unwrap();
+
+        let psuedo_id = data
+            .db
+            .analytics_get_psuedo_id_from_capmaign_id(&key.key)
+            .await
+            .unwrap();
+
+        for i in 0..60 {
+            println!("[{i}] Saving analytics");
+            let analytics = db_core::CreatePerformanceAnalytics {
+                time: 0,
+                difficulty_factor: 0,
+                worker_type: "wasm".into(),
+            };
+            data.db.analysis_save(&key.key, &analytics).await.unwrap();
+        }
+
+        for p in 1..3 {
+            let download_rotue = V1_API_ROUTES.survey.get_download_route(&psuedo_id, p);
+            println!("page={p}, download={download_rotue}");
+
+            let download_req = test::call_service(
+                &app,
+                test::TestRequest::get().uri(&download_rotue).to_request(),
+            )
+            .await;
+            assert_eq!(download_req.status(), StatusCode::OK);
+            let analytics: Vec<db_core::PerformanceAnalytics> =
+                test::read_body_json(download_req).await;
+            if p == 1 {
+                assert_eq!(analytics.len(), 50);
+            } else if p == 2 {
+                assert_eq!(analytics.len(), 10);
+            } else {
+                assert_eq!(analytics.len(), 0);
+            }
+        }
+
+        let download_rotue = V1_API_ROUTES.survey.get_download_route(&psuedo_id, 0);
+        data.db
+            .analytics_delete_all_records_for_campaign(&key.key)
+            .await
+            .unwrap();
+
+        let download_req = test::call_service(
+            &app,
+            test::TestRequest::get().uri(&download_rotue).to_request(),
+        )
+        .await;
+        assert_eq!(download_req.status(), StatusCode::NOT_FOUND);
+    }
+}

src/data.rs

@@ -4,8 +4,10 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
 //! App data: redis cache, database connections, etc.
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::thread;
+use std::time::Duration;
 
 use actix::prelude::*;
 use argon2_creds::{Config, ConfigBuilder, PasswordPolicy};
@@ -28,11 +30,17 @@ use libmcaptcha::{
     pow::Work,
     system::{System, SystemBuilder},
 };
+use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use tokio::task::JoinHandle;
+use tokio::time::sleep;
 
 use crate::db::{self, BoxDB};
 use crate::errors::ServiceResult;
 use crate::settings::Settings;
 use crate::stats::{Dummy, Real, Stats};
+use crate::survey::SecretsStore;
+use crate::AppData;
 
 macro_rules! enum_system_actor {
     ($name:ident, $type:ident) => {
@@ -166,6 +174,8 @@ pub struct Data {
     pub settings: Settings,
     /// stats recorder
     pub stats: Box<dyn Stats>,
+    /// survey secret store
+    pub survey_secrets: SecretsStore,
 }
 
 impl Data {
@@ -180,7 +190,7 @@ impl Data {
     }
     #[cfg(not(tarpaulin_include))]
     /// create new instance of app data
-    pub async fn new(s: &Settings) -> Arc<Self> {
+    pub async fn new(s: &Settings, survey_secrets: SecretsStore) -> Arc<Self> {
         let creds = Self::get_creds();
         let c = creds.clone();
 
@@ -209,6 +219,7 @@ impl Data {
             mailer: Self::get_mailer(s),
             settings: s.clone(),
             stats,
+            survey_secrets,
         };
 
         #[cfg(not(debug_assertions))]
@@ -242,6 +253,13 @@ impl Data {
             None
         }
     }
+
+    async fn upload_survey_job(&self) -> ServiceResult<()> {
+        unimplemented!()
+    }
+    async fn register_survey(&self) -> ServiceResult<()> {
+        unimplemented!()
+    }
 }
 
 /// Mailer data type AsyncSmtpTransport<Tokio1Executor>

src/date.rs

@@ -52,7 +52,12 @@ impl Date {
 
     /// print date
     pub fn date(&self) -> String {
-            format!("{}{}{}", self.time.year(), self.time.month(), self.time.date())
+        format!(
+            "{}{}{}",
+            self.time.year(),
+            self.time.month(),
+            self.time.date()
+        )
     }
 
     pub fn new(unix: i64) -> Self {

src/main.rs

@@ -30,6 +30,7 @@ mod routes;
 mod settings;
 mod static_assets;
 mod stats;
+mod survey;
 #[cfg(test)]
 #[macro_use]
 mod tests;
@@ -45,6 +46,7 @@ use static_assets::FileMap;
 pub use widget::WIDGET_ROUTES;
 
 use crate::demo::DemoUser;
+use survey::SurveyClientTrait;
 
 lazy_static! {
     pub static ref SETTINGS: Settings = Settings::new().unwrap();
@@ -93,7 +95,9 @@ pub type AppData = actix_web::web::Data<ArcData>;
 async fn main() -> std::io::Result<()> {
     use std::time::Duration;
 
-    env::set_var("RUST_LOG", "info");
+    if env::var("RUST_LOG").is_err() {
+        env::set_var("RUST_LOG", "info");
+    }
 
     pretty_env_logger::init();
     info!(
@@ -102,7 +106,8 @@ async fn main() -> std::io::Result<()> {
     );
 
     let settings = Settings::new().unwrap();
-    let data = Data::new(&settings).await;
+    let secrets = survey::SecretsStore::default();
+    let data = Data::new(&settings, secrets.clone()).await;
     let data = actix_web::web::Data::new(data);
 
     let mut demo_user: Option<DemoUser> = None;
@@ -115,6 +120,13 @@ async fn main() -> std::io::Result<()> {
         );
     }
 
+    let (mut survey_upload_tx, mut survey_upload_handle) = (None, None);
+    if settings.survey.is_some() {
+        let survey_runner_ctx = survey::Survey::new(data.clone());
+        let (x, y) = survey_runner_ctx.start_job().await.unwrap();
+        (survey_upload_tx, survey_upload_handle) = (Some(x), Some(y));
+    }
+
     let ip = settings.server.get_ip();
     println!("Starting server on: http://{ip}");
 
@@ -139,9 +151,18 @@ async fn main() -> std::io::Result<()> {
     .run()
     .await?;
 
+    if let Some(survey_upload_tx) = survey_upload_tx {
+        survey_upload_tx.send(()).unwrap();
+    }
+
     if let Some(demo_user) = demo_user {
         demo_user.abort();
     }
+
+    if let Some(survey_upload_handle) = survey_upload_handle {
+        survey_upload_handle.await.unwrap();
+    }
+
     Ok(())
 }
 

src/pages/mod.rs

@@ -63,7 +63,8 @@ mod tests {
 
         let edit_sitekey_url = PAGES.panel.sitekey.get_edit_advance(&token_key.key);
         let delete_sitekey_url = PAGES.panel.sitekey.get_delete(&token_key.key);
-        let urls = [PAGES.home,
+        let urls = [
+            PAGES.home,
             PAGES.panel.sitekey.add_advance,
             PAGES.panel.sitekey.add_easy,
             PAGES.panel.sitekey.list,
@@ -72,7 +73,8 @@ mod tests {
             PAGES.panel.settings.delete_account,
             PAGES.panel.settings.update_secret,
             &delete_sitekey_url,
-            &edit_sitekey_url];
+            &edit_sitekey_url,
+        ];
 
         for url in urls.iter() {
             let resp =

src/pages/panel/notifications.rs

@@ -95,9 +95,11 @@ mod tests {
         assert!(n.print_date().contains("seconds ago"));
 
         // minutes test
-        n.received = OffsetDateTime::from_unix_timestamp(timestamp - MINUTE * 2).unwrap();
+        n.received =
+            OffsetDateTime::from_unix_timestamp(timestamp - MINUTE * 2).unwrap();
         assert!(n.print_date().contains("minutes ago"));
-        n.received = OffsetDateTime::from_unix_timestamp(timestamp - MINUTE * 56).unwrap();
+        n.received =
+            OffsetDateTime::from_unix_timestamp(timestamp - MINUTE * 56).unwrap();
         assert!(n.print_date().contains("minutes ago"));
 
         // hours test
@@ -112,7 +114,12 @@ mod tests {
 
         // date test
         n.received = OffsetDateTime::from_unix_timestamp(timestamp - 6 * WEEK).unwrap();
-        let date = format!("{}{}{}", n.received.year(), n.received.month(), n.received.date());
+        let date = format!(
+            "{}{}{}",
+            n.received.year(),
+            n.received.month(),
+            n.received.date()
+        );
         assert!(n.print_date().contains(&date))
     }
 }

src/settings.rs

@@ -91,6 +91,13 @@ pub struct Redis {
     pub pool: u32,
 }
 
+#[derive(Debug, Clone, Deserialize, Eq, PartialEq)]
+pub struct Survey {
+    pub nodes: Vec<url::Url>,
+    pub rate_limit: u64,
+    pub instance_root_url: Url,
+}
+
 #[derive(Debug, Clone, Deserialize, Eq, PartialEq)]
 pub struct Settings {
     pub debug: bool,
@@ -99,6 +106,7 @@ pub struct Settings {
     pub allow_registration: bool,
     pub allow_demo: bool,
     pub database: Database,
+    pub survey: Option<Survey>,
     pub redis: Option<Redis>,
     pub server: Server,
     pub captcha: Captcha,
@@ -118,8 +126,8 @@ const ENV_VAR_CONFIG: [(&str, &str); 29] = [
     ("database.pool", "MCAPTCHA_database_POOL"),
 
     /* redis */
-    ("redis.url", "MCPATCHA_redis_URL"),
+    ("redis.url", "MCAPTCHA_redis_URL"),
-    ("redis.pool", "MCPATCHA_redis_POOL"),
+    ("redis.pool", "MCAPTCHA_redis_POOL"),
 
     /* server */
     ("server.port", "PORT"),
@@ -145,17 +153,52 @@ const ENV_VAR_CONFIG: [(&str, &str); 29] = [
 
 
     /* SMTP */
-    ("smtp.from", "MCPATCHA_smtp_FROM"),
+    ("smtp.from", "MCAPTCHA_smtp_FROM"),
-    ("smtp.reply", "MCPATCHA_smtp_REPLY"),
+    ("smtp.reply", "MCAPTCHA_smtp_REPLY"),
-    ("smtp.url", "MCPATCHA_smtp_URL"),
+    ("smtp.url", "MCAPTCHA_smtp_URL"),
-    ("smtp.username", "MCPATCHA_smtp_USERNAME"),
+    ("smtp.username", "MCAPTCHA_smtp_USERNAME"),
-    ("smtp.password", "MCPATCHA_smtp_PASSWORD"),
+    ("smtp.password", "MCAPTCHA_smtp_PASSWORD"),
-    ("smtp.port", "MCPATCHA_smtp_PORT"),
+    ("smtp.port", "MCAPTCHA_smtp_PORT"),
 
 
 
 ];
 
+const DEPRECATED_ENV_VARS: [(&str, &str); 23] = [
+    ("debug", "MCAPTCHA_DEBUG"),
+    ("commercial", "MCAPTCHA_COMMERCIAL"),
+    ("source_code", "MCAPTCHA_SOURCE_CODE"),
+    ("allow_registration", "MCAPTCHA_ALLOW_REGISTRATION"),
+    ("allow_demo", "MCAPTCHA_ALLOW_DEMO"),
+    ("redis.pool", "MCAPTCHA_REDIS_POOL"),
+    ("redis.url", "MCAPTCHA_REDIS_URL"),
+    ("server.port", "MCAPTCHA_SERVER_PORT"),
+    ("server.ip", "MCAPTCHA_SERVER_IP"),
+    ("server.domain", "MCAPTCHA_SERVER_DOMAIN"),
+    ("server.cookie_secret", "MCAPTCHA_SERVER_COOKIE_SECRET"),
+    ("server.proxy_has_tls", "MCAPTCHA_SERVER_PROXY_HAS_TLS"),
+    ("captcha.salt", "MCAPTCHA_CAPTCHA_SALT"),
+    ("captcha.gc", "MCAPTCHA_CAPTCHA_GC"),
+    (
+        "captcha.default_difficulty_strategy.avg_traffic_difficulty",
+        "MCAPTCHA_CAPTCHA_AVG_TRAFFIC_DIFFICULTY",
+    ),
+    (
+        "captcha.default_difficulty_strategy.peak_sustainable_traffic_difficulty",
+        "MCAPTCHA_CAPTCHA_PEAK_TRAFFIC_DIFFICULTY",
+    ),
+    (
+        "captcha.default_difficulty_strategy.broke_my_site_traffic_difficulty",
+        "MCAPTCHA_CAPTCHA_BROKE_MY_SITE_TRAFFIC",
+    ),
+    ("smtp.from", "MCAPTCHA_SMTP_FROM"),
+    ("smtp.reply", "MCAPTCHA_SMTP_REPLY_TO"),
+    ("smtp.url", "MCAPTCHA_SMTP_URL"),
+    ("smtp.username", "MCAPTCHA_SMTP_USERNAME"),
+    ("smtp.password", "MCAPTCHA_SMTP_PASSWORD"),
+    ("smtp.port", "MCAPTCHA_SMTP_PORT"),
+];
+
 #[cfg(not(tarpaulin_include))]
 impl Settings {
     pub fn new() -> Result<Self, ConfigError> {
@@ -210,6 +253,15 @@ impl Settings {
     }
 
     fn env_override(mut s: ConfigBuilder<DefaultState>) -> ConfigBuilder<DefaultState> {
+        for (parameter, env_var_name) in DEPRECATED_ENV_VARS.iter() {
+            if let Ok(val) = env::var(env_var_name) {
+                log::warn!(
+                    "Found {env_var_name}. {env_var_name} will be deprecated soon. Please see https://github.com/mCaptcha/mCaptcha/blob/master/docs/CONFIGURATION.md for latest environment variable names"
+                );
+                s = s.set_override(parameter, val).unwrap();
+            }
+        }
+
         for (parameter, env_var_name) in ENV_VAR_CONFIG.iter() {
             if let Ok(val) = env::var(env_var_name) {
                 log::debug!(
@@ -240,7 +292,7 @@ mod tests {
     use super::*;
 
     #[test]
-    fn env_override_works() {
+    fn deprecated_env_override_works() {
         use crate::tests::get_settings;
         let init_settings = get_settings();
         // so that it can be tested outside the macro (helper) too
@@ -249,6 +301,141 @@ mod tests {
         macro_rules! helper {
 
 
+            ($env:expr, $val:expr, $val_typed:expr, $($param:ident).+) => {
+                println!("Setting env var {} to {} for test", $env, $val);
+                env::set_var($env, $val);
+                new_settings = get_settings();
+                assert_eq!(new_settings.$($param).+, $val_typed);
+                assert_ne!(new_settings.$($param).+, init_settings.$($param).+);
+                env::remove_var($env);
+            };
+
+
+            ($env:expr, $val:expr, $($param:ident).+) => {
+                helper!($env, $val.to_string(), $val, $($param).+);
+            };
+        }
+
+        /* top level */
+        helper!("MCAPTCHA_DEBUG", !init_settings.debug, debug);
+        helper!("MCAPTCHA_COMMERCIAL", !init_settings.commercial, commercial);
+        helper!(
+            "MCAPTCHA_ALLOW_REGISTRATION",
+            !init_settings.allow_registration,
+            allow_registration
+        );
+        helper!("MCAPTCHA_ALLOW_DEMO", !init_settings.allow_demo, allow_demo);
+
+        /* database_type */
+
+        /* redis.url */
+        let env = "MCAPTCHA_REDIS_URL";
+        let val = "redis://redis.example.org";
+        println!("Setting env var {} to {} for test", env, val);
+        env::set_var(env, val);
+        new_settings = get_settings();
+        assert_eq!(new_settings.redis.as_ref().unwrap().url, val);
+        assert_ne!(
+            new_settings.redis.as_ref().unwrap().url,
+            init_settings.redis.as_ref().unwrap().url
+        );
+        env::remove_var(env);
+
+        /* redis.pool */
+        let env = "MCAPTCHA_REDIS_POOL";
+        let val = 999;
+        println!("Setting env var {} to {} for test", env, val);
+        env::set_var(env, val.to_string());
+        new_settings = get_settings();
+        assert_eq!(new_settings.redis.as_ref().unwrap().pool, val);
+        assert_ne!(
+            new_settings.redis.as_ref().unwrap().pool,
+            init_settings.redis.as_ref().unwrap().pool
+        );
+        env::remove_var(env);
+
+        helper!("PORT", 0, server.port);
+        helper!("MCAPTCHA_SERVER_DOMAIN", "example.org", server.domain);
+        helper!(
+            "MCAPTCHA_SERVER_COOKIE_SECRET",
+            "dafasdfsdf",
+            server.cookie_secret
+        );
+        helper!("MCAPTCHA_SERVER_IP", "9.9.9.9", server.ip);
+        helper!("MCAPTCHA_SERVER_PROXY_HAS_TLS", true, server.proxy_has_tls);
+
+        /* captcha */
+
+        helper!("MCAPTCHA_CAPTCHA_SALT", "foobarasdfasdf", captcha.salt);
+        helper!("MCAPTCHA_CAPTCHA_GC", 500, captcha.gc);
+        helper!(
+            "MCAPTCHA_captcha_RUNNERS",
+            "500",
+            Some(500),
+            captcha.runners
+        );
+
+        helper!(
+            "MCAPTCHA_CAPTCHA_AVG_TRAFFIC_DIFFICULTY",
+            999,
+            captcha.default_difficulty_strategy.avg_traffic_difficulty
+        );
+        helper!(
+            "MCAPTCHA_CAPTCHA_PEAK_TRAFFIC_DIFFICULTY",
+            999,
+            captcha
+                .default_difficulty_strategy
+                .peak_sustainable_traffic_difficulty
+        );
+        helper!(
+            "MCAPTCHA_CAPTCHA_BROKE_MY_SITE_TRAFFIC",
+            999,
+            captcha
+                .default_difficulty_strategy
+                .broke_my_site_traffic_difficulty
+        );
+
+        /* SMTP */
+
+        let vals = [
+            "MCAPTCHA_SMTP_FROM",
+            "MCAPTCHA_SMTP_REPLY_TO",
+            "MCAPTCHA_SMTP_URL",
+            "MCAPTCHA_SMTP_USERNAME",
+            "MCAPTCHA_SMTP_PASSWORD",
+            "MCAPTCHA_SMTP_PORT",
+        ];
+        for env in vals.iter() {
+            println!("Setting env var {} to {} for test", env, env);
+            env::set_var(env, env);
+        }
+
+        let port = 9999;
+        env::set_var("MCAPTCHA_SMTP_PORT", port.to_string());
+
+        new_settings = get_settings();
+        let smtp_new = new_settings.smtp.as_ref().unwrap();
+        let smtp_old = init_settings.smtp.as_ref().unwrap();
+        assert_eq!(smtp_new.from, "MCAPTCHA_SMTP_FROM");
+        assert_eq!(smtp_new.reply, "MCAPTCHA_SMTP_REPLY_TO");
+        assert_eq!(smtp_new.username, "MCAPTCHA_SMTP_USERNAME");
+        assert_eq!(smtp_new.password, "MCAPTCHA_SMTP_PASSWORD");
+        assert_eq!(smtp_new.port, port);
+        assert_ne!(smtp_new, smtp_old);
+
+        for env in vals.iter() {
+            env::remove_var(env);
+        }
+    }
+
+    #[test]
+    fn env_override_works() {
+        use crate::tests::get_settings;
+        let init_settings = get_settings();
+        // so that it can be tested outside the macro (helper) too
+        let mut new_settings;
+
+        macro_rules! helper {
 
 
             ($env:expr, $val:expr, $val_typed:expr, $($param:ident).+) => {
@@ -291,7 +478,7 @@ mod tests {
         /* redis */
 
         /* redis.url */
-        let env = "MCPATCHA_redis_URL";
+        let env = "MCAPTCHA_redis_URL";
         let val = "redis://redis.example.org";
         println!("Setting env var {} to {} for test", env, val);
         env::set_var(env, val);
@@ -304,7 +491,7 @@ mod tests {
         env::remove_var(env);
 
         /* redis.pool */
-        let env = "MCPATCHA_redis_POOL";
+        let env = "MCAPTCHA_redis_POOL";
         let val = 999;
         println!("Setting env var {} to {} for test", env, val);
         env::set_var(env, val.to_string());
@@ -355,12 +542,12 @@ mod tests {
         /* SMTP */
 
         let vals = [
-            "MCPATCHA_smtp_FROM",
+            "MCAPTCHA_smtp_FROM",
-            "MCPATCHA_smtp_REPLY",
+            "MCAPTCHA_smtp_REPLY",
-            "MCPATCHA_smtp_URL",
+            "MCAPTCHA_smtp_URL",
-            "MCPATCHA_smtp_USERNAME",
+            "MCAPTCHA_smtp_USERNAME",
-            "MCPATCHA_smtp_PASSWORD",
+            "MCAPTCHA_smtp_PASSWORD",
-            "MCPATCHA_smtp_PORT",
+            "MCAPTCHA_smtp_PORT",
         ];
         for env in vals.iter() {
             println!("Setting env var {} to {} for test", env, env);
@@ -368,15 +555,15 @@ mod tests {
         }
 
         let port = 9999;
-        env::set_var("MCPATCHA_smtp_PORT", port.to_string());
+        env::set_var("MCAPTCHA_smtp_PORT", port.to_string());
 
         new_settings = get_settings();
         let smtp_new = new_settings.smtp.as_ref().unwrap();
         let smtp_old = init_settings.smtp.as_ref().unwrap();
-        assert_eq!(smtp_new.from, "MCPATCHA_smtp_FROM");
+        assert_eq!(smtp_new.from, "MCAPTCHA_smtp_FROM");
-        assert_eq!(smtp_new.reply, "MCPATCHA_smtp_REPLY");
+        assert_eq!(smtp_new.reply, "MCAPTCHA_smtp_REPLY");
-        assert_eq!(smtp_new.username, "MCPATCHA_smtp_USERNAME");
+        assert_eq!(smtp_new.username, "MCAPTCHA_smtp_USERNAME");
-        assert_eq!(smtp_new.password, "MCPATCHA_smtp_PASSWORD");
+        assert_eq!(smtp_new.password, "MCAPTCHA_smtp_PASSWORD");
         assert_eq!(smtp_new.port, port);
         assert_ne!(smtp_new, smtp_old);
 

src/static_assets/filemap.rs

@@ -3,7 +3,7 @@
 //
 // SPDX-License-Identifier: AGPL-3.0-or-later
 
-use cache_buster::Files;
+use libcachebust::Files;
 
 pub struct FileMap {
     pub files: Files,
@@ -12,7 +12,7 @@ pub struct FileMap {
 impl FileMap {
     #[allow(clippy::new_without_default)]
     pub fn new() -> Self {
-        let map = include_str!("../cache_buster_data.json");
+        let map = include_str!("../libcachebust_data.json");
         let files = Files::new(map);
         Self { files }
     }

src/survey.rs

@@ -0,0 +1,209 @@
+// Copyright (C) 2022  Aravinth Manivannan <realaravinth@batsense.net>
+// SPDX-FileCopyrightText: 2023 Aravinth Manivannan <realaravinth@batsense.net>
+//
+// SPDX-License-Identifier: AGPL-3.0-or-later
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::sync::RwLock;
+use std::time::Duration;
+
+use reqwest::Client;
+use serde::{Deserialize, Serialize};
+use tokio::sync::oneshot;
+use tokio::task::JoinHandle;
+use tokio::time::sleep;
+
+use crate::errors::*;
+use crate::settings::Settings;
+use crate::AppData;
+use crate::V1_API_ROUTES;
+
+#[async_trait::async_trait]
+pub trait SurveyClientTrait {
+    async fn start_job(&self) -> ServiceResult<(oneshot::Sender<()>, JoinHandle<()>)>;
+    async fn schedule_upload_job(&self) -> ServiceResult<()>;
+    async fn is_online(&self) -> ServiceResult<bool>;
+    async fn register(&self) -> ServiceResult<()>;
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct SecretsStore {
+    store: Arc<RwLock<HashMap<String, String>>>,
+}
+
+impl SecretsStore {
+    pub fn get(&self, key: &str) -> Option<String> {
+        let r = self.store.read().unwrap();
+        r.get(key).map(|x| x.to_owned())
+    }
+
+    pub fn rm(&self, key: &str) {
+        let mut w = self.store.write().unwrap();
+        w.remove(key);
+        drop(w);
+    }
+
+    pub fn set(&self, key: String, value: String) {
+        let mut w = self.store.write().unwrap();
+        w.insert(key, value);
+        drop(w);
+    }
+}
+
+#[derive(Clone)]
+pub struct Survey {
+    client: Client,
+    app_ctx: AppData,
+}
+impl Survey {
+    pub fn new(app_ctx: AppData) -> Self {
+        if app_ctx.settings.survey.is_none() {
+            panic!("Survey uploader shouldn't be initialized it isn't configured, please report this bug")
+        }
+        Survey {
+            client: Client::new(),
+            app_ctx,
+        }
+    }
+}
+
+#[async_trait::async_trait]
+impl SurveyClientTrait for Survey {
+    async fn start_job(&self) -> ServiceResult<(oneshot::Sender<()>, JoinHandle<()>)> {
+        fn can_run(rx: &mut oneshot::Receiver<()>) -> bool {
+            match rx.try_recv() {
+                Err(oneshot::error::TryRecvError::Empty) => true,
+                _ => false,
+            }
+        }
+
+        let (tx, mut rx) = oneshot::channel();
+        let this = self.clone();
+        let mut register = false;
+        let fut = async move {
+            loop {
+                if !can_run(&mut rx) {
+                    log::info!("Stopping survey uploads");
+                    break;
+                }
+
+                if !register {
+                    loop {
+                        if this.is_online().await.unwrap() {
+                            this.register().await.unwrap();
+                            register = true;
+                            break;
+                        } else {
+                            sleep(Duration::new(1, 0)).await;
+                        }
+                    }
+                }
+
+                for i in 0..this.app_ctx.settings.survey.as_ref().unwrap().rate_limit {
+                    if !can_run(&mut rx) {
+                        log::info!("Stopping survey uploads");
+                        break;
+                    }
+                    sleep(Duration::new(1, 0)).await;
+                }
+                let _ = this.schedule_upload_job().await;
+
+                // for url in this.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() {
+                //     if !can_run(&mut rx) {
+                //         log::info!("Stopping survey uploads");
+                //         break;
+                //     }
+                //     log::info!("Uploading to survey instance {}", url);
+                // }
+            }
+        };
+        let handle = tokio::spawn(fut);
+        Ok((tx, handle))
+    }
+    async fn is_online(&self) -> ServiceResult<bool> {
+        let res = self
+            .client
+            .get(format!(
+                "http://{}{}",
+                self.app_ctx.settings.server.get_ip(),
+                V1_API_ROUTES.meta.health
+            ))
+            .send()
+            .await
+            .unwrap();
+        Ok(res.status() == 200)
+    }
+
+    async fn schedule_upload_job(&self) -> ServiceResult<()> {
+        log::debug!("Running upload job");
+        #[derive(Serialize)]
+        struct Secret {
+            secret: String,
+        }
+        let mut page = 0;
+        loop {
+            let psuedo_ids = self.app_ctx.db.analytics_get_all_psuedo_ids(page).await?;
+            if psuedo_ids.is_empty() {
+                log::debug!("upload job complete, no more IDs to upload");
+                break;
+            }
+            for id in psuedo_ids {
+                for url in self.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() {
+                    if let Some(secret) = self.app_ctx.survey_secrets.get(url.as_str()) {
+                        let payload = Secret { secret };
+
+                        log::info!("Uploading to survey instance {} campaign {id}", url);
+                        let mut url = url.clone();
+                        url.set_path(&format!("/mcaptcha/api/v1/{id}/upload"));
+                        let resp =
+                            self.client.post(url).json(&payload).send().await.unwrap();
+                        println!("{}", resp.text().await.unwrap());
+                    }
+                }
+            }
+            page += 1;
+        }
+        Ok(())
+    }
+
+    async fn register(&self) -> ServiceResult<()> {
+        #[derive(Serialize)]
+        struct MCaptchaInstance {
+            url: url::Url,
+            auth_token: String,
+        }
+
+        let this_instance_url = self
+            .app_ctx
+            .settings
+            .survey
+            .as_ref()
+            .unwrap()
+            .instance_root_url
+            .clone();
+        for url in self.app_ctx.settings.survey.as_ref().unwrap().nodes.iter() {
+            // mCaptcha/survey must send this token while uploading secret to authenticate itself
+            // this token must be sent to mCaptcha/survey with the registration payload
+            let secret_upload_auth_token = crate::api::v1::mcaptcha::get_random(20);
+
+            let payload = MCaptchaInstance {
+                url: this_instance_url.clone(),
+                auth_token: secret_upload_auth_token.clone(),
+            };
+
+            // SecretsStore will store auth tokens generated by both mCaptcha/mCaptcha and
+            // mCaptcha/survey
+            //
+            // Storage schema:
+            // - mCaptcha/mCaptcha generated auth token: (<auth_token>, <survey_instance_url>)
+            // - mCaptcha/survey generated auth token (<survey_instance_url>, <auth_token)
+            self.app_ctx
+                .survey_secrets
+                .set(secret_upload_auth_token, url.to_string());
+            let mut url = url.clone();
+            url.set_path("/mcaptcha/api/v1/register");
+            let resp = self.client.post(url).json(&payload).send().await.unwrap();
+        }
+        Ok(())
+    }
+}

src/tests/mod.rs

@@ -20,6 +20,7 @@ use crate::api::v1::mcaptcha::create::CreateCaptcha;
 use crate::api::v1::mcaptcha::create::MCaptchaDetails;
 use crate::api::v1::ROUTES;
 use crate::errors::*;
+use crate::survey::SecretsStore;
 use crate::ArcData;
 
 pub fn get_settings() -> Settings {
@@ -30,6 +31,7 @@ pub mod pg {
 
     use crate::data::Data;
     use crate::settings::*;
+    use crate::survey::SecretsStore;
     use crate::ArcData;
 
     use super::get_settings;
@@ -42,7 +44,7 @@ pub mod pg {
         settings.database.database_type = DBType::Postgres;
         settings.database.pool = 2;
 
-        Data::new(&settings).await
+        Data::new(&settings, SecretsStore::default()).await
     }
 }
 pub mod maria {
@@ -50,6 +52,7 @@ pub mod maria {
 
     use crate::data::Data;
     use crate::settings::*;
+    use crate::survey::SecretsStore;
     use crate::ArcData;
 
     use super::get_settings;
@@ -62,7 +65,7 @@ pub mod maria {
         settings.database.database_type = DBType::Maria;
         settings.database.pool = 2;
 
-        Data::new(&settings).await
+        Data::new(&settings, SecretsStore::default()).await
     }
 }
 //pub async fn get_data() -> ArcData {
@@ -181,6 +184,26 @@ pub async fn signin(
     (creds, signin_resp)
 }
 
+/// pub duplicate test
+pub async fn bad_post_req_test_no_auth<T: Serialize>(
+    data: &ArcData,
+    url: &str,
+    payload: &T,
+    err: ServiceError,
+) {
+    let app = get_app!(data).await;
+
+    let resp = test::call_service(&app, post_request!(&payload, url).to_request()).await;
+    if resp.status() != err.status_code() {
+        let resp_err: ErrorToResponse = test::read_body_json(resp).await;
+        panic!("error {}", resp_err.error);
+    }
+    assert_eq!(resp.status(), err.status_code());
+    let resp_err: ErrorToResponse = test::read_body_json(resp).await;
+    //println!("{}", txt.error);
+    assert_eq!(resp_err.error, format!("{}", err));
+}
+
 /// pub duplicate test
 pub async fn bad_post_req_test<T: Serialize>(
     data: &ArcData,

utils/cache-bust/Cargo.lock

@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "block-buffer"
-version = "0.10.2"
+version = "0.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
 dependencies = [
  "generic-array",
 ]
@@ -15,26 +15,11 @@ dependencies = [
 name = "cache-bust"
 version = "0.1.0"
 dependencies = [
- "cache-buster",
+ "libcachebust",
  "serde",
  "serde_json",
 ]
 
-[[package]]
-name = "cache-buster"
-version = "0.2.0"
-source = "git+https://github.com/realaravinth/cache-buster#7ca4545722fb99be30698a5e72c7d982a70fa11f"
-dependencies = [
- "data-encoding",
- "derive_builder",
- "mime",
- "mime_guess",
- "serde",
- "serde_json",
- "sha2",
- "walkdir",
-]
-
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -43,18 +28,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cpufeatures"
-version = "0.2.2"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
+checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "crypto-common"
-version = "0.1.3"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
 dependencies = [
  "generic-array",
  "typenum",
@@ -62,9 +47,9 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.14.1"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4529658bdda7fd6769b8614be250cdcfc3aeb0ee72fe66f9e41e5e5eb73eac02"
+checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -72,71 +57,71 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.14.1"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "649c91bc01e8b1eac09fb91e8dbc7d517684ca6be8ebc75bb9cafc894f9fdb6f"
+checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.14.1"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddfc69c5bfcbd2fc09a0f38451d2daf0e372e367986a83906d1b0dbc88134fb5"
+checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
 dependencies = [
  "darling_core",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
 name = "data-encoding"
-version = "2.3.2"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57"
+checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308"
 
 [[package]]
 name = "derive_builder"
-version = "0.11.2"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d07adf7be193b71cc36b193d0f5fe60b918a3a9db4dad0449f57bcfd519704a3"
+checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
 dependencies = [
  "derive_builder_macro",
 ]
 
 [[package]]
 name = "derive_builder_core"
-version = "0.11.2"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f91d4cfa921f1c05904dc3c57b4a32c38aed3340cce209f3a6fd1478babafc4"
+checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
 dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
 name = "derive_builder_macro"
-version = "0.11.2"
+version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f0314b72bed045f3a68671b3c86328386762c93f82d98c65c3cb5e5f573dd68"
+checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
 dependencies = [
  "derive_builder_core",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
 name = "digest"
-version = "0.10.3"
+version = "0.10.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
 dependencies = [
  "block-buffer",
  "crypto-common",
@@ -150,9 +135,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
 name = "generic-array"
-version = "0.14.5"
+version = "0.14.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
 dependencies = [
  "typenum",
  "version_check",
@@ -166,21 +151,37 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
 
 [[package]]
 name = "itoa"
-version = "1.0.1"
+version = "1.0.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
 
 [[package]]
 name = "libc"
-version = "0.2.125"
+version = "0.2.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b"
+checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
+
+[[package]]
+name = "libcachebust"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e65010b256a3c85a90e88212d3e360d2ac15643d087878f09837ba6b2c83c120"
+dependencies = [
+ "data-encoding",
+ "derive_builder",
+ "mime",
+ "mime_guess",
+ "serde",
+ "serde_json",
+ "sha2",
+ "walkdir",
+]
 
 [[package]]
 name = "mime"
-version = "0.3.16"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 
 [[package]]
 name = "mime_guess"
@@ -194,27 +195,27 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.38"
+version = "1.0.69"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa"
+checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.18"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
  "proc-macro2",
 ]
 
 [[package]]
 name = "ryu"
-version = "1.0.9"
+version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
 
 [[package]]
 name = "same-file"
@@ -227,29 +228,29 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.137"
+version = "1.0.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.137"
+version = "1.0.189"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
+checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.38",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.81"
+version = "1.0.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
+checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
 dependencies = [
  "itoa",
  "ryu",
@@ -258,9 +259,9 @@ dependencies = [
 
 [[package]]
 name = "sha2"
-version = "0.10.2"
+version = "0.10.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
 dependencies = [
  "cfg-if",
  "cpufeatures",
@@ -275,35 +276,46 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "1.0.94"
+version = "1.0.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
 dependencies = [
  "proc-macro2",
  "quote",
- "unicode-xid",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
 ]
 
 [[package]]
 name = "typenum"
-version = "1.15.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
 
 [[package]]
 name = "unicase"
-version = "2.6.0"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
+checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
 dependencies = [
  "version_check",
 ]
 
 [[package]]
-name = "unicode-xid"
+name = "unicode-ident"
-version = "0.2.3"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
 [[package]]
 name = "version_check"
@@ -313,12 +325,11 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "walkdir"
-version = "2.3.2"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
+checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
 dependencies = [
  "same-file",
- "winapi",
  "winapi-util",
 ]
 
@@ -340,9 +351,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
 [[package]]
 name = "winapi-util"
-version = "0.1.5"
+version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
 dependencies = [
  "winapi",
 ]

utils/cache-bust/Cargo.toml

@@ -12,6 +12,6 @@ authors = ["realaravinth <realaravinth@batsense.net>"]
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-cache-buster = { version = "0.2.0", git = "https://github.com/realaravinth/cache-buster" }
+libcachebust = "0.3.0"
 serde = { version = "1", features  = ["derive"] }
 serde_json = "1"

utils/cache-bust/src/main.rs

@@ -7,7 +7,7 @@ use std::fs;
 use std::path::Path;
 use std::collections::HashMap;
 
-use cache_buster::{BusterBuilder, CACHE_BUSTER_DATA_FILE, NoHashCategory};
+use libcachebust::{BusterBuilder, CACHE_BUSTER_DATA_FILE, NoHashCategory};
 use serde::{Serialize, Deserialize};
 
 #[derive(Deserialize, Serialize)]