From e6ce3dd3b8ca5ceda6579f4708b5f932ee055ecb Mon Sep 17 00:00:00 2001 From: Peter Schuemann Date: Mon, 14 Jul 2025 08:32:43 +0200 Subject: [PATCH] initial publish commit --- .env.template | 20 ++ .gitignore | 19 ++ Dockerfile | 16 ++ LICENSE | 18 ++ README.md | 96 ++++++++ docker-compose.yml | 36 +++ package-lock.json | 565 +++++++++++++++++++++++++++++++++++++++++++++ package.json | 12 + tdarr_requeue.mjs | 151 ++++++++++++ 9 files changed, 933 insertions(+) create mode 100644 .env.template create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 tdarr_requeue.mjs diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..50dbdc0 --- /dev/null +++ b/.env.template @@ -0,0 +1,20 @@ +# Tdarr connection +TDARR_URL=https://your.tdarr.url.tld/api/v2 +TDARR_API_KEY=tapi_XXXXXXXXXXXX + +# Behaviour +TDARR_STAGING_LIMIT=50 # refill if staging count < 50 +TDARR_BATCH_SIZE=50 # requeue at most 50 files per cycle +TDARR_RETRIES=4 # API retries +TDARR_BACKOFF_MS=2000 # initial back-off in ms + +# Logging +LOG_LEVEL=info # or debug / warn / error +LOG_PRETTY=1 # pretty-printed logs + +# How long the script waits for /bulk-update-files to respond (ms) +BULK_TIMEOUT_MS=120000 + +# Interval (minutes) between checks +TDARR_INTERVAL_MIN=60 + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..262e6ac --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +# ---> VisualStudioCode +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +# safety +.env + +#declutter +node_modules diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e2e70d9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +# ─────────────────────────────────────────────── +# Tdarr Auto-Requeue – minimal Node 20 image +# ─────────────────────────────────────────────── +FROM node:20-alpine + +WORKDIR /app + +# If you keep a package.json (recommended): +COPY package*.json ./ +RUN npm ci --omit=dev # installs axios, pino, minimist, dotenv + +# Script & docs +COPY tdarr-requeue.mjs . +COPY README.md . + +CMD ["node", "tdarr-requeue.mjs"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..7aebdd3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,18 @@ +MIT License + +Copyright (c) 2025 ComputerLiebe_ORG_private + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT +LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO +EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..169441e --- /dev/null +++ b/README.md @@ -0,0 +1,96 @@ +# Tdarr - Auto-Requeue Script + +Automatically refills the Tdarr **staging section** whenever the number of +actively processed files drops below a configurable threshold. + +## Features + +| Feature | What it does | +| --------------------------- | -------------------------------------------------------------------------------------------------- | +| **Staging guard** | Checks how many files are currently being processed. | +| **Smart picker** | Pulls up to `BATCH_SIZE` items from Status ▪ **Processed** (`table2`) whose **New Size** is `"-"`. | +| **GUI-identical requeue** | Re-queues the files with **one** `bulk-update-files` call (same payload the web UI sends). | +| **Retry + back-off** | Network errors or timeouts are retried with exponential back-off. | +| **Timeout-resilient** | If `bulk-update-files` times out, the script checks if the operation succeeded anyway. | +| **Internal scheduler** | No cron needed – the script runs continuously and checks every `TDARR_INTERVAL_MIN`. | +| **JSON / pretty logging** | Toggle pretty output with `LOG_PRETTY=1`. | +| **All settings via `.env`** | No hard-coded values; perfect for CI or Docker. | + +## File Selection Criteria + +The script selects files based on the following logic: + +* It queries the internal `table2` (equivalent to the **Status ▪ Processed** tab in the Tdarr UI). +* It filters for files that have the **“New Size” field set to `"-"`**, meaning: + + * The file was either skipped by a plugin decision or + * The file was marked as “Transcode Success / Not Required” + * And **no new output file** was produced (Tdarr left the file untouched). +* The top `BATCH_SIZE` of these files (sorted by Tdarr’s internal logic) are requeued. + +This selection ensures that: + +* You don’t accidentally requeue already optimized/transcoded files +* Only skipped or pass-through candidates get another chance (e.g., after plugin changes) + +## Requirements + +* Tdarr Server ≥ 2.40 (the `bulk-update-files` endpoint exists since 2024) +* Node 18+ (ESM support) + +## Installation + +```bash +git clone https://github.com/your-org/tdarr-auto-requeue.git +cd tdarr-auto-requeue +npm i axios, pino, minimist, dotenv +``` + +## Configuration + +Create a `.env` file in the project root (or set the variables in your CI / +container manager): + +```dotenv +# Tdarr connection +TDARR_URL=https://encode.computerliebe.org/api/v2 +TDARR_API_KEY=tapi_XXXXXXXXXXXX + +# Behaviour +TDARR_STAGING_LIMIT=50 # refill if staging count < 50 +TDARR_BATCH_SIZE=50 # requeue at most 50 files per cycle +TDARR_INTERVAL_MIN=60 # how often the script runs (minutes) +TDARR_RETRIES=4 # API retries +TDARR_BACKOFF_MS=2000 # initial back-off in ms +BULK_TIMEOUT_MS=120000 # max wait time for bulk-update-files in ms + +# Logging +LOG_LEVEL=info # or debug / warn / error +LOG_PRETTY=1 # pretty-printed logs +``` + +> **Tip:** create a dedicated API key in *Tdarr ▪ Tools ▪ API keys* with **Server +> write** permissions only. + +## Usage + +```bash +node tdarr_requeue.mjs # runs once, or in interval mode if TDARR_INTERVAL_MIN is set +``` + +The script will automatically re-run every `TDARR_INTERVAL_MIN` minutes. +No cron, systemd, or external timers needed. + +## Troubleshooting + +| Symptom | Fix | +| ------------------------------- | ----------------------------------------------------------------------------------------------- | +| `401 Unauthorized` | API key wrong or missing. | +| `FST_ERR_VALIDATION` | Your server expects a different payload – update Tdarr or open an issue with the error JSON. | +| Files not visible after requeue | Refresh the **Status ▪ Queued / Staging** view; workers may take a few seconds to pick them up. | +| `ECONNABORTED` timeout | This is handled gracefully. If the staging count increased, the script proceeds. | +| “Too many open files” | Increase `ulimit -n` on the host; Tdarr can be I/O intensive. | + +--- + +**Happy transcoding!** diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..95f12a7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +version: "3.9" + +services: + tdarr-requeue: + build: . + container_name: tdarr-requeue + restart: unless-stopped + + environment: + # Tdarr connection + TDARR_URL: ${TDARR_URL} + TDARR_API_KEY: ${TDARR_API_KEY} + + # Behaviour + TDARR_STAGING_LIMIT: ${TDARR_STAGING_LIMIT:-50} + TDARR_BATCH_SIZE: ${TDARR_BATCH_SIZE:-50} + TDARR_INTERVAL_MIN: ${TDARR_INTERVAL_MIN:-60} + TDARR_RETRIES: ${TDARR_RETRIES:-4} + TDARR_BACKOFF_MS: ${TDARR_BACKOFF_MS:-2000} + BULK_TIMEOUT_MS: ${BULK_TIMEOUT_MS:-120000} + + # Logging + LOG_LEVEL: ${LOG_LEVEL:-info} + LOG_PRETTY: ${LOG_PRETTY:-0} + + # Optional: if your Tdarr server lives on the same Docker network + # networks: + # - tdarr_net + + # Optional: host-side log directory + # volumes: + # - ./logs:/app/logs + +# networks: +# tdarr_net: +# external: true diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..a38aec3 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,565 @@ +{ + "name": "tdarr-auto-fill", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "axios": "^1.10.0", + "dotenv": "^17.2.0", + "minimist": "^1.2.8", + "pino": "^9.7.0", + "pino-pretty": "^13.0.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/axios": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.10.0.tgz", + "integrity": "sha512-/1xYAC4MP/HEG+3duIhFr4ZQXR4sQXOIe+o6sdqzeykGLx6Upp/1p8MHqhINOvGeP7xyNHe7tsiJByc4SSVUxw==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/dateformat": { + "version": "4.6.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", + "integrity": "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/dotenv": { + "version": "17.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.0.tgz", + "integrity": "sha512-Q4sgBT60gzd0BB0lSyYD3xM4YxrXA9y4uBDof1JNYGzOXrQdQ6yX+7XIAqoFOGQFOTK1D3Hts5OllpxMDZFONQ==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/fast-copy": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz", + "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==", + "license": "MIT" + }, + "node_modules/fast-redact": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.5.0.tgz", + "integrity": "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-safe-stringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==", + "license": "MIT" + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.3.tgz", + "integrity": "sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/help-me": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==", + "license": "MIT" + }, + "node_modules/joycon": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/joycon/-/joycon-3.1.1.tgz", + "integrity": "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/pino": { + "version": "9.7.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.7.0.tgz", + "integrity": "sha512-vnMCM6xZTb1WDmLvtG2lE/2p+t9hDEIvTWJsu6FejkE62vB7gDhvzrpFR4Cw2to+9JNQxVnkAKVPA1KPB98vWg==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0", + "fast-redact": "^3.1.1", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", + "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-pretty": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.0.0.tgz", + "integrity": "sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA==", + "license": "MIT", + "dependencies": { + "colorette": "^2.0.7", + "dateformat": "^4.6.3", + "fast-copy": "^3.0.2", + "fast-safe-stringify": "^2.1.1", + "help-me": "^5.0.0", + "joycon": "^3.1.1", + "minimist": "^1.2.6", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^2.0.0", + "pump": "^3.0.0", + "secure-json-parse": "^2.4.0", + "sonic-boom": "^4.0.1", + "strip-json-comments": "^3.1.1" + }, + "bin": { + "pino-pretty": "bin.js" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.0.0.tgz", + "integrity": "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==", + "license": "MIT" + }, + "node_modules/process-warning": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.0.0.tgz", + "integrity": "sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, + "node_modules/sonic-boom": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", + "integrity": "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/thread-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", + "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==", + "license": "MIT", + "dependencies": { + "real-require": "^0.2.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..2b7c525 --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "dependencies": { + "axios": "^1.10.0", + "dotenv": "^17.2.0", + "minimist": "^1.2.8", + "pino": "^9.7.0", + "pino-pretty": "^13.0.0" + }, + "scripts": { + "start": "node tdarr_requeue.mjs" + } +} diff --git a/tdarr_requeue.mjs b/tdarr_requeue.mjs new file mode 100644 index 0000000..514e460 --- /dev/null +++ b/tdarr_requeue.mjs @@ -0,0 +1,151 @@ +#!/usr/bin/env node +/** + * tdarr-requeue.mjs – v17 (2025-07-13) + * + * • Runs continuously; wakes up every TDARR_INTERVAL_MIN minutes. + * • If staging count < TDARR_STAGING_LIMIT: + * – Pull up to TDARR_BATCH_SIZE items from Status ▸ “Processed” (table2) + * where newSize == '-'. + * – Re-queue them via a single /bulk-update-files call + * (same payload as the Tdarr GUI). + * + * All behaviour is configured via .env variables – see README. + */ + +import 'dotenv/config'; +import axios from 'axios'; +import pino from 'pino'; +import minimist from 'minimist'; + +/* ──────────────────────────────────────────────────────────── + * Environment / CLI + * ─────────────────────────────────────────────────────────── */ +const cli = minimist(process.argv.slice(2)); +const API_BASE = (cli.url || process.env.TDARR_URL || 'http://localhost:8265/api/v2').replace(/\/?$/, '/'); +const API_KEY = (cli['api-key'] || process.env.TDARR_API_KEY || '').trim(); + +const STAGING_LIMIT = +process.env.TDARR_STAGING_LIMIT || 50; +const BATCH_SIZE = +process.env.TDARR_BATCH_SIZE || 50; +const INTERVAL_MIN = +process.env.TDARR_INTERVAL_MIN || 60; // scheduler interval +const RETRIES = +process.env.TDARR_RETRIES || 4; +const BACKOFF_MS = +process.env.TDARR_BACKOFF_MS || 2_000; +const BULK_TIMEOUT_MS= +process.env.BULK_TIMEOUT_MS || 120_000; // generous timeout + +if (!API_KEY) { + console.error('❌ TDARR_API_KEY is missing'); process.exit(20); +} + +/* ─── Logger ----------------------------------------------------------- */ +const log = pino({ + level: process.env.LOG_LEVEL ?? 'info', + transport: process.env.LOG_PRETTY === '1' + ? { target: 'pino-pretty', options: { translateTime: 'SYS:standard', colorize: true } } + : undefined +}); + +/* ─── Axios client ----------------------------------------------------- */ +const http = axios.create({ + baseURL: API_BASE, + timeout: 30_000, + headers: { 'content-type': 'application/json', 'x-api-key': API_KEY } +}); +http.interceptors.response.use(r => r, e => { + if ([401, 403].includes(e.response?.status)) { + log.error('🔑 Authentication failed – check TDARR_API_KEY'); + process.exit(20); + } + return Promise.reject(e); +}); + +/* ─── API helper with retry & back-off ------------------------------- */ +async function api(endpoint, payload, tries = RETRIES) { + for (let attempt = 1; attempt <= tries; attempt++) { + try { + const { data } = await http.post(endpoint, { data: payload }); + return data.data ?? data; // Tdarr wraps its result in {data: …} + } catch (err) { + log.warn({ endpoint, attempt, status: err.response?.status, body: err.response?.data }, + '⏳ retrying…'); + if (attempt === tries) throw err; + await new Promise(r => setTimeout(r, BACKOFF_MS * 2 ** attempt)); + } + } +} + +/* ─── Tdarr helpers ---------------------------------------------------- */ +const stagingCount = () => + api('cruddb', { collection: 'StagedJSONDB', mode: 'getAll' }) + .then(arr => arr.length || 0); + +const candidatePaths = () => + api('client/status-tables', { + start: 0, + pageSize: BATCH_SIZE, + filters: [{ id: 'newSize', value: '-' }], + sorts: [], + opts: { table: 'table2' } // “Processed” table + }).then(res => Array.isArray(res.array) ? res.array.map(f => f._id) : []); + +/* ─── Robust re-queue -------------------------------------------------- */ +async function requeue(paths) { + if (!paths.length) { + log.warn('⚠️ No candidates found'); + return 0; + } + + const payload = { + fileIds: paths, + updatedObj:{ TranscodeDecisionMaker: 'Queued' } + }; + log.debug({ payload }, '📨 bulk-update-files payload'); + + const before = await stagingCount(); + + try { + await http.post('bulk-update-files', { data: payload }, { timeout: BULK_TIMEOUT_MS }); + } catch (err) { + if (err.code === 'ECONNABORTED') { + log.warn('⌛ bulk-update-files timed out – verifying outcome …'); + const after = await stagingCount(); + if (after > before) { + log.info({ before, after }, '✅ bulk-update-files succeeded despite timeout'); + return paths.length; + } + } + throw err; // propagate any other failure or verification miss + } + return paths.length; +} + +/* ─── One processing cycle -------------------------------------------- */ +async function runCycle() { + log.info('🚀 Requeue cycle started'); + + const before = await stagingCount(); + log.info({ before }, '📦 items currently in staging'); + + if (before >= STAGING_LIMIT) { + log.info('🟢 staging limit reached – nothing to do'); + return; + } + + const paths = await candidatePaths(); + log.debug({ sample: paths.slice(0, 3) }, '🔍 example candidates'); + + const queued = await requeue(paths); + log.info({ queued }, '✅ files requeued'); + + const after = await stagingCount(); + log.info({ after }, '📈 staging count after requeue'); +} + +/* ─── Built-in scheduler ---------------------------------------------- */ +(async () => { + log.info(`📅 Scheduler running every ${INTERVAL_MIN} minute(s)`); + while (true) { + try { await runCycle(); } + catch (err) { log.error({ err }, '💥 cycle failed'); } + + await new Promise(r => setTimeout(r, INTERVAL_MIN * 60_000)); + } +})();