From bbef192a0563cffcc432a1beb0a42807f65e8628 Mon Sep 17 00:00:00 2001 From: Jose134 Date: Sun, 2 Feb 2025 01:43:43 +0100 Subject: [PATCH] Install czkawka via download in dockerfile + fix uvicorn dependency --- .gitignore | 6 +----- Dockerfile | 19 ++++++++++++------- entrypoint.sh | 14 ++++++++++++++ requirements.txt | 2 ++ src/deduplication.py | 18 +++++++----------- src/filemoving.py | 8 ++++++-- src/main.py | 2 +- src/qbittorrent_api.py | 10 +++++++--- 8 files changed, 50 insertions(+), 29 deletions(-) create mode 100644 entrypoint.sh diff --git a/.gitignore b/.gitignore index c76fa4f..efa407c 100644 --- a/.gitignore +++ b/.gitignore @@ -159,8 +159,4 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - - -vendor/czkawka/* -vendor/ffmpeg/* \ No newline at end of file +#.idea/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4bb6c45..b218773 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,16 +14,21 @@ RUN pip install --no-cache-dir -r requirements.txt RUN pip install gunicorn # Copy the FastAPI project files into the container -COPY . . +COPY src/ . +COPY entrypoint.sh . -# Copy the czkawka binary into the container -COPY vendor/czkawka /bin/ +# Give execute permissions to the entrypoint script +RUN chmod +x entrypoint.sh -# Install ffmpeg with ffprobe -RUN apt-get update && apt-get install -y ffmpeg +# Copy the patterns.txt file into the container +COPY config/patterns.txt /config/patterns.txt + +# Install binary dependencies +RUN apt-get update && apt-get install -y wget ffmpeg +RUN wget -O /usr/local/bin/czkawka https://github.com/qarmin/czkawka/releases/download/8.0.0/linux_czkawka_cli +RUN chmod +x /usr/local/bin/czkawka # Expose the port the app runs on EXPOSE 8000 -# Command to run the FastAPI app using gunicorn with uvicorn workers -CMD ["gunicorn", "-w", "4", "-k", "uvicorn.workers.UvicornWorker", "main:app", "--bind", "0.0.0.0:8000"] \ No newline at end of file +ENTRYPOINT ["./entrypoint.sh"] \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..0f39dbd --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Create user and group using UID and GID from docker run command +useradd -u $UID -o -m user +groupadd -g $GID -o group + +# assign user to group +usermod -aG group user + +# switch to user +su user + +# launch the application +gunicorn -w 4 -k uvicorn.workers.UvicornWorker main:app --bind 0.0.0.0:8000 diff --git a/requirements.txt b/requirements.txt index 49e0235..4ac5332 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ requests +python-dotenv fastapi +uvicorn pytest ; extra == 'dev' \ No newline at end of file diff --git a/src/deduplication.py b/src/deduplication.py index 9f4f1dd..f4865b6 100644 --- a/src/deduplication.py +++ b/src/deduplication.py @@ -1,6 +1,10 @@ import subprocess import os from enum import Enum +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) class CZKAWKA_DELETION_METHOD(Enum): ALL_EXCEPT_NEWEST = "AEN" @@ -10,15 +14,7 @@ class CZKAWKA_DELETION_METHOD(Enum): NONE="NONE" def deduplicate_files(target_dir, exclude_files): - executables = { - "linux": "linux_czkawka_cli", - "win32": "windows_czkawka_cli.exe", - "darwin": "mac_czkawka_cli", - "docker": "linux_czkawka_cli" - } - - platform = "docker" if os.path.exists("/.dockerenv") else os.sys.platform - czkawka_executable = executables.get(platform, "czkawka_cli") + czkawka_executable = "czkawka" czkawka_deletion_method = CZKAWKA_DELETION_METHOD.ALL_EXCEPT_SMALLEST czkawka_tolerance = os.environ.get("CK_DUPLICATE_TOLERANCE", "2") _remove_duplicates(czkawka_executable, target_dir, exclude_files, czkawka_deletion_method, czkawka_tolerance) @@ -28,10 +24,10 @@ def _remove_duplicates(czkawka_path, target_dir, exclude_files, delete_method, t flags = _build_czkawka_flags(target_dir, exclude_files, delete_method, tolerance) result = subprocess.run([czkawka_path, *flags], capture_output=True, text=True, check=True) - print(result.stdout) + logger.log(result.stdout) except subprocess.CalledProcessError as e: - print(f"Failed to find duplicates: {e.stderr}") + logger.error(f"Failed to find duplicates: {e.stderr}") def _build_czkawka_flags(target_dir, exclude_files, delete_method, tolerance): flags = ["video", "--directories", target_dir, "--not-recursive", "--delete-method", delete_method.value] diff --git a/src/filemoving.py b/src/filemoving.py index 5d9c610..2336213 100644 --- a/src/filemoving.py +++ b/src/filemoving.py @@ -1,10 +1,14 @@ import os import shutil import re +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) def group_files_by_prefix(directory, downloading_files, patterns): if not os.path.isdir(directory): - print(f"The directory {directory} does not exist.") + logger.error(f"The directory {directory} does not exist.") return files = [f for f in os.listdir(directory) if os.path.isfile(os.path.join(directory, f))] @@ -13,7 +17,7 @@ def group_files_by_prefix(directory, downloading_files, patterns): skip = False for downloading_file in downloading_files: if file.startswith(downloading_file): - print(f"File {file} is downloading. Skipping...") + logger.info(f"File {file} is downloading. Skipping...") skip = True continue diff --git a/src/main.py b/src/main.py index 6fdf5ec..4edad53 100644 --- a/src/main.py +++ b/src/main.py @@ -7,9 +7,9 @@ from qbittorrent_api import get_qbittorrent_files_downloading from filemoving import group_files_by_prefix from deduplication import deduplicate_files import uuid -import time logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) load_dotenv() diff --git a/src/qbittorrent_api.py b/src/qbittorrent_api.py index 7410fec..2c71216 100644 --- a/src/qbittorrent_api.py +++ b/src/qbittorrent_api.py @@ -1,5 +1,9 @@ import requests from os import environ +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) def get_qbittorrent_files_downloading(api_url, user, password): cookies = _login_qbittorrent(api_url, user, password) @@ -10,7 +14,7 @@ def get_qbittorrent_files_downloading(api_url, user, password): response = requests.get(f'{api_url}/api/v2/torrents/info?filter=downloading', cookies=cookies) if response.status_code != 200: - print(f"Failed to fetch data from qBittorrent API: {response.status_code}") + logger.error(f"Failed to fetch data from qBittorrent API: {response.status_code}") return [] torrents = response.json() @@ -32,7 +36,7 @@ def _login_qbittorrent(api_url, username, password): response = requests.post(login_url, data=data) if response.status_code != 200 or response.text != 'Ok.': - print(f"Failed to login to qBittorrent: {response.status_code}") + logger.error(f"Failed to login to qBittorrent: {response.status_code}") return None return response.cookies @@ -41,7 +45,7 @@ def _logout_qbittorrent(api_url, cookies): response = requests.get(f'{api_url}/api/v2/auth/logout', cookies=cookies) if response.status_code != 200 or response.text != 'Ok.': - print(f"Failed to logout from qBittorrent: {response.status_code}") + logger.error(f"Failed to logout from qBittorrent: {response.status_code}") return False return True \ No newline at end of file