From a186cbe263ced5a8feacf4ea340eac0c659ac2f3 Mon Sep 17 00:00:00 2001 From: Jose134 Date: Fri, 31 Jan 2025 19:20:53 +0100 Subject: [PATCH] Docker & tests --- .dockerignore | 2 ++ .vscode/settings.json | 7 ++++ Dockerfile | 23 ++++++++++++ Jenkinsfile | 41 ++++++++++++++++++++++ README.md | 8 +++++ config/patterns.txt | 2 +- requirements.txt | 3 +- src/deduplication.py | 5 +-- src/filemoving.py | 20 ++--------- src/main.py | 9 +++++ src/qbittorrent_api.py | 4 +-- tests/filemoving_test.py | 75 ++++++++++++++++++++++++++++++++++++++++ 12 files changed, 176 insertions(+), 23 deletions(-) create mode 100644 .dockerignore create mode 100644 .vscode/settings.json create mode 100644 Dockerfile create mode 100644 Jenkinsfile create mode 100644 tests/filemoving_test.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..df4e914 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +tests/ +docs/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..ad7af29 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "python.testing.pytestArgs": [ + "tests" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..aba06e5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +# Use the official Python image from the Docker Hub +FROM python:3.9-slim + +# Set the working directory in the container +WORKDIR /app + +# Copy the requirements file into the container +COPY requirements.txt . + +# Install the dependencies +RUN pip install --no-cache-dir -r requirements.txt + +# Install gunicorn +RUN pip install gunicorn + +# Copy the FastAPI project files into the container +COPY . . + +# Expose the port the app runs on +EXPOSE 8000 + +# Command to run the FastAPI app using gunicorn with uvicorn workers +CMD ["gunicorn", "-w", "4", "-k", "uvicorn.workers.UvicornWorker", "main:app", "--bind", "0.0.0.0:8000"] \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 0000000..15f88b6 --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,41 @@ +pipeline { + agent any + + environment { + IMAGE_NAME = "darkbird/anime-file-organizer:latest" + REGISTRY_URL = "registry.xdarkbird.duckdns.org" + } + + stages { + stage('Docker build') { + steps { + sh """ + docker build --network="host" -t ${IMAGE_NAME} . + """ + } + } + stage('Docker tag') { + steps { + sh """ + docker image tag ${IMAGE_NAME} ${REGISTRY_URL}/${IMAGE_NAME} + """ + } + } + stage('Docker push') { + steps { + sh """ + docker push ${REGISTRY_URL}/${IMAGE_NAME} + """ + } + } + stage('Docker clean') { + steps { + sh """ + docker rmi ${IMAGE_NAME} + docker rmi ${REGISTRY_URL}/${IMAGE_NAME} + docker image prune -f + """ + } + } + } +} diff --git a/README.md b/README.md index f52c210..9534ca4 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,14 @@ Using regex, identify the directory where each file should be located, and the n 4. Log storage:\ Store all the logs for the current job in order to be able to debug and keep trazability of the process. +# Development + +```bash +git clone https://github.com/Jose134/file-organizer.git +cd file-organizer +pip install -r requirements.txt[dev] +``` + # Roadmap: - [x] Be able to move and rename files based on a regex - [x] Read qbittorrent credentials from .env file diff --git a/config/patterns.txt b/config/patterns.txt index 299ff23..90d90fa 100644 --- a/config/patterns.txt +++ b/config/patterns.txt @@ -1 +1 @@ -^\[Erai-raws\] (.*) - .*$ \ No newline at end of file +^\[Erai-raws\] (.*)(?= - ).*$ \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index af93f53..49e0235 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests -fastapi \ No newline at end of file +fastapi +pytest ; extra == 'dev' \ No newline at end of file diff --git a/src/deduplication.py b/src/deduplication.py index 13f6b77..036c9f8 100644 --- a/src/deduplication.py +++ b/src/deduplication.py @@ -17,9 +17,10 @@ def deduplicate_files(target_dir, exclude_files): def _remove_duplicates(czkawka_path, target_dir, exclude_files, delete_method: CZKAWKA_DELETION_METHOD): try: flags = ["video", "--directories", target_dir, "--not-recursive", "--delete-method", delete_method.value] - if exclude_files: + for file in exclude_files: flags.append("--excluded-items") - flags.extend(exclude_files) + flags.append(file) + flags.append("--tolerance") flags.append(os.environ.get("CK_DUPLICATE_TOLERANCE", "2")) print(flags) diff --git a/src/filemoving.py b/src/filemoving.py index e984bc0..5d9c610 100644 --- a/src/filemoving.py +++ b/src/filemoving.py @@ -2,9 +2,7 @@ import os import shutil import re -def group_files_by_prefix(directory, downloading_files): - patterns_str = _get_patterns() - +def group_files_by_prefix(directory, downloading_files, patterns): if not os.path.isdir(directory): print(f"The directory {directory} does not exist.") return @@ -22,7 +20,7 @@ def group_files_by_prefix(directory, downloading_files): if skip: continue - for pattern_str in patterns_str: + for pattern_str in patterns: pattern = re.compile(pattern_str) match = pattern.match(file) if match: @@ -31,16 +29,4 @@ def group_files_by_prefix(directory, downloading_files): if not os.path.exists(prefix_dir): os.makedirs(prefix_dir) - shutil.move(os.path.join(directory, file), os.path.join(prefix_dir, file)) - -def _get_patterns(): - config_file_path = os.path.join(os.getcwd(), 'config', 'patterns.txt') - - patterns = [] - if os.path.isfile(config_file_path): - with open(config_file_path, 'r') as file: - patterns = [line.strip() for line in file if line.strip()] - else: - print(f"The config file {config_file_path} does not exist.") - - return patterns \ No newline at end of file + shutil.move(os.path.join(directory, file), os.path.join(prefix_dir, file)) \ No newline at end of file diff --git a/src/main.py b/src/main.py index 3111f0f..6feffaf 100644 --- a/src/main.py +++ b/src/main.py @@ -34,6 +34,15 @@ def launch_job(job_id): with open("jobs.txt", "a") as f: f.write(f"{job_id}\n") + patterns = _load_patterns_file() # downloading = get_qbittorrent_files_downloading(qbit_url, qbit_user, qbit_password) # deduplicate_files(target_dir, downloading) # group_files_by_prefix(target_dir, downloading) + +def _load_patterns_file(): + config_file_path = os.path.join(os.getcwd(), 'config', 'patterns.txt') + if not path.exists(config_file_path): + print(f"The config file {config_file_path} does not exist.") + return [] + with open(config_file_path, 'r') as file: + return [line.strip() for line in file if line.strip()] \ No newline at end of file diff --git a/src/qbittorrent_api.py b/src/qbittorrent_api.py index b09b252..7410fec 100644 --- a/src/qbittorrent_api.py +++ b/src/qbittorrent_api.py @@ -2,7 +2,7 @@ import requests from os import environ def get_qbittorrent_files_downloading(api_url, user, password): - cookies = _login_qbittorrent('http://qbittorrent.xdarkbird.duckdns.org', user, password) + cookies = _login_qbittorrent(api_url, user, password) if not cookies: return [] @@ -20,7 +20,7 @@ def get_qbittorrent_files_downloading(api_url, user, password): if file_extension in ALLOWED_EXTENSIONS: files.append(torrent['name']) - _logout_qbittorrent('http://qbittorrent.xdarkbird.duckdns.org', cookies) + _logout_qbittorrent(api_url, cookies) return files def _login_qbittorrent(api_url, username, password): diff --git a/tests/filemoving_test.py b/tests/filemoving_test.py new file mode 100644 index 0000000..f4ebf35 --- /dev/null +++ b/tests/filemoving_test.py @@ -0,0 +1,75 @@ +import os +import shutil +import pytest + +_test_filenames = [ + "[Erai-raws] Fate Stay Night - Heaven s Feel - THE MOVIE III. spring song [1080p CR WEB-DL AVC AAC][MultiSub][BE0BDDA7]", + "[Erai-raws] fugukan - 03 [1080p CR WEB-DL AVC AAC][MultiSub][983AA6EF]", + "[Erai-raws] Golden Kamuy OVA - 02 [1080p CR WEB-DL AVC AAC][MultiSub][A731ADF8]", + "[Erai-raws] Goukon ni Ittara Onna ga Inakatta Hanashi - 12 [1080p HIDIVE WEB-DL AVC AAC][23F96132]", + "[Erai-raws] Grisaia Phantom Trigger - 04 [1080p CR WEBRip HEVC EAC3][MultiSub][27B78FFC]", + "[Erai-raws] Detective Conan - 1147 [1080p CR WEBRip HEVC EAC3][476E9FBE]", + "[Erai-raws] 2_5 Jigen no Ririsa - 23 [1080p HIDIVE WEB-DL AVC AAC][F6A89707]" +] + +@pytest.fixture(autouse=True) +def test_setup(): + os.makedirs("tests/test_data", exist_ok=True) + for filename in _test_filenames: + with open(os.path.join("tests/test_data", filename), 'w') as f: + f.write("test content") + + yield + + shutil.rmtree("tests/test_data", ignore_errors=True) + +def test_filemoving_nodownloads(): + from src.filemoving import group_files_by_prefix + patterns = [r"^\[Erai-raws\] (.*)(?= - ).*$"] + + + expected_dirs = [ + "Fate Stay Night - Heaven s Feel", + "fugukan", + "Golden Kamuy OVA", + "Goukon ni Ittara Onna ga Inakatta Hanashi", + "Grisaia Phantom Trigger", + "Detective Conan", + "2_5 Jigen no Ririsa" + ] + + group_files_by_prefix("tests/test_data", [], patterns) + + for i in range(len(expected_dirs)): + assert os.path.exists(os.path.join("tests/test_data", expected_dirs[i])) + assert os.path.exists(os.path.join("tests/test_data", expected_dirs[i], _test_filenames[i])) + assert not os.path.exists(os.path.join("tests/test_data", _test_filenames[i])) + + +def test_filemoving_ignores_downloads(): + from src.filemoving import group_files_by_prefix + patterns = [r"^\[Erai-raws\] (.*)(?= - ).*$"] + + downloads = [ + _test_filenames[0], + _test_filenames[2], + _test_filenames[4] + ] + + expected_dirs = [ + "fugukan", + "Goukon ni Ittara Onna ga Inakatta Hanashi", + "Detective Conan", + "2_5 Jigen no Ririsa" + ] + + + group_files_by_prefix("tests/test_data", downloads, patterns) + + + for root, dirs, files in os.walk("tests/test_data"): + if root == "tests/test_data": + for dir in dirs: + assert dir in expected_dirs + for file in files: + assert file in downloads \ No newline at end of file