diff --git a/docker-compose.debugger.yml b/docker-compose.debugger.yml new file mode 100644 index 0000000..6af827e --- /dev/null +++ b/docker-compose.debugger.yml @@ -0,0 +1,37 @@ +# A copy of the docker compose, specifically for debuggers. +# Will not run django by default, your debugger needs to run ``manage.py``. +services: + db: + container_name: mapdb-postgres-debugger + image: postgres + volumes: + - ${POSTGRES_DATA_DIR}/debugger-db/:/var/lib/postgresql/data + environment: + - POSTGRES_DB=${POSTGRES_DB} + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + env_file: + - .env + ports: + - "127.0.0.1:${POSTGRES_PORT}:${POSTGRES_PORT}" + command: -p ${POSTGRES_PORT} + + debugger-django: + container_name: mapdb-django-debugger + build: + context: . + dockerfile: docker/Dockerfile + target: debugger + volumes: + - .:/cncnet-map-api + - ${HOST_MEDIA_ROOT}:/data/cncnet_silo # django will save user-uploaded files here. MEDIA_ROOT + - ${HOST_STATIC_ROOT}:/data/cncnet_static # django will gather static files here. STATIC_ROOT + - ./data/tmp:/tmp/pytest-of-root # For inspecting files during pytests + ports: + - "8000:8000" + env_file: + - .env + environment: + POSTGRES_TEST_HOST: db # Necessary to connect to docker db. Overrides the .env setting. + depends_on: + - db diff --git a/docker-compose.yml b/docker-compose.yml index 6bee93e..52f55dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,22 +43,3 @@ services: - "80:80" depends_on: - django - - debugger: - # Use this for launching via a debugger like PyCharm or VSCode. - # Just builds, and doesn't execute anything, your debugger will be in charge of executing. - build: - context: . - dockerfile: docker/Dockerfile - target: debugger - volumes: - - .:/cncnet-map-api - - ./data/tmp:/tmp/pytest-of-root # For inspecting files during pytests - env_file: - - .env - environment: - POSTGRES_TEST_HOST: mapdb-postgres-dev # Necessary to connect to docker db. Overrides the .env setting. -# ports: -# - "80:80" - depends_on: - - db diff --git a/kirovy/constants/api_codes.py b/kirovy/constants/api_codes.py index fc0e3f5..2bcac02 100644 --- a/kirovy/constants/api_codes.py +++ b/kirovy/constants/api_codes.py @@ -9,6 +9,7 @@ class UploadApiCodes(enum.StrEnum): EMPTY_UPLOAD = "where-file" DUPLICATE_MAP = "duplicate-map" FILE_EXTENSION_NOT_SUPPORTED = "file-extension-not-supported" + INVALID = "invalid-data-upload" class LegacyUploadApiCodes(enum.StrEnum): diff --git a/kirovy/migrations/0014_cncmapimagefile.py b/kirovy/migrations/0014_cncmapimagefile.py new file mode 100644 index 0000000..c6005e2 --- /dev/null +++ b/kirovy/migrations/0014_cncmapimagefile.py @@ -0,0 +1,51 @@ +# Generated by Django 4.2.21 on 2025-05-19 17:59 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import kirovy.models.file_base +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ("kirovy", "0013_cncmap_is_mapdb1_compatible_alter_cncmapfile_file"), + ] + + operations = [ + migrations.CreateModel( + name="CncMapImageFile", + fields=[ + ("id", models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ("created", models.DateTimeField(auto_now_add=True, null=True)), + ("modified", models.DateTimeField(auto_now=True, null=True)), + ("name", models.CharField(max_length=255)), + ("file", models.FileField(upload_to=kirovy.models.file_base._generate_upload_to)), + ("hash_md5", models.CharField(max_length=32)), + ("hash_sha512", models.CharField(max_length=512)), + ("hash_sha1", models.CharField(max_length=50, null=True)), + ("width", models.IntegerField()), + ("height", models.IntegerField()), + ("version", models.IntegerField(editable=False)), + ("cnc_game", models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to="kirovy.cncgame")), + ("cnc_map", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="kirovy.cncmap")), + ( + "file_extension", + models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to="kirovy.cncfileextension"), + ), + ( + "last_modified_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="modified_%(class)s_set", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/kirovy/models/cnc_map.py b/kirovy/models/cnc_map.py index 1b1175d..a3cf93a 100644 --- a/kirovy/models/cnc_map.py +++ b/kirovy/models/cnc_map.py @@ -227,3 +227,49 @@ def generate_upload_to(instance: "CncMapFile", filename: str) -> pathlib.Path: # e.g. "yr/maps/CNC_NET_MAP_ID_HEX/ra2_CNC_NET_MAP_ID_HEX_v1.map return pathlib.Path(instance.cnc_map.get_map_directory_path(), final_file_name) + + +class CncMapImageFile(file_base.CncNetFileBaseModel): + """Represents an image file to display on the website for a map. + + .. warning:: + + ``name`` is auto-generated for this file subclass. + """ + + objects = CncMapFileManager() + + width = models.IntegerField() + height = models.IntegerField() + version = models.IntegerField(editable=False) + + cnc_map = models.ForeignKey(CncMap, on_delete=models.CASCADE, null=False) + + ALLOWED_EXTENSION_TYPES = {game_models.CncFileExtension.ExtensionTypes.IMAGE.value} + + UPLOAD_TYPE = settings.CNC_MAP_DIRECTORY + + def save(self, *args, **kwargs): + super().save(*args, **kwargs) + + @staticmethod + def generate_upload_to(instance: "CncMapFile", filename: str) -> pathlib.Path: + """Generate the path to upload map files to. + + Gets called by :func:`kirovy.models.file_base._generate_upload_to` when ``CncMapImageFile.save`` is called. + See [the django docs for file fields](https://docs.djangoproject.com/en/5.0/ref/models/fields/#filefield). + ``upload_to`` is set in :attr:`kirovy.models.file_base.CncNetFileBaseModel.file`, which calls + ``_generate_upload_to``, which calls this function. + + :param instance: + Acts as ``self``. The image file object that we are creating an upload path for. + :param filename: + The filename of the uploaded image file. + :return: + Path to upload map to relative to :attr:`~kirovy.settings.base.MEDIA_ROOT`. + """ + filename = pathlib.Path(filename) + final_file_name = f"{instance.name}{filename.suffix}" + + # e.g. "yr/maps/CNC_NET_MAP_ID_HEX/screenshot_of_map.jpg + return pathlib.Path(instance.cnc_map.get_map_directory_path(), final_file_name) diff --git a/kirovy/serializers/__init__.py b/kirovy/serializers/__init__.py index 07dace2..f45a8cf 100644 --- a/kirovy/serializers/__init__.py +++ b/kirovy/serializers/__init__.py @@ -30,7 +30,7 @@ def get_fields(self): """ fields = super().get_fields() request: t.Optional[KirovyRequest] = self.context.get("request") - if not all([request, request.user.is_authenticated, request.user.is_staff]): + if not (request and request.user.is_authenticated and request.user.is_staff): fields.pop("last_modified_by_id", None) return fields diff --git a/kirovy/serializers/cnc_map_serializers.py b/kirovy/serializers/cnc_map_serializers.py index f9a777a..f228f5e 100644 --- a/kirovy/serializers/cnc_map_serializers.py +++ b/kirovy/serializers/cnc_map_serializers.py @@ -81,7 +81,7 @@ class Meta: hash_sha512 = serializers.CharField(required=True, allow_blank=False) hash_sha1 = serializers.CharField(required=True, allow_blank=False) - def create(self, validated_data: t) -> cnc_map.CncMapFile: + def create(self, validated_data: t.DictStrAny) -> cnc_map.CncMapFile: map_file = cnc_map.CncMapFile(**validated_data) map_file.save() return map_file @@ -106,7 +106,7 @@ class CncMapBaseSerializer(CncNetUserOwnedModelSerializer): description = serializers.CharField( required=True, allow_null=False, - allow_blank=False, + allow_blank=True, trim_whitespace=True, min_length=10, ) @@ -117,11 +117,9 @@ class CncMapBaseSerializer(CncNetUserOwnedModelSerializer): ) category_ids = serializers.PrimaryKeyRelatedField( source="categories", - queryset=cnc_map.MapCategory.objects.all(), pk_field=serializers.UUIDField(), many=True, - allow_null=False, - allow_empty=False, + read_only=True, # Set it manually. ) is_published = serializers.BooleanField( default=False, @@ -139,9 +137,27 @@ class CncMapBaseSerializer(CncNetUserOwnedModelSerializer): legacy_upload_date = serializers.DateTimeField( read_only=True, ) + incomplete_upload = serializers.BooleanField( + default=False, + ) + + parent_id = serializers.PrimaryKeyRelatedField( + source="parent", + queryset=cnc_map.CncMap.objects.all(), + pk_field=serializers.UUIDField(), + many=False, + allow_null=True, + allow_empty=False, + default=None, + ) class Meta: model = cnc_map.CncMap # We return the ID instead of the whole object. - exclude = ["cnc_game", "categories"] + exclude = ["cnc_game", "categories", "parent"] fields = "__all__" + + def create(self, validated_data: t.DictStrAny) -> cnc_map.CncMap: + cnc_map_instance = cnc_map.CncMap(**validated_data) + cnc_map_instance.save() + return cnc_map_instance diff --git a/kirovy/services/cnc_gen_2_services.py b/kirovy/services/cnc_gen_2_services.py index f0d3294..478e9be 100644 --- a/kirovy/services/cnc_gen_2_services.py +++ b/kirovy/services/cnc_gen_2_services.py @@ -60,7 +60,7 @@ def read_django_file(self, file: File): # We can't use ConfigParser.read_file because parser expects the file to be read as a string, # but django uploaded files are read as bytes. So we need to convert to string first. # If `decode` is crashing in a test, make sure your test file is read in read-mode "rb". - self.read_string(file.read().decode()) + self.read_string(file.read().decode(errors="ignore")) except configparser.ParsingError as e: raise exceptions.InvalidMapFile( ParseErrorMsg.CORRUPT_MAP, diff --git a/kirovy/settings/_base.py b/kirovy/settings/_base.py index 23b7895..a3bba82 100644 --- a/kirovy/settings/_base.py +++ b/kirovy/settings/_base.py @@ -169,6 +169,10 @@ """ str: The directory inside of :attr:`~kirovy.settings._base.STATIC_URL` where we store game-specific and mod-specific logos and backgrounds. So a Red Alert 2 icon would be in e.g. ``URL/static/game_images/ra2/icons/allies.png`` + +.. warning:: + + This is **not** where we store user-uploaded images. Do not store them here. """ diff --git a/kirovy/templates/__init__.py b/kirovy/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kirovy/templates/map_legacy_upload_ui.html b/kirovy/templates/map_legacy_upload_ui.html new file mode 100644 index 0000000..c248228 --- /dev/null +++ b/kirovy/templates/map_legacy_upload_ui.html @@ -0,0 +1,59 @@ + + + + + + CncNet Test Map Upload + {% load static %} + + + + + +
+
+ +
+

+ Select a <sha1>.zip from your file system. + It should contain the .mpr file for + Red Alert or .ini and + .bin for Tiberian Dawn. +

+

+ The archive will then be extracted, validated and rebuilt for storage. + You will receive a 200 OK status code if your + uploaded file was valid. +

+
+ + + + + +
+

The CnCNet client will function exactly like this form.

+
+ +
+
+ + diff --git a/kirovy/urls.py b/kirovy/urls.py index 3765965..827929d 100644 --- a/kirovy/urls.py +++ b/kirovy/urls.py @@ -18,7 +18,7 @@ from django.conf.urls.static import static from django.contrib import admin from django.db import connection -from django.urls import path, include +from django.urls import path, include, URLPattern, URLResolver from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView, SpectacularRedocView from kirovy.models import CncGame @@ -26,8 +26,10 @@ from kirovy.views import test, cnc_map_views, permission_views, admin_views, map_upload_views from kirovy import typing as t, constants +_DjangoPath = URLPattern | URLResolver -def _get_games_url_patterns() -> list[path]: + +def _get_games_url_patterns() -> list[_DjangoPath]: """Return URLs compatible with legacy CnCNet clients. - URLs are loaded when the :mod:`kirovy.urls` module is loaded, which happens when Django starts. @@ -48,6 +50,7 @@ def _get_games_url_patterns() -> list[path]: return [] return [ + path("upload-manual", cnc_map_views.MapLegacyStaticUI.as_view()), path("upload", map_upload_views.CncNetBackwardsCompatibleUploadView.as_view()), *( # Make e.g. /yr/map_hash, /ra2/map_hash, etc @@ -61,7 +64,7 @@ def _get_games_url_patterns() -> list[path]: ] -def _get_url_patterns() -> list[path]: +def _get_url_patterns() -> list[_DjangoPath]: """Return the root level url patterns. I added this because I wanted to have the root URLs at the top of the file, diff --git a/kirovy/views/cnc_map_views.py b/kirovy/views/cnc_map_views.py index 251b696..8109cf9 100644 --- a/kirovy/views/cnc_map_views.py +++ b/kirovy/views/cnc_map_views.py @@ -8,6 +8,7 @@ from rest_framework.filters import SearchFilter, OrderingFilter from django_filters import rest_framework as filters from rest_framework.permissions import AllowAny +from rest_framework.renderers import TemplateHTMLRenderer from rest_framework.views import APIView from kirovy import permissions @@ -17,6 +18,7 @@ CncMap, CncMapFile, ) +from kirovy.request import KirovyRequest from kirovy.response import KirovyResponse from kirovy.serializers import cnc_map_serializers from kirovy.views import base_views @@ -141,7 +143,9 @@ def get_queryset(self): """ base_query = ( CncMap.objects.filter( - Q(is_banned=False, is_published=True, incomplete_upload=False, is_temporary=False) | Q(is_legacy=True) + Q(is_banned=False, is_published=True, incomplete_upload=False, is_temporary=False) + | Q(is_legacy=True) + | Q(is_mapdb1_compatible=True) ).filter(cnc_game__is_visible=True) # Prefetch data necessary to the map grid. Pre-fetching avoids hitting the database in a loop. .select_related("cnc_user", "cnc_game", "parent", "parent__cnc_user") @@ -232,6 +236,16 @@ def perform_destroy(self, instance: CncMap): class BackwardsCompatibleMapView(APIView): + """Match the legacy mapdb download endpoints. + + This is needed until the new UI is running for the clients CnCNet owns. + + This will need to be kept around so that we maintain support for the clients that + we don't have the source code for. + + The backwards compatible URL is ``/{game_slug}/{map_hash_sha1}`` + """ + permission_classes = [AllowAny] def get(self, request, sha1_hash_filename: str, game_id: UUID, format=None): @@ -245,3 +259,23 @@ def get(self, request, sha1_hash_filename: str, game_id: UUID, format=None): return KirovyResponse(status=status.HTTP_404_NOT_FOUND) return FileResponse(map_file.file.open("rb"), as_attachment=True, filename=f"{map_file.hash_sha1}.zip") + + +class MapLegacyStaticUI(APIView): + """Temporary upload page for backwards compatible upload testing. + + Map authors need an easy way to upload their maps to the database so that they + can debug a failed client upload. + + This should be deprecated once the new UI is fully up and running. + (Or move it to a more django-friendly endpoint and give it space in the nw UI.) + + Emulates `the legacy uploader `_ + """ + + permission_classes = [AllowAny] + renderer_classes = [TemplateHTMLRenderer] + template_name = "map_legacy_upload_ui.html" + + def get(self, request: KirovyRequest) -> KirovyResponse: + return KirovyResponse() diff --git a/kirovy/views/map_upload_views.py b/kirovy/views/map_upload_views.py index 5027333..898707a 100644 --- a/kirovy/views/map_upload_views.py +++ b/kirovy/views/map_upload_views.py @@ -21,6 +21,7 @@ from kirovy.request import KirovyRequest from kirovy.response import KirovyResponse from kirovy.serializers import cnc_map_serializers +from kirovy.serializers.cnc_map_serializers import CncMapBaseSerializer from kirovy.services import legacy_upload from kirovy.services.cnc_gen_2_services import CncGen2MapParser, CncGen2MapSections from kirovy.utils import file_utils @@ -67,15 +68,24 @@ def post(self, request: KirovyRequest, format=None) -> KirovyResponse: parent_map = self.get_map_parent(map_parser) # Make the map that we will attach the map file too. - new_map = cnc_map.CncMap( - map_name=map_parser.ini.map_name, - cnc_game_id=game.id, - is_published=False, - incomplete_upload=True, - cnc_user=request.user, - parent=parent_map, + map_serializer = CncMapBaseSerializer( + data=dict( + map_name=map_parser.ini.map_name, + description="", + cnc_game_id=game.id, + is_published=False, + incomplete_upload=True, + cnc_user_id=request.user.id, + parent_id=parent_map.id if parent_map else None, + ), + context={"request": self.request}, ) - new_map.save() + if not map_serializer.is_valid(): + raise KirovyValidationError( + "Map failed validation", code=UploadApiCodes.INVALID, additional=map_serializer.errors + ) + + new_map = map_serializer.save() # Set the cncnet map ID in the map file ini. cnc_net_ini = {constants.CNCNET_INI_MAP_ID_KEY: str(new_map.id)} @@ -376,7 +386,6 @@ def post(self, request: KirovyRequest, format=None) -> KirovyResponse: # Will raise validation errors if the upload is invalid legacy_map_service = legacy_upload.get_legacy_service_for_slug(game.slug.lower())(uploaded_file) - # These hashes are for the full zip file and won't match map_hashes = self._get_file_hashes(ContentFile(legacy_map_service.file_contents_merged.read())) self.verify_file_does_not_exist(map_hashes) @@ -417,6 +426,7 @@ def post(self, request: KirovyRequest, format=None) -> KirovyResponse: "cnc_map_file": new_map_file.file.url, "cnc_map_id": new_map.id, "extracted_preview_file": None, + "download_url": f"/{game.slug}/{new_map_file.hash_sha1}.zip", }, ), status=status.HTTP_200_OK, diff --git a/static/bc-assets/bc-main.css b/static/bc-assets/bc-main.css new file mode 100644 index 0000000..a836a86 --- /dev/null +++ b/static/bc-assets/bc-main.css @@ -0,0 +1,104 @@ +body { + --bs-body-color: #f1f1f1; + --bs-body-bg: #141619; + color: var(--bs-body-color); + background-color: var(--bs-body-bg); + font-family: "Noto Sans", sans-serif; +} +#flex-main { + display: flex block; + flex-direction: column; + width: 100%; + align-items: center; + align-self: center; +} + +#flex-container { + display: flex block; + flex-direction: column; + padding: 0 15px; + max-width: 1400px; +} + +#header { + flex-direction: row; + flex-grow: 1; + display: flex; + align-items: center; + height: 65px; + padding-bottom: 2.5rem; +} + +#header > h1 { + padding: 0; + margin: 0; + flex-basis: max-content; + text-align-last: right; +} + +#header > * { + flex: 1 1 auto; +} + +.navbar-brand { + display: flex; + width: 225px; + height: 40px; + padding: 0; + position: relative; + margin-right: 2.5rem; + font-size: 1.25rem; +} + +.navbar-brand img { + max-width: unset; +} + +.logo-tagline { + position: absolute; + font-size: 10px; + font-weight: 700; + text-transform: uppercase; + right: 0; + top: 33px; + left: 67px; + color: #f9f9f98c; + transition: ease .35s all; + pointer-events: none; + -webkit-user-select: none; + user-select: none; +} + +span.inline-code { + color: #e6742b; + display: inline-block; + border: solid 1px #e6742b; + border-radius: 4px; + padding: 1px 3px; + font-family: monospace; + font-size: 100%; + font-weight: bolder; +} + +.it { + font-style: italic; +} + +form > * { + margin: 1rem 0; + padding: .3rem; +} + +form { + display: flex; + flex-direction: column; + padding: 1.5rem; + border-radius: 4px; + border: 1px solid #00fe89; +} + +#footer { + display: flex; + flex-direction: row-reverse; + padding: 3.5rem 0; +} diff --git a/tests/test_views/test_backwards_compatibility.py b/tests/test_views/test_backwards_compatibility.py index e093013..1c4a441 100644 --- a/tests/test_views/test_backwards_compatibility.py +++ b/tests/test_views/test_backwards_compatibility.py @@ -91,6 +91,7 @@ def test_map_upload_single_file_backwards_compatible( ) assert upload_response.status_code == status.HTTP_200_OK + assert upload_response.data["result"]["download_url"] == f"/{game.slug}/{file_sha1}.zip" _download_and_check_hash(client_anonymous, file_sha1, game, map_name, [original_extension])