improved bulk_update

- fixes support for 'MemoryFileUploadHandler' in bulk_update - add `dry_run` option to bulk_update - bulk_update now returns a page with pre/post action field values
saxix · Oct 24, 2023 · 0844dd8 · 0844dd8
1 parent e554428
commit 0844dd8
Show file tree

Hide file tree

Showing 9 changed files with 192 additions and 128 deletions.
diff --git a/CHANGES b/CHANGES
@@ -3,6 +3,9 @@ Release <dev>
 * new `MassUpdateForm.sort_fields`. Make optional MassUpdateForm fields sorting
 * make possible globally customize MassUpdateForm
 * removes async feature from Bulk update
+* fixes support for 'MemoryFileUploadHandler' in bulk_update
+* add `dry_run` option to bulk_update
+* bulk_update now returns a page with pre/post action field values
 
 
 Release 2.1

diff --git a/src/adminactions/bulk_update.py b/src/adminactions/bulk_update.py
@@ -1,16 +1,22 @@
+import codecs
+
+import io
+
 import csv
 import logging
+from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.core.files.utils import FileProxyMixin
 from pathlib import Path
 from typing import Dict, Optional, Sequence
 
 from django import forms
 from django.contrib import messages
 from django.contrib.admin import helpers
 from django.core.exceptions import ValidationError
-from django.core.validators import FileExtensionValidator
+from django.core.validators import FileExtensionValidator, MaxValueValidator, MinValueValidator
 from django.db.transaction import atomic
 from django.forms import Media
-from django.http import HttpResponseRedirect
+from django.http import HttpResponseRedirect, HttpResponse
 from django.shortcuts import render
 from django.utils.encoding import smart_str
 from django.utils.safestring import mark_safe
@@ -45,11 +51,6 @@ class BulkUpdateForm(forms.Form):
         help_text=_("CSV file"),
         validators=[FileExtensionValidator(allowed_extensions=["csv", "txt"])],
     )
-    # _async = forms.BooleanField(
-    #     label="Async",
-    #     required=False,
-    #     help_text=_("use Celery to run update in background"),
-    # )
     _clean = forms.BooleanField(
         label="Clean()", required=False, help_text=_("if checked calls obj.clean()")
     )
@@ -59,10 +60,8 @@ class BulkUpdateForm(forms.Form):
         required=False,
         help_text=_("if checked use obj.save() instead of manager.bulk_update()"),
     )
-
-    # _date_format = forms.CharField(
-    #     label="Date format", required=True, help_text=_("Date format")
-    # )
+    _dry_run = forms.BooleanField(label="DryRun", required=False,
+                                  help_text="Do not perform updates, just display results")
 
     @property
     def media(self):
@@ -154,6 +153,7 @@ def bulk_update(modeladmin, request, queryset):  # noqa
         form_initial = {
             "_selected_action": request.POST.getlist(helpers.ACTION_CHECKBOX_NAME),
             "_date_format": "%Y-%m-%d",
+            "_dry_run": False,
             "select_across": request.POST.get("select_across") == "1",
             "action": "bulk_update",
         }
@@ -174,19 +174,19 @@ def bulk_update(modeladmin, request, queryset):  # noqa
             if form.is_valid() and csv_form.is_valid() and map_form.is_valid():
                 header = csv_form.cleaned_data.pop("header")
                 csv_options = csv_form.cleaned_data
-                # validate = form.cleaned_data.get("_validate", False)
                 clean = form.cleaned_data.get("_clean", False)
-                # use_celery = form.cleaned_data.get("_async", False)
+                dry_run = form.cleaned_data.get("_dry_run", False)
                 try:
                     f = form.cleaned_data.pop("_file")
                     res = _bulk_update(
                         queryset,
-                        f.file.name,
+                        f,
                         mapping=map_form.get_mapping(),
                         header=header,
                         clean=clean,
                         indexes=map_form.cleaned_data["index_field"],
                         csv_options=csv_options,
+                        dry_run=dry_run
                     )
                     c = len(res["updated"])
                     messages.info(request, _("Updated %s records") % c)
@@ -202,7 +202,12 @@ def bulk_update(modeladmin, request, queryset):  # noqa
                     messages.error(request, f"{e.__class__.__name__}: {e}")
                     return HttpResponseRedirect(request.get_full_path())
                 else:
-                    return HttpResponseRedirect(request.get_full_path())
+                    return render(request, "adminactions/bulk_update_results.html",
+                                  context={"results": res,
+                                           "dry_run": dry_run,
+                                           "media": Media(css={"all": ["adminactions/css/bulkupdate.css"]}),
+                                           "action_short_description": bulk_update.short_description,
+                                           "opts": queryset.model._meta})
         else:
             form = bulk_update_form(initial=form_initial)
             csv_form = CSVConfigForm(initial=csv_initial, prefix="csv")
@@ -220,10 +225,10 @@ def bulk_update(modeladmin, request, queryset):  # noqa
             "map_form": map_form,
             "action_short_description": bulk_update.short_description,
             "title": "%s (%s)"
-            % (
-                bulk_update.short_description.capitalize(),
-                smart_str(modeladmin.opts.verbose_name_plural),
-            ),
+                     % (
+                         bulk_update.short_description.capitalize(),
+                         smart_str(modeladmin.opts.verbose_name_plural),
+                     ),
             "change": True,
             "is_popup": False,
             "save_as": False,
@@ -248,62 +253,73 @@ def bulk_update(modeladmin, request, queryset):  # noqa
 
 def _bulk_update(  # noqa: max-complexity: 18
     queryset,
-    filename,
+    file_name_or_object,
     *,
     mapping: Dict,
     indexes: Sequence[str],
     clean=False,
     header: bool = True,
     csv_options: Optional[Dict] = None,
     request=None,
+    dry_run: bool = False
 ):
     results = {
         "updated": [],
         "errors": [],
         "missing": [],
         "duplicates": [],
+        "changes": {}
     }
     adminaction_start.send(
         sender=queryset.model, action="bulk_update", request=request, queryset=queryset
     )
     try:
-        with Path(filename).open("r") as f:
-            if header:
-                reader = csv.DictReader(f.readlines(), **(csv_options or {}))
-                for k, v in mapping.items():
-                    if v not in reader.fieldnames:
-                        raise ValidationError(
-                            _("%s column is not present in the file") % v
-                        )
-            else:
-                reader = csv.reader(f.readlines(), **(csv_options or {}))
-                mapping = {k: int(v) - 1 for k, v in mapping.items()}
+        if isinstance(file_name_or_object, FileProxyMixin):
+            f = file_name_or_object
+        else:
+            f = Path(file_name_or_object).open("rb")
 
-            reverse = {v: k for k, v in mapping.items()}
-            with atomic():
-                for row in reader:
-                    key = {k: row[mapping[k]] for k in indexes}
-                    try:
-                        obj = queryset.get(**key)
-                        if header:
-                            for colname, value in row.items():
-                                field = reverse[colname]
+        if header:
+            reader = csv.DictReader(codecs.iterdecode(f, 'utf-8'), **(csv_options or {}))
+            for k, v in mapping.items():
+                if v not in reader.fieldnames:
+                    raise ValidationError(
+                        _("%s column is not present in the file") % v
+                    )
+        else:
+            reader = csv.reader(codecs.iterdecode(f, 'utf-8'), **(csv_options or {}))
+            mapping = {k: int(v) - 1 for k, v in mapping.items()}
+
+        reverse = {v: k for k, v in mapping.items()}
+        with atomic():
+            for i, row in enumerate(reader, 1):
+                key = {k: row[mapping[k]] for k in indexes}
+                try:
+                    obj = queryset.get(**key)
+                    changes = {}
+                    if header:
+                        for colname, value in row.items():
+                            field = reverse[colname]
+                            if field not in indexes:
+                                changes[field] = [getattr(obj, field), value]
+                                setattr(obj, field, value)
+                    else:
+                        for i, value in enumerate(row):
+                            if i in reverse.keys():
+                                field = reverse[i]
                                 if field not in indexes:
+                                    changes[field] = [getattr(obj, field), value]
                                     setattr(obj, field, value)
-                        else:
-                            for i, value in enumerate(row):
-                                if i in reverse.keys():
-                                    field = reverse[i]
-                                    if field not in indexes:
-                                        setattr(obj, field, value)
-                        if clean:
-                            obj.clean()
+                    results["changes"][str(key)] = changes
+                    if clean:
+                        obj.clean()
+                    if not dry_run:
                         obj.save()
-                        results["updated"].append(key)
-                    except queryset.model.DoesNotExist:
-                        results["missing"].append(key)
-                    except queryset.model.MultipleObjectsReturned:
-                        results["duplicates"].append(key)
+                    results["updated"].append(key)
+                except queryset.model.DoesNotExist:
+                    results["missing"].append(key)
+                except queryset.model.MultipleObjectsReturned:
+                    results["duplicates"].append(key)
         adminaction_end.send(
             sender=queryset.model,
             action="bulk_update",

diff --git a/src/adminactions/static/adminactions/css/bulkupdate.css b/src/adminactions/static/adminactions/css/bulkupdate.css
diff --git a/src/adminactions/static/adminactions/css/bulkupdate.css.map b/src/adminactions/static/adminactions/css/bulkupdate.css.map
diff --git a/src/adminactions/static/adminactions/css/bulkupdate.scss b/src/adminactions/static/adminactions/css/bulkupdate.scss
@@ -1,33 +1,40 @@
+table {
+    th.title {
+        background-color: var(--header-bg);
+    }
 
-table.bulk-update {
-    border: 1px solid #c9c9c9;
+    &.bulk-update {
+        border: 1px solid #c9c9c9;
 
-    td {
-        border-left: 1px solid #c9c9c9;
-    }
+        td {
+            border-left: 1px solid #c9c9c9;
+        }
 
-    select.func_select {
-        min-width: 100px;
-    }
-}
-table{
-    th.title{
-        background-color: var(--header-bg);
+        select.func_select {
+            min-width: 100px;
+        }
     }
+     &.bulk-update-results {
+
+     }
+
 }
+
 #col1 {
     float: left;
     //padding-right: 10px;
     width: 70%;
-    table{
+
+    table {
         width: 90%;
     }
 }
 
 #col2 {
     float: right;
     width: 30%;
-    table{
+
+    table {
         width: 90%;
     }
 }

diff --git a/src/adminactions/templates/adminactions/bulk_update_results.html b/src/adminactions/templates/adminactions/bulk_update_results.html
@@ -0,0 +1,46 @@
+{% extends "admin/change_form.html" %}
+{% load i18n admin_modify actions massupdate static %}
+{% block extrahead %}{{ block.super }}
+  <meta name="opts.label" content="{{ opts.label }}">
+  {{ media }}
+{% endblock %}
+
+{% block breadcrumbs %}{% if not is_popup %}
+  <div class="breadcrumbs">
+    <a href="../../">{% trans "Home" %}</a> &rsaquo;
+    <a href="../">{{ opts.app_label|capfirst|escape }}</a> &rsaquo;
+    <a href=".">{{ opts.verbose_name_plural|capfirst }}</a> &rsaquo;
+    {{ action_short_description|capfirst }} &rsaquo; Simulate
+  </div>
+{% endif %}{% endblock %}
+
+{% block content %}
+  {#  {'updated': [{'char': 'AAAA', 'id': '3'}], 'errors': [], 'missing': [], 'duplicates': [], 'changes': {<DemoModel: DemoModel object (3)>: <DemoModel: DemoModel object (3)>}}#}
+  {#  {{ results }}#}
+  <h1>Bulk Update results {% if dry_run %}(simulated) {% endif %}</h1>
+  <table class="bulk-update-results">
+    <thead>
+    <tr>
+      <th>Key</th>
+      <th>Changes
+      </th>
+    </tr>
+    </thead>
+    {% for key, changes in results.changes.items %}
+      <tr>
+        <td>{{ key }}</td>
+        <td>
+          <table>
+            {% for fld, values in changes.items %}
+              <tr>
+                <td>{{ fld }}</td>
+                <td>{{ values.0 }}</td>
+                <td>{{ values.1 }}</td>
+              </tr>
+            {% endfor %}
+          </table>
+        </td>
+      </tr>
+    {% endfor %}
+  </table>
+{% endblock %}
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -1,3 +1,7 @@
+import tempfile
+
+import shutil
+
 import logging
 import os
 
@@ -62,6 +66,7 @@ def pytest_configure(config):
     # import warnings
     # enable this to remove deprecations
     # warnings.simplefilter('once', DeprecationWarning)
+    from django.conf import settings
 
     if (
         config.option.markexpr.find("selenium") < 0
@@ -71,6 +76,10 @@ def pytest_configure(config):
         if not config.option.selenium_enable:
             setattr(config.option, "markexpr", "not selenium")
     os.environ["CELERY_ALWAYS_EAGER"] = "1"
+    os.environ["MEDIA_ROOT"] = "/tmp/media/"
+    settings.MEDIA_ROOT = tempfile.TemporaryDirectory().name
+    original_media = os.path.join(settings.DEMO_DIR, "media")
+    shutil.copytree(original_media, settings.MEDIA_ROOT)
 
     if config.option.log_level:
         import logging

diff --git a/tests/demo/settings.py b/tests/demo/settings.py
@@ -8,13 +8,13 @@
     CELERY_ALWAYS_EAGER=(bool, "true"), CELERY_BROKER_URL=(str, "redis://127.0.0.1")
 )
 
-here = os.path.dirname(__file__)
+DEMO_DIR = os.path.dirname(__file__)
 
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
 AUTHENTICATION_BACKENDS = ("demo.backends.AnyUserAuthBackend",)
 FILE_UPLOAD_HANDLERS = [
-    # "django.core.files.uploadhandler.MemoryFileUploadHandler",
+    "django.core.files.uploadhandler.MemoryFileUploadHandler",
     "django.core.files.uploadhandler.TemporaryFileUploadHandler",
 ]
 
@@ -90,9 +90,9 @@
 USE_I18N = True
 USE_L10N = True
 USE_TZ = True
-MEDIA_ROOT = os.path.join(here, "media")
+MEDIA_ROOT = os.environ.get("MEDIA_ROOT", os.path.join(DEMO_DIR, "media"))
 MEDIA_URL = ""
-STATIC_ROOT = os.path.join(here, "static")
+STATIC_ROOT = os.path.join(DEMO_DIR, "static")
 STATIC_URL = "/static/"
 SECRET_KEY = "c73*n!y=)tziu^2)y*@5i2^)$8z$tx#b9*_r3i6o1ohxo%*2^a"