diff --git a/horilla/signals.py b/horilla/signals.py index d0d23a7e5..ac02e9d15 100644 --- a/horilla/signals.py +++ b/horilla/signals.py @@ -7,5 +7,14 @@ from django.dispatch import Signal pre_bulk_update = Signal() post_bulk_update = Signal() +pre_model_clean = Signal() +post_model_clean = Signal() + +pre_scheduler = Signal() +post_scheduler = Signal() + pre_generic_delete = Signal() post_generic_delete = Signal() + +pre_generic_import = Signal() +post_generic_import = Signal() diff --git a/horilla_views/cbv_methods.py b/horilla_views/cbv_methods.py index c4e96b056..31b1fa93b 100644 --- a/horilla_views/cbv_methods.py +++ b/horilla_views/cbv_methods.py @@ -302,7 +302,7 @@ def paginator_qry(qryset, page_number, records_per_page=50): """ This method is used to paginate queryset """ - if not isinstance(qryset, Page) and not qryset.ordered: + if hasattr(qryset, "ordered") and not qryset.ordered: qryset = ( qryset.order_by("-created_at") if hasattr(qryset.model, "created_at") @@ -378,7 +378,10 @@ def sortby( none_queryset = [] model = queryset.model model_attr = getmodelattribute(model, sort_key) - is_method = isinstance(model_attr, types.FunctionType) + is_method = ( + isinstance(model_attr, types.FunctionType) + or model_attr not in model._meta.get_fields() + ) if not is_method: none_queryset = queryset.filter(**{f"{sort_key}__isnull": True}) none_ids = list(none_queryset.values_list("id", flat=True)) @@ -480,6 +483,20 @@ def structured(self): return table_html +def get_original_model_field(historical_model): + """ + Given a historical model and a field name, + return the actual model field from the original model. + """ + model_name = historical_model.__name__.replace("Historical", "") + app_label = historical_model._meta.app_label + try: + original_model = apps.get_model(app_label, model_name) + return original_model + except Exception as e: + return historical_model + + def value_to_field(field: object, value: list) -> Any: """ return value according to the format of the field @@ -666,3 +683,244 @@ def export_xlsx(json_data, columns, file_name="quick_export"): ) response["Content-Disposition"] = f'attachment; filename="{file_name}.xlsx"' return response + + +from django.apps import apps +from django.core.exceptions import FieldDoesNotExist +from django.db.models import Model +from django.db.models.fields.related import ( + ForeignKey, + ManyToManyRel, + ManyToOneRel, + OneToOneField, + OneToOneRel, +) +from openpyxl import Workbook + + +def get_verbose_name_from_field_path(model, field_path, import_mapping): + """ + Get verbose name + """ + parts = field_path.split("__") + current_model = model + verbose_name = None + + for i, part in enumerate(parts): + try: + field = current_model._meta.get_field(part) + + # Skip reverse relations (e.g., OneToOneRel) + if isinstance(field, (OneToOneRel, ManyToOneRel, ManyToManyRel)): + related_model = field.related_model + field = getattr(related_model, parts[-1]).field + return field.verbose_name.title() + + verbose_name = field.verbose_name + + if isinstance(field, (ForeignKey, OneToOneField)): + current_model = field.related_model + + except FieldDoesNotExist: + return f"[Invalid: {field_path}]" + + return verbose_name.title() if verbose_name else field_path + + +def generate_import_excel( + base_model, import_fields, reference_field="id", import_mapping={}, queryset=[] +): + """ + Generate import excel + """ + wb = Workbook() + ws = wb.active + ws.title = "Import Sheet" + + # Style definitions + header_fill = PatternFill( + start_color="FFD700", end_color="FFD700", fill_type="solid" + ) + bold_font = Font(bold=True) + wrap_alignment = Alignment(wrap_text=True, vertical="center", horizontal="center") + thin_border = Border( + left=Side(style="thin"), + right=Side(style="thin"), + top=Side(style="thin"), + bottom=Side(style="thin"), + ) + + # Generate headers + headers = [ + get_verbose_name_from_field_path(base_model, field, import_mapping) + for field in import_fields + ] + headers = [ + f"{get_verbose_name_from_field_path(base_model, reference_field,import_mapping)} | Reference" + ] + headers + ws.append(headers) + + # Apply styles to header row + for col_num, _ in enumerate(headers, 1): + cell = ws.cell(row=1, column=col_num) + cell.font = bold_font + cell.fill = header_fill + cell.alignment = wrap_alignment + cell.border = thin_border + + col_letter = get_column_letter(col_num) + ws.column_dimensions[col_letter].width = 30 + + for obj in queryset: + row = [str(getattribute(obj, reference_field))] + [ + str(getattribute(obj, import_mapping.get(field, field))) + for field in import_fields + ] + ws.append(row) + ws.freeze_panes = "A2" + ws.freeze_panes = "B2" + return wb + + +def split_by_import_reference(employee_data): + with_import_reference = [] + without_import_reference = [] + + for record in employee_data: + if record.get("id_import_reference") is not None: + with_import_reference.append(record) + else: + without_import_reference.append(record) + + return with_import_reference, without_import_reference + + +def resolve_foreign_keys( + base_model, + record, + import_column_mapping, + model_lookup, + primary_key_mapping, + pk_values_mapping, + prefix="", +): + resolved = {} + + for key, value in record.items(): + full_key = f"{prefix}__{key}" if prefix else key + + if isinstance(value, dict): + try: + field = base_model._meta.get_field(key) + related_model = field.related_model + except Exception: + resolved[key] = value + continue + + # Recursively resolve nested foreign keys + nested_data = resolve_foreign_keys( + related_model, + value, + import_column_mapping, + model_lookup, + primary_key_mapping, + pk_values_mapping, + prefix=full_key, + ) + instance = related_model.objects.create(**nested_data) + resolved[key] = instance + + else: + model_class = model_lookup.get(full_key) + lookup_field = primary_key_mapping.get(full_key) + + if model_class and lookup_field: + if value in [None, ""]: + resolved[key] = None + continue + + try: + instance, _ = model_class.objects.get_or_create( + **{lookup_field: value} + ) + resolved[key] = instance + except Exception as e: + raise ValueError( + f"Failed to get_or_create '{model_class.__name__}' using {lookup_field}={value}: {e}" + ) + else: + resolved[key] = value + + return resolved + + +def update_related( + obj, + record, + primary_key_mapping, + reverse_model_relation_to_base_model, +): + related_objects = { + key: getattribute(obj, key) or None + for key in reverse_model_relation_to_base_model + } + for relation in reverse_model_relation_to_base_model: + related_record_info = record.get(relation) + for key, value in related_record_info.items(): + related_object = related_objects[relation] + obj_related_field = relation + "__" + key + pk_mapping = primary_key_mapping.get(obj_related_field) + if obj_related_field in primary_key_mapping and pk_mapping: + previous_obj = getattr(related_object, key, None) + if previous_obj and value is not None: + new_obj = previous_obj._meta.model.objects.get( + **{pk_mapping: value} + ) + setattr(related_object, key, new_obj) + else: + if value is not None: + setattr(related_object, key, value) + if related_object: + related_object.save() + + +def assign_related( + record, + reverse_field, + pk_values_mapping, + pk_field_mapping, +): + """ + Method to assign related records + """ + reverse_obj_dict = {} + if reverse_field in record: + if isinstance(record[reverse_field], dict): + for field, value in record[reverse_field].items(): + full_field = reverse_field + "__" + field + if full_field in pk_values_mapping: + reverse_obj_dict.update( + { + field: data + for data in pk_values_mapping[full_field] + if getattr(data, pk_field_mapping[full_field], None) + == value + } + ) + else: + reverse_obj_dict[field] = value + else: + instances = [ + data + for data in pk_values_mapping[reverse_field] + if getattr( + data, + pk_field_mapping[reverse_field], + record[reverse_field], + ) + == record[reverse_field] + ] + if instances: + instance = instances[0] + reverse_obj_dict.update({reverse_field: instance}) + return reverse_obj_dict diff --git a/horilla_views/forms.py b/horilla_views/forms.py index ad994f5a7..02aa1d7d5 100644 --- a/horilla_views/forms.py +++ b/horilla_views/forms.py @@ -19,6 +19,7 @@ from horilla_views.cbv_methods import ( FIELD_WIDGET_MAP, MODEL_FORM_FIELD_MAP, get_field_class_map, + get_original_model_field, structured, ) from horilla_views.templatetags.generic_template_filters import getattribute @@ -202,6 +203,8 @@ class DynamicBulkUpdateForm(forms.Form): fiels_mapping = {} parent_model = self.root_model for key, val in mappings.items(): + if val.model.__name__.startswith("Historical"): + val.model = get_original_model_field(val.model) field = MODEL_FORM_FIELD_MAP.get(type(val)) if field: if not fiels_mapping.get(val.model): @@ -216,6 +219,8 @@ class DynamicBulkUpdateForm(forms.Form): else: related_key = key.split("__")[-2] field = getattribute(parent_model, related_key) + if not hasattr(field, "related"): + continue relation_mapp[val.model] = ( field.related.field.name + "__" @@ -246,6 +251,8 @@ class DynamicBulkUpdateForm(forms.Form): data_mapp[val.model][key] = value[0] for model, data in data_mapp.items(): + if not model in relation_mapp: + continue queryset = model.objects.filter(**{relation_mapp[model]: self.ids}) # here fields, files, and related fields- # get updated but need to save the files manually diff --git a/horilla_views/generic/cbv/kanban.py b/horilla_views/generic/cbv/kanban.py new file mode 100644 index 000000000..0f8805320 --- /dev/null +++ b/horilla_views/generic/cbv/kanban.py @@ -0,0 +1,3 @@ +""" +horilla_views/generic/cbv/kanban.py +""" diff --git a/horilla_views/generic/cbv/pipeline.py b/horilla_views/generic/cbv/pipeline.py index b099e85b9..278b09869 100644 --- a/horilla_views/generic/cbv/pipeline.py +++ b/horilla_views/generic/cbv/pipeline.py @@ -32,6 +32,8 @@ class Pipeline(ListView): super().__init__(**kwargs) self.request = getattr(_thread_locals, "request", None) self.grouper = self.request.GET.get("grouper", self.grouper) + if kwargs.get("view") == "kanban": + self.template_name = "generic/pipeline/kanban.html" for allowed_field in self.allowed_fields: if self.grouper == allowed_field["field"]: self.field_model = allowed_field["model"] @@ -40,7 +42,23 @@ class Pipeline(ListView): self.parameters = allowed_field["parameters"] self.actions = allowed_field["actions"] + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + # Inject URL params into initkwargs + initkwargs_with_url = {**initkwargs, **kwargs} + self = cls(**initkwargs_with_url) + self.request = request + self.args = args + self.kwargs = kwargs + return self.dispatch(request, *args, **kwargs) + + return view + def get_queryset(self): + if self.kwargs.get("view"): + del self.kwargs["view"] + if not self.queryset: self.queryset = self.field_filter_class(self.request.GET).qs.filter( **self.kwargs diff --git a/horilla_views/generic/cbv/views.py b/horilla_views/generic/cbv/views.py index d024317ba..225ed2c75 100644 --- a/horilla_views/generic/cbv/views.py +++ b/horilla_views/generic/cbv/views.py @@ -5,16 +5,21 @@ horilla/generic/views.py import io import json import logging +import traceback from typing import Any -from urllib.parse import parse_qs +from urllib.parse import parse_qs, urlencode +import pandas as pd from bs4 import BeautifulSoup from django import forms from django.contrib import messages from django.core.cache import cache as CACHE from django.core.exceptions import FieldDoesNotExist from django.core.paginator import Page -from django.http import HttpRequest, HttpResponse, QueryDict +from django.db import transaction +from django.db.models import CharField, F +from django.db.models.functions import Cast +from django.http import HttpRequest, HttpResponse, JsonResponse, QueryDict from django.shortcuts import render from django.template.loader import render_to_string from django.urls import resolve, reverse @@ -27,14 +32,21 @@ from base.methods import closest_numbers, eval_validate, get_key_instances from horilla.filters import FilterSet from horilla.group_by import group_by_queryset from horilla.horilla_middlewares import _thread_locals +from horilla.signals import post_generic_import, pre_generic_import from horilla_views import models from horilla_views.cbv_methods import ( # update_initial_cache, + assign_related, export_xlsx, + generate_import_excel, get_short_uuid, + get_verbose_name_from_field_path, hx_request_required, paginator_qry, + resolve_foreign_keys, sortby, + split_by_import_reference, structured, + update_related, update_saved_filter_cache, ) from horilla_views.forms import DynamicBulkUpdateForm, ToggleColumnForm @@ -71,6 +83,18 @@ class HorillaListView(ListView): filter_selected: bool = True quick_export: bool = True bulk_update: bool = True + import_fields: list = [] + import_file_name: str = "Quick Import" + update_reference: str = "pk" + import_related_model_column_mapping: dict = {} + primary_key_mapping: dict = {} + import_related_column_export_mapping: dict = {} + reverse_model_relation_to_base_model: dict = {} + fk_mapping: dict = {} + import_help: dict = {} + fk_o2o_field_in_base_model: list = [] + individual_update: bool = False + o2o_related_name_mapping: dict = {} custom_empty_template: str = "" @@ -118,6 +142,19 @@ class HorillaListView(ListView): header_attrs: dict = {} + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + # Inject URL params into initkwargs + initkwargs_with_url = {**initkwargs, **kwargs} + self = cls(**initkwargs_with_url) + self.request = request + self.args = args + self.kwargs = kwargs + return self.dispatch(request, *args, **kwargs) + + return view + def post(self, *args, **kwargs): """ POST method to handle post submissions @@ -128,39 +165,58 @@ class HorillaListView(ListView): if not self.view_id: self.view_id = get_short_uuid(4) super().__init__(**kwargs) - self.ordered_ids_key = f"ordered_ids_{self.model.__name__.lower()}" + self.ordered_ids_key = f"ordered_ids_{self.model.__name__.lower()}" request = getattr(_thread_locals, "request", None) self.request = request - # # update_initial_cache(request, CACHE, HorillaListView) - # hidden columns configuration - existing_instance = models.ToggleColumn.objects.filter( - user_id=request.user, path=request.path_info - ).first() + self.visible_column = list(self.columns) - hidden_fields = ( - [] if not existing_instance else existing_instance.excluded_columns + hidden_fields = [] + existing_instance = None + if request: + existing_instance = models.ToggleColumn.objects.filter( + user_id=request.user, path=request.path_info + ).first() + if existing_instance: + hidden_fields = existing_instance.excluded_columns + + if not self.default_columns: + self.default_columns = self.columns + + self.toggle_form = ToggleColumnForm( + self.columns, self.default_columns, hidden_fields ) - self.visible_column = self.columns.copy() + # Remove hidden columns from visible_column + hidden_field_names = ( + { + col[1] if isinstance(col, tuple) else col + for col in self.columns + if col[1] in hidden_fields + } + if existing_instance + else {col[1] for col in self.columns if col not in self.default_columns} + ) + self.visible_column = [ + col + for col in self.visible_column + if (col[1] if isinstance(col, tuple) else col) not in hidden_field_names + ] - if not existing_instance: - if not self.default_columns: - self.default_columns = self.columns - self.toggle_form = ToggleColumnForm( - self.columns, self.default_columns, hidden_fields - ) - for column in self.columns: - if column not in self.default_columns: - self.visible_column.remove(column) - else: - self.toggle_form = ToggleColumnForm( - self.columns, self.default_columns, hidden_fields - ) - for column in self.columns: - if column[1] in hidden_fields: - self.visible_column.remove(column) + # Add verbose names to fields if possible + updated_column = [] + get_field = self.model()._meta.get_field + for col in self.visible_column: + if isinstance(col, str): + try: + updated_column.append((get_field(col).verbose_name, col)) + except FieldDoesNotExist: + updated_column.append(col) + else: + updated_column.append(col) + + self.visible_column = updated_column def bulk_update_accessibility(self) -> bool: """ @@ -170,6 +226,14 @@ class HorillaListView(ListView): f"{self.model._meta.app_label}.change_{self.model.__name__.lower()}" ) + def import_accessibility(self) -> bool: + """ + Accessibility method for bulk importz + """ + return self.request.user.has_perm( + f"{self.model._meta.app_label}.add_{self.model.__name__.lower()}" + ) + def serve_bulk_form(self, request: HttpRequest) -> HttpResponse: """ Bulk form serve method @@ -234,6 +298,682 @@ class HorillaListView(ListView): root_model=self.model, bulk_update_fields=self.bulk_update_fields ) + def serve_import_sheet(self, request, *args, **kwargs): + """ + Method to serve bulk import sheet + """ + if not self.import_accessibility(): + messages.info(request, "You dont have permission") + return HorillaFormView.HttpResponse() + ids = eval_validate(request.POST["selected_ids"]) + + wb = generate_import_excel( + self.model, + self.import_fields, + reference_field=self.update_reference, + import_mapping=self.import_related_column_export_mapping, + queryset=self.model.objects.filter(id__in=ids), + ) + + # Create response + response = HttpResponse( + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + ) + filename = f"{self.import_file_name}.xlsx" + response["Content-Disposition"] = f'attachment; filename="{filename}"' + wb.save(response) + return response + + def import_records(self, request, *args, **kwargs): + """ + Method to import records + """ + try: + if not self.import_accessibility(): + messages.info(request, "You dont have permission") + field_column_mapping = { + field: get_verbose_name_from_field_path( + self.model, field, self.import_related_model_column_mapping + ) + for field in self.import_fields + } + update_reference = f"{get_verbose_name_from_field_path(self.model, self.update_reference, self.import_related_model_column_mapping)} | Reference" + update_reference_key = f"{self.update_reference}_import_reference" + field_column_mapping[f"{self.update_reference}_import_reference"] = ( + update_reference + ) + + excel_file = request.FILES.get("file") + if not excel_file: + return JsonResponse({"error": "No file uploaded"}, status=400) + + df = pd.read_excel(excel_file) + + serialized = [] + field_column_mapping_values = {} + for _, row in df.iterrows(): + record = {} + for model_field, excel_col in field_column_mapping.items(): + if excel_col in row: + value = row[excel_col] + if model_field in ( + list(self.primary_key_mapping.keys()) + + list(self.import_related_model_column_mapping.keys()) + ) and not pd.isna(value): + field_column_mapping_values[model_field] = ( + field_column_mapping_values.get( + model_field, set({}) + ).union({value}) + ) + if pd.isna(value): + value = None + if isinstance(value, str): + value = value.strip() + parts = model_field.split("__") + current = record + for part in parts[:-1]: + current = current.setdefault(part, {}) + if isinstance(value, float) and value.is_integer(): + value = int(value) + current[parts[-1]] = value + serialized.append(record) + with_ref, without_ref = split_by_import_reference(serialized) + + error_records = [] + + error_records = [] + pk_values_mapping = {} + fk_values_mapping = {} + + for mapping, values in field_column_mapping_values.items(): + related_model = self.import_related_model_column_mapping[mapping] + if mapping in self.primary_key_mapping: + field = self.primary_key_mapping[mapping] + if hasattr(related_model.objects, "entire"): + existing_objects = ( + related_model.objects.entire() + .filter(**{f"{field}__in": list(values)}) + .only("pk", field) + ) + else: + existing_objects = related_model.objects.filter( + **{f"{field}__in": list(values)} + ).only("pk", field) + + existing_values = existing_objects.annotate( + field_as_str=Cast(F(field), CharField()) + ).values_list("field_as_str", flat=True) + + to_create = [ + related_model(**{field: value}) + for value in values + if str(value) not in existing_values + ] + + if to_create: + pre_generic_import.send( + sender=related_model, + records=to_create, + view=self, + ) + related_model.objects.bulk_create(to_create) + post_generic_import.send( + sender=related_model, + records=to_create, + view=self, + ) + pk_values_mapping[mapping] = pk_values_mapping.get( + mapping, [] + ) + list(existing_objects) + elif mapping in self.fk_mapping: + field = self.fk_mapping[mapping] + related_model = self.import_related_model_column_mapping[mapping] + existing_objects = related_model.objects.filter( + **{f"{field}__in": list(values)} + ).only("pk", field) + existing_values = existing_objects.values_list(field, flat=True) + to_create = [ + related_model(**{field: value}) + for value in values + if value not in existing_values + ] + if to_create: + pre_generic_import.send( + sender=related_model, + records=to_create, + view=self, + ) + related_model.objects.bulk_create(to_create) + post_generic_import.send( + sender=related_model, + records=to_create, + view=self, + ) + fk_values_mapping[mapping] = fk_values_mapping.get( + mapping, [] + ) + list(existing_objects) + if without_ref: + with transaction.atomic(): + records_to_import = [] + for record in without_ref: + try: + for reverse_field in ( + list(self.reverse_model_relation_to_base_model.keys()) + + self.fk_o2o_field_in_base_model + ): + if reverse_field in list( + self.primary_key_mapping.keys() + ) + list( + self.reverse_model_relation_to_base_model.keys() + ): + result = assign_related( + record, + reverse_field, + pk_values_mapping, + self.primary_key_mapping, + ) + record[reverse_field] = result + elif reverse_field in self.fk_mapping: + record.update( + { + reverse_field: data + for data in fk_values_mapping[reverse_field] + if getattr( + data, + self.fk_mapping[reverse_field], + None, + ) + == record[reverse_field] + } + ) + records_to_import.append(record) + + except Exception as e: + error_records.append( + { + "record": record.get(next(iter(record)), "Unknown"), + "error": str(e), + } + ) + bulk_base_fk_grouping = {} + bulk_create_reverse_related_grouping = {} + bulk_create_base_grouping = [] + items = [] + + related_fields = list( + self.reverse_model_relation_to_base_model.keys() + ) + fk_fields = self.fk_o2o_field_in_base_model + + for record in records_to_import: + if record.get(update_reference_key): + del record[update_reference_key] + instance_record = record.copy() + if update_reference_key in instance_record: + del instance_record[update_reference_key] + for relation in related_fields: + if relation in instance_record: + del instance_record[relation] + for fk_field in self.fk_o2o_field_in_base_model: + if ( + fk_field in instance_record + and fk_field not in self.fk_mapping + ): + del instance_record[fk_field] + + instance = self.model(**instance_record) + for relation in related_fields: + related_record = record[relation] + related_record[ + self.reverse_model_relation_to_base_model[relation] + ] = instance + related_instance = self.import_related_model_column_mapping[ + relation + ](**related_record) + bulk_create_reverse_related_grouping[relation] = ( + bulk_create_reverse_related_grouping.get(relation, []) + + [related_instance] + ) + + for fk_field in fk_fields: + fk_record = record[fk_field] + if isinstance(fk_record, dict): + fk_instance = self.import_related_model_column_mapping[ + fk_field + ](**fk_record) + else: + fk_instance = fk_record + bulk_base_fk_grouping[fk_field] = bulk_base_fk_grouping.get( + fk_field, [] + ) + [fk_instance] + setattr(instance, fk_field, fk_instance) + + bulk_create_base_grouping.append(instance) + + for fk_field in self.fk_o2o_field_in_base_model: + if fk_field not in self.fk_mapping: + for relation, items in bulk_base_fk_grouping.items(): + pre_generic_import.send( + sender=self.import_related_model_column_mapping[ + fk_field + ], + records=items, + view=self, + ) + + if relation not in self.fk_mapping: + pre_generic_import.send( + sender=self.import_related_model_column_mapping[ + fk_field + ], + records=items, + view=self, + ) + self.import_related_model_column_mapping[ + fk_field + ].objects.bulk_create(items) + post_generic_import.send( + sender=self.import_related_model_column_mapping[ + fk_field + ], + records=items, + view=self, + ) + + if not items: + items = bulk_create_base_grouping + pre_generic_import.send( + sender=self.model, + records=items, + view=self, + ) + + self.model.objects.bulk_create(bulk_create_base_grouping) + + post_generic_import.send( + sender=self.model, + records=items, + view=self, + ) + for related, items in bulk_create_reverse_related_grouping.items(): + pre_generic_import.send( + sender=self.import_related_model_column_mapping[related], + records=items, + view=self, + ) + self.import_related_model_column_mapping[ + related + ].objects.bulk_create(items) + post_generic_import.send( + sender=self.import_related_model_column_mapping[related], + records=items, + view=self, + ) + if with_ref: + base_instance_ids = [item["id_import_reference"] for item in with_ref] + fields = ( + list(self.reverse_model_relation_to_base_model) + + ["pk"] + + self.fk_o2o_field_in_base_model + ) + mapped_ids_queryset = ( + self.model.objects.filter(pk__in=base_instance_ids) + .only(*fields) + .values(*fields) + ) + mapped_ids_with_reverse = { + item["pk"]: { + key: item[key] + for key in list( + self.reverse_model_relation_to_base_model.keys() + ) + + self.fk_o2o_field_in_base_model + if key not in self.fk_mapping + } + for item in mapped_ids_queryset + } + field_to_update_o2o = {} + o2o_to_create = [] + o2o_create_base_mapping = {} + with transaction.atomic(): + records_to_update = [] + for record in with_ref: + try: + for reverse_field in ( + list(self.reverse_model_relation_to_base_model.keys()) + + self.fk_o2o_field_in_base_model + ): + if reverse_field in list( + self.primary_key_mapping.keys() + ) + list( + self.reverse_model_relation_to_base_model.keys() + ): + result = assign_related( + record, + reverse_field, + pk_values_mapping, + self.primary_key_mapping, + ) + if list(result.keys())[0] not in self.fk_mapping: + if ( + reverse_field + in self.fk_o2o_field_in_base_model + ): + result["main_instance_id"] = record[ + "id_import_reference" + ] + record[reverse_field] = result + else: + record[list(result.keys())[0]] = list( + result.values() + )[0] + elif reverse_field in self.fk_mapping: + record.update( + { + reverse_field: data + for data in fk_values_mapping[reverse_field] + if getattr( + data, + self.fk_mapping[reverse_field], + None, + ) + == record[reverse_field] + } + ) + records_to_update.append(record) + + except Exception as e: + logger.error(traceback.format_exc()) + error_records.append( + { + "record": record[list(record.keys())[0]], + "error": str(e), + } + ) + bulk_base_fk_grouping = {} + bulk_update_reverse_related_grouping = {} + bulk_create_from_update_reverse_related_grouping = {} + bulk_update_base_grouping = [] + + related_fields = list( + self.reverse_model_relation_to_base_model.keys() + ) + fk_fields = self.fk_o2o_field_in_base_model + related_update_fields = {} + for record in records_to_update: + instance_record = record.copy() + for relation in related_fields: + if relation in instance_record: + del instance_record[relation] + for fk_field in self.fk_o2o_field_in_base_model: + if ( + fk_field in instance_record + and fk_field not in self.fk_mapping + ): + del instance_record[fk_field] + + instance_record["id"] = instance_record["id_import_reference"] + instance_record["pk"] = instance_record["id_import_reference"] + del instance_record["id_import_reference"] + instance = self.model(**instance_record) + for relation in related_fields: + related_update_fields[relation] = record[relation].keys() + + for relation in related_fields: + related_record = record[relation] + related_record[ + self.reverse_model_relation_to_base_model[relation] + ] = instance + related_record["id"] = mapped_ids_with_reverse[instance.id][ + relation + ] + related_record["pk"] = mapped_ids_with_reverse[instance.id][ + relation + ] + related_instance = self.import_related_model_column_mapping[ + relation + ](**related_record) + if related_instance.pk is not None: + bulk_update_reverse_related_grouping[relation] = ( + bulk_update_reverse_related_grouping.get( + relation, [] + ) + + [related_instance] + ) + else: + bulk_create_from_update_reverse_related_grouping[ + relation + ] = bulk_update_reverse_related_grouping.get( + relation, [] + ) + [ + related_instance + ] + + for fk_field in fk_fields: + if fk_field not in self.fk_mapping: + fk_record = record[fk_field] + pk = mapped_ids_with_reverse[ + fk_record["main_instance_id"] + ][fk_field] + del fk_record["main_instance_id"] + if pk is None: + fk_record[ + self.o2o_related_name_mapping[fk_field] + ] = self.model(pk=pk, id=pk) + o2o_related_instance = ( + self.import_related_model_column_mapping[ + fk_field + ](**fk_record) + ) + setattr(instance, fk_field, o2o_related_instance) + o2o_to_create.append(o2o_related_instance) + continue + fk_record["pk"] = pk + fk_record["id"] = pk + if fk_field not in field_to_update_o2o: + field_to_update_o2o[fk_field] = list( + fk_record.keys() + ) + fk_instance = self.import_related_model_column_mapping[ + fk_field + ](**fk_record) + bulk_base_fk_grouping[fk_field] = ( + bulk_base_fk_grouping.get(fk_field, []) + + [fk_instance] + ) + setattr(instance, fk_field, fk_instance) + bulk_update_base_grouping.append(instance) + if with_ref and bulk_update_base_grouping: + for o2o_field, records in bulk_base_fk_grouping.items(): + field_to_update_o2o = [ + field + for field in field_to_update_o2o[o2o_field] + if field not in ["pk", "id"] + ] + related_model = self.import_related_model_column_mapping[o2o_field] + pre_generic_import.send( + sender=related_model, + records=records, + view=self, + ) + if not self.individual_update: + related_model.objects.bulk_update( + records, + field_to_update_o2o, + ) + if o2o_to_create: + pre_generic_import.send( + sender=related_model, + records=o2o_to_create, + view=self, + ) + related_model.objects.bulk_create(o2o_to_create) + post_generic_import.send( + sender=related_model, + records=o2o_to_create, + view=self, + ) + else: + if o2o_to_create: + pre_generic_import.send( + sender=related_model, + records=o2o_to_create, + view=self, + ) + related_model.objects.bulk_create(o2o_to_create) + post_generic_import.send( + sender=related_model, + records=o2o_to_create, + view=self, + ) + for o2o_instance in records: + related_model.objects.update_or_create( + id=o2o_instance.id, + defaults={ + field: getattr(o2o_instance, field) + for field in field_to_update_o2o + }, + ) + post_generic_import.send( + sender=related_model, + records=records, + view=self, + ) + field_to_update = [ + key for key in instance_record.keys() if key not in ["id", "pk"] + ] + [key for key in self.o2o_related_name_mapping] + pre_generic_import.send( + sender=self.model, + records=bulk_update_base_grouping, + view=self, + ) + if not self.individual_update: + self.model.objects.bulk_update( + bulk_update_base_grouping, field_to_update + ) + else: + for model_obj in bulk_update_base_grouping: + self.model.objects.update_or_create( + id=model_obj.id, + defaults={ + field: getattr(model_obj, field) + for field in field_to_update + }, + ) + post_generic_import.send( + sender=self.model, + records=bulk_update_base_grouping, + view=self, + ) + + if with_ref and related_update_fields: + for field in related_fields: + related_model = self.import_related_model_column_mapping[field] + if field in bulk_update_reverse_related_grouping: + update_fields = [ + key + for key in related_update_fields[field] + if key not in ["id", "pk"] + ] + pre_generic_import.send( + sender=related_model, + records=bulk_update_reverse_related_grouping[field], + view=self, + ) + if not self.individual_update: + related_model.objects.bulk_update( + bulk_update_reverse_related_grouping[field], + update_fields, + ) + else: + for ( + reverse_related_obj + ) in bulk_update_reverse_related_grouping[field]: + related_model.objects.update_or_create( + id=reverse_related_obj.id, + defaults={ + field: getattr(reverse_related_obj, field) + for field in update_fields + }, + ) + post_generic_import.send( + sender=related_model, + records=bulk_update_reverse_related_grouping[field], + view=self, + ) + + elif field in bulk_create_from_update_reverse_related_grouping: + update_fields = [ + key + for key in related_update_fields[field] + if key not in ["id", "pk"] + ] + pre_generic_import.send( + sender=related_model, + records=bulk_create_from_update_reverse_related_grouping[ + field + ], + view=self, + ) + if not self.individual_update: + related_model.objects.bulk_create( + bulk_create_from_update_reverse_related_grouping[field] + ) + else: + for ( + reverse_related_obj + ) in bulk_create_from_update_reverse_related_grouping[ + field + ]: + # reverse_related_obj.save() + related_model.objects.update_or_create( + id=reverse_related_obj.id, + defaults={ + field: getattr(reverse_related_obj, field) + for field in update_fields + }, + ) + post_generic_import.send( + sender=related_model, + records=bulk_create_from_update_reverse_related_grouping[ + field + ], + view=self, + ) + status = "Success" + if error_records: + status = "Error Found" + + return render( + request, + "cbv/import_response.html", + context={ + "view_id": self.view_id, + "status": status, + "imported": len(without_ref), + "updated": len(with_ref), + "errors": error_records[:10], # Optional: truncate if too large + "total_errors": error_records, # Optional: truncate if too large + "more_error": len(error_records) > 10, + }, + ) + except Exception as e: + traceback_message = traceback.format_exc() + error = e + return render( + request, + "cbv/import_response.html", + context={ + "view_id": self.view_id, + "status": "Error Found", + "imported": 0, + "updated": 0, + "errors": 0, # Optional: truncate if too large + "error_message": error, # Optional: truncate if too large + "traceback_message": traceback_message, # Optional: truncate if too large + }, + ) + def get_queryset(self, queryset=None, filtered=False, *args, **kwargs): if not self.queryset: self.queryset = super().get_queryset() if not queryset else queryset @@ -304,6 +1044,8 @@ class HorillaListView(ListView): def get_context_data(self, **kwargs: Any): context = super().get_context_data(**kwargs) + if not self.search_url: + self.search_url = self.request.path context["view_id"] = self.view_id context["columns"] = self.visible_column context["hidden_columns"] = list(set(self.columns) - set(self.visible_column)) @@ -433,6 +1175,28 @@ class HorillaListView(ListView): urlpatterns.append(path(self.export_path, self.export_data)) context["export_path"] = self.export_path + if self.import_fields: + get_import_sheet_path = ( + f"get-import-sheet-{self.view_id}-{self.request.session.session_key}/" + ) + post_import_sheet_path = ( + f"post-import-sheet-{self.view_id}-{self.request.session.session_key}/" + ) + urlpatterns.append( + path( + get_import_sheet_path, + self.serve_import_sheet, + ) + ) + urlpatterns.append( + path( + post_import_sheet_path, + self.import_records, + ) + ) + context["get_import_sheet_path"] = get_import_sheet_path + context["post_import_sheet_path"] = post_import_sheet_path + context["import_fields"] = self.import_fields if self.bulk_update_fields and self.bulk_update_accessibility(): get_bulk_path = ( f"get-bulk-update-{self.view_id}-{self.request.session.session_key}/" @@ -456,6 +1220,8 @@ class HorillaListView(ListView): context["bulk_update_fields"] = self.bulk_update_fields context["bulk_path"] = get_bulk_path context["export_formats"] = self.export_formats + context["import_help"] = self.import_help + context["import_accessibility"] = self.import_accessibility() return context def select_all(self, *args, **kwargs): @@ -643,7 +1409,7 @@ class HorillaListView(ListView): f'attachment; filename="{self.export_file_name}.pdf"' ) return response - return export_xlsx(json_data, columns) + return export_xlsx(json_data, columns, file_name=self.export_file_name) class HorillaSectionView(TemplateView): @@ -677,6 +1443,19 @@ class HorillaSectionView(TemplateView): context["style_static_paths"] = self.style_static_paths return context + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + # Inject URL params into initkwargs + initkwargs_with_url = {**initkwargs, **kwargs} + self = cls(**initkwargs_with_url) + self.request = request + self.args = args + self.kwargs = kwargs + return self.dispatch(request, *args, **kwargs) + + return view + @method_decorator(hx_request_required, name="dispatch") class HorillaDetailedView(DetailView): @@ -726,31 +1505,18 @@ class HorillaDetailedView(DetailView): def get_context_data(self, **kwargs: Any): context = super().get_context_data(**kwargs) - instance_ids = self.request.session.get(self.ordered_ids_key, []) - if not context.get("object", False): + obj = context.get("object") + + if not obj: return context - pk = context["object"].pk - # if instance_ids: - # context["object"].ordered_ids = instance_ids - context["instance"] = context["object"] - - url = resolve(self.request.path) - key = list(url.kwargs.keys())[0] - - url_name = url.url_name - - previous_id, next_id = closest_numbers(instance_ids, pk) - - next_url = reverse(url_name, kwargs={key: next_id}) - previous_url = reverse(url_name, kwargs={key: previous_id}) - if instance_ids: - context["instance_ids"] = str(instance_ids) - context["ids_key"] = self.ids_key - - context["next_url"] = next_url - context["previous_url"] = previous_url + pk = obj.pk + instance_ids = self.request.session.get(self.ordered_ids_key, []) + url_info = resolve(self.request.path) + url_name = url_info.url_name + key = next(iter(url_info.kwargs), "pk") + context["instance"] = obj context["title"] = self.title context["header"] = self.header context["body"] = self.body @@ -758,9 +1524,23 @@ class HorillaDetailedView(DetailView): context["action_method"] = self.action_method context["cols"] = self.cols - # CACHE.get(self.request.session.session_key + "cbv")[ - # HorillaDetailedView - # ] = context + if instance_ids: + prev_id, next_id = closest_numbers(instance_ids, pk) + context.update( + { + "instance_ids": str(instance_ids), + "ids_key": self.ids_key, + "next_url": reverse(url_name, kwargs={key: next_id}), + "previous_url": reverse(url_name, kwargs={key: prev_id}), + } + ) + + # Filter out instance_ids key from GET params + get_params = self.request.GET.copy() + get_params.pop(self.ids_key, None) + context["extra_query"] = get_params.urlencode() + else: + context["extra_query"] = "" return context @@ -773,13 +1553,28 @@ class HorillaTabView(TemplateView): view_id: str = get_short_uuid(3, "htv") template_name = "generic/horilla_tabs.html" + show_filter_tags = False tabs: list = [] + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + # Inject URL params into initkwargs + initkwargs_with_url = {**initkwargs, **kwargs} + self = cls(**initkwargs_with_url) + self.request = request + self.args = args + self.kwargs = kwargs + return self.dispatch(request, *args, **kwargs) + + return view + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) request = getattr(_thread_locals, "request", None) self.request = request + self.query_params = {} # update_initial_cache(request, CACHE, HorillaTabView) def get_context_data(self, **kwargs): @@ -790,6 +1585,14 @@ class HorillaTabView(TemplateView): ).first() if active_tab: context["active_target"] = active_tab.tab_target + + for tab in self.tabs: + base_url = tab.get("url") + query_params = {**self.request.GET.dict()} + query_params.update(self.query_params) + + tab["url"] = f"{base_url}?{urlencode(query_params)}" + context["tabs"] = self.tabs context["view_id"] = self.view_id @@ -942,6 +1745,19 @@ class HorillaCardView(ListView): ) return context + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + # Inject URL params into initkwargs + initkwargs_with_url = {**initkwargs, **kwargs} + self = cls(**initkwargs_with_url) + self.request = request + self.args = args + self.kwargs = kwargs + return self.dispatch(request, *args, **kwargs) + + return view + @method_decorator(hx_request_required, name="dispatch") class ReloadMessages(TemplateView): @@ -1082,7 +1898,6 @@ class HorillaFormView(FormView): context["form_class_path"] = self.form_class_path context["view_id"] = self.view_id context["hx_confirm"] = self.hx_confirm - context["hx_target"] = self.request.META.get("HTTP_HX_TARGET") or "this" # 855 pk = None if self.form.instance: pk = self.form.instance.pk @@ -1236,6 +2051,19 @@ class HorillaFormView(FormView): self.form = form return self.form + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + # Inject URL params into initkwargs + initkwargs_with_url = {**initkwargs, **kwargs} + self = cls(**initkwargs_with_url) + self.request = request + self.args = args + self.kwargs = kwargs + return self.dispatch(request, *args, **kwargs) + + return view + @method_decorator(hx_request_required, name="dispatch") class HorillaNavView(TemplateView): @@ -1284,7 +2112,10 @@ class HorillaNavView(TemplateView): model_instance = model_class_ref() self.nav_title = self.nav_title or model_instance._meta.verbose_name_plural - self.filter_instance = self.filter_instance.__class__() + try: + self.filter_instance = self.filter_instance.__class__() + except: + pass if not self.group_by_fields: return @@ -1347,6 +2178,19 @@ class HorillaNavView(TemplateView): # CACHE.get(self.request.session.session_key + "cbv")[HorillaNavView] = context return context + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + # Inject URL params into initkwargs + initkwargs_with_url = {**initkwargs, **kwargs} + self = cls(**initkwargs_with_url) + self.request = request + self.args = args + self.kwargs = kwargs + return self.dispatch(request, *args, **kwargs) + + return view + @method_decorator(hx_request_required, name="dispatch") class HorillaProfileView(DetailView): @@ -1439,6 +2283,19 @@ class HorillaProfileView(DetailView): return cls.tabs.index(index, tab) + @classmethod + def as_view(cls, **initkwargs): + def view(request, *args, **kwargs): + # Inject URL params into initkwargs + initkwargs_with_url = {**initkwargs, **kwargs} + self = cls(**initkwargs_with_url) + self.request = request + self.args = args + self.kwargs = kwargs + return self.dispatch(request, *args, **kwargs) + + return view + def get_context_data(self, **kwargs: Any) -> dict: context = super().get_context_data(**kwargs) context["instance"] = context["object"] diff --git a/horilla_views/models.py b/horilla_views/models.py index e6f2df5d8..123a4d67a 100644 --- a/horilla_views/models.py +++ b/horilla_views/models.py @@ -65,6 +65,10 @@ class SavedFilter(HorillaModel): path = models.CharField(max_length=256) referrer = models.CharField(max_length=256, default="") + xss_exempt_fields = [ + "urlencode", + ] + def save(self, *args, **kwargs): SavedFilter.objects.filter( is_default=True, path=self.path, created_by=self.created_by diff --git a/horilla_views/templates/cbv/import_response.html b/horilla_views/templates/cbv/import_response.html new file mode 100644 index 000000000..ec7d643d8 --- /dev/null +++ b/horilla_views/templates/cbv/import_response.html @@ -0,0 +1,69 @@ +{% load i18n %} + + + + + + + + + + + + +
StatusImportedUpdated
{{ status }}{{ imported }}{{ updated }}
+{% if traceback_message %} +
{{error_message}}
+ + + +
+ 
{{ traceback_message }}
+
+ + +{% endif %} +
+ +
diff --git a/horilla_views/templates/generic/components.html b/horilla_views/templates/generic/components.html index ef1b8b458..7a16fcd68 100644 --- a/horilla_views/templates/generic/components.html +++ b/horilla_views/templates/generic/components.html @@ -76,3 +76,5 @@ } } +{% block generic_components %} +{% endblock generic_components %} diff --git a/horilla_views/templates/generic/horilla_form.html b/horilla_views/templates/generic/horilla_form.html index 23d89a5b5..1c921f97f 100644 --- a/horilla_views/templates/generic/horilla_form.html +++ b/horilla_views/templates/generic/horilla_form.html @@ -1,76 +1,80 @@ {% load generic_template_filters %} -
- {% for field_tuple in dynamic_create_fields %} - + {% for field_tuple in dynamic_create_fields %} + +
+ {% endfor %} +
{{form.structured}}
+ {% for field_tuple in dynamic_create_fields %} +
+ + +
- - - {{form.structured}} - - -{% for field_tuple in dynamic_create_fields %} -
- - - - - - - -
-{% endfor %} + + + + + + {% endfor %} diff --git a/horilla_views/templates/generic/horilla_list_table.html b/horilla_views/templates/generic/horilla_list_table.html index b7cca8af8..53bec7adc 100644 --- a/horilla_views/templates/generic/horilla_list_table.html +++ b/horilla_views/templates/generic/horilla_list_table.html @@ -362,6 +362,7 @@

{% trans "No Records found" %}

{% trans "No records found." %} + {% include "generic/import_block.html" %}

diff --git a/horilla_views/templates/generic/horilla_section.html b/horilla_views/templates/generic/horilla_section.html index a7b231fcd..c4a5054f1 100644 --- a/horilla_views/templates/generic/horilla_section.html +++ b/horilla_views/templates/generic/horilla_section.html @@ -21,6 +21,7 @@ > +{% if nav_url %}
+{% endif %} +{% if view_url %}
+{% endif %} {% endblock content %} diff --git a/horilla_views/templates/generic/import_block.html b/horilla_views/templates/generic/import_block.html new file mode 100644 index 000000000..e657da34d --- /dev/null +++ b/horilla_views/templates/generic/import_block.html @@ -0,0 +1,82 @@ +{% load generic_template_filters i18n %} + +{% if import_fields and import_accessibility %} +
+ {% trans 'Import' %} + () +
+
+
+
+

Import Records

+ +
+
+ +
+
+
+ + +
+ + + Download Template + +
+
+
+ {% if import_help %} +
{% trans 'Import Help' %}
+
+ {% for key in import_help %} +
+ {{ key }} + {% for val in import_help|get_item:key %} +
  • {{ val }}
    • + {% endfor %} +
    + {% endfor %} +
    + {% endif %} +
    + +
    +
    +
    + {% csrf_token %} + + +
    +
    +
    +
    +
    +{% endif %} diff --git a/horilla_views/templates/generic/pipeline/kanban.html b/horilla_views/templates/generic/pipeline/kanban.html new file mode 100644 index 000000000..17148c336 --- /dev/null +++ b/horilla_views/templates/generic/pipeline/kanban.html @@ -0,0 +1,575 @@ +{% load i18n recruitmentfilters %} +{% load generic_template_filters %} +
    +
    + {% for group in groups %} +
    +
    +
    +

    {{ group|stringformat:"s"|truncatechars:22 }}

    +
    +
    + 2
    + {% if actions %} +
    + +
    + {% for action in actions %} + {% if action.accessibility|accessibility:group %} + {{action.action}} + {% endif %} + {% endfor %} +
    +
    + {% endif %} +
    +
    +
    +
    +
    +
    Design Homepage
    +
    Create wireframes and mockups for the new homepage design.
    +
    + High Priority +
    + May 16 +
    +
    +
    +
    +
    +
    JD
    + John Doe +
    +
    +
    AS
    + Alice Smith +
    +
    +
    RJ
    + Robert Johnson +
    +
    +
    +
    + Unassigned +
    +
    +
    +
    +
    +
    +
    +
    Update Documentation
    +
    Review and update the API documentation for v2.0 release.
    +
    + Medium Priority +
    + May 20 +
    +
    AS
    +
    +
    +
    JD
    + John Doe +
    +
    +
    AS
    + Alice Smith +
    +
    +
    RJ
    + Robert Johnson +
    +
    +
    +
    + Unassigned +
    +
    +
    +
    +
    +
    +
    + Add new task
    +
    +
    + {% endfor %} +
    +
    + + + + diff --git a/horilla_views/templates/generic/quick_actions.html b/horilla_views/templates/generic/quick_actions.html index 8ed5025fa..7930c36bf 100644 --- a/horilla_views/templates/generic/quick_actions.html +++ b/horilla_views/templates/generic/quick_actions.html @@ -1,4 +1,4 @@ -{% load i18n %} +{% load i18n generic_template_filters %} {% if request.actual_ids and request.session.prev_path == request.path %}