added dynamic table that loads via AJAX. This should support thousands of rows without issue

requirements.txt

@@ -33,6 +33,7 @@ mockito==1.2.2
 nexmo==2.3.0
 numpy==1.16.1
 pandas==1.1.3
+django-datatables-view==1.19.1
 phonenumberslite==8.9.14
 Pillow==3.4.2
 psycopg2==2.8.6

smash/web/api_urls.py

@@ -16,7 +16,7 @@ Including another URLconf
 from django.conf.urls import url
 
 from web.api_views import worker, location, subject, appointment_type, appointment, configuration, daily_planning, \
-    redcap, flying_team, visit, voucher, voucher_type
+    redcap, flying_team, visit, voucher, voucher_type, provenance
 
 urlpatterns = [
     # appointments
@@ -35,6 +35,8 @@ urlpatterns = [
     # subjects data
     url(r'^cities$', subject.cities, name='web.api.cities'),
     url(r'^referrals$', subject.referrals, name='web.api.referrals'),
+    url(r'^export_log$', provenance.ExportLog.as_view(), name='web.api.export_log'),
+    url(r'^provenances$', provenance.CompleteLog.as_view(), name='web.api.provenances'),
     url(r'^subjects/(?P<type>[A-z]+)$', subject.subjects, name='web.api.subjects'),
     url(r'^subjects:columns/(?P<subject_list_type>[A-z]+)$', subject.get_subject_columns,
         name='web.api.subjects.columns'),

smash/web/api_views/provenance.py

+from ..models import Provenance, Worker
+from django_datatables_view.base_datatable_view import BaseDatatableView
+
+class ExportLog(BaseDatatableView):
+    model = Provenance
+    columns = [
+        'modification_date',
+        'modification_author',
+        'modification_description',
+        'request_path',
+        'request_ip_addr'
+    ]
+
+    # define column names that will be used in sorting
+    # order is important and should be same as order of columns
+    # displayed by datatables. For non sortable columns use empty
+    # value like ''
+    order_columns = [
+        'modification_date',
+        'modification_author',
+        'modification_description',
+        'request_path',
+        'request_ip_addr'
+    ]
+    
+    # set max limit of records returned, this is used to protect our site if someone tries to attack our site
+    # and make it return huge amount of data
+    max_display_length = 10
+
+    def filter_queryset(self, qs):
+        # simple example:
+        search = self.request.GET.get('search[value]', None)
+        if search != '' and search is not None:
+            inner_qs = Worker.objects.filter(first_name__icontains=search) | Worker.objects.filter(last_name__icontains=search)
+            qs = qs.filter(modification_author__in=inner_qs) | qs.filter(modification_description__icontains=search) | qs.filter(request_path__icontains=search) | qs.filter(request_ip_addr__icontains=search)
+        
+        qs = qs & qs.filter(modified_table__isnull=True, 
+                            previous_value__isnull=True, 
+                            new_value__isnull=True, 
+                            modified_field='', 
+                            request_path__startswith='/export/')
+
+        return qs
+
+class CompleteLog(BaseDatatableView):
+    model = Provenance
+    columns = ['modified_table',
+        'modified_table_id',
+        'modification_date',
+        'modification_author',
+        'modified_field',
+        'previous_value',
+        'new_value',
+        'modification_description',
+        'request_path',
+        'request_ip_addr'
+    ]
+
+    # define column names that will be used in sorting
+    # order is important and should be same as order of columns
+    # displayed by datatables. For non sortable columns use empty
+    # value like ''
+    order_columns = ['modified_table',
+        'modified_table_id',
+        'modification_date',
+        'modification_author',
+        'modified_field',
+        'previous_value',
+        'new_value',
+        'modification_description',
+        'request_path',
+        'request_ip_addr'
+    ]
+    
+    # set max limit of records returned, this is used to protect our site if someone tries to attack our site
+    # and make it return huge amount of data
+    max_display_length = 10
+
+    def filter_queryset(self, qs):
+        # simple example:
+        search = self.request.GET.get('search[value]', None)
+        if search != '' and search is not None:
+            inner_qs = Worker.objects.filter(first_name__icontains=search) | Worker.objects.filter(last_name__icontains=search)
+            qs = qs.filter(modification_author__in=inner_qs) | qs.filter(modification_description__icontains=search) | qs.filter(request_path__icontains=search) | qs.filter(request_ip_addr__icontains=search)
+
+        return qs
\ No newline at end of file

smash/web/templates/export/index.html

@@ -105,20 +105,15 @@
                     <th>Author</th>
                     <th>Description</th>
                     <th>Request Path</th>
+                    <th>Request IP</th>
                 </tr>
                 </thead>
                 <tbody>
-                {% for provenance in provenances %}
-                    <tr>
-                        <td data-sort="{{ provenance.modification_date | timestamp }}">{{ provenance.modification_date }}</td>
-                        <td>{{ provenance.modification_author }}</td>
-                        <td>{{ provenance.modification_description }}</td>
-                        <td>{{ provenance.request_path }}</td>
-                    </tr>
-                {% endfor %}
+
                 </tbody>
             </table>
         </div>
+
     </div>
 
     <div class="box-body">
@@ -139,6 +134,9 @@
                 $(e.target).children('i.fa').toggleClass('fa-expand');
                 $(e.target).children('i.fa').toggleClass('fa-compress');
             });
+
+
+
         };
         function addFields(e, cls){
             var fields = $(`.${cls} > li > input:checked`).map(function(i,e){
@@ -161,14 +159,42 @@
         }
 
         $(function () {
-            $('#table').DataTable({
-                "paging": true,
-                "lengthChange": false,
-                "searching": true,
-                "ordering": true,
-                "order": [[ 0, "desc" ]],
-                "info": true,
-                "autoWidth": false
+            console.log("{% url 'web.api.provenances' %}")
+            var oTable = $('#table').DataTable({
+                processing: true,
+                serverSide: true,
+                ordering: true,
+                autoWidth: false,
+                lengthChange: false,
+                columns: [
+                    {
+                        data: 'modification_date',
+                        orderable: true,
+                        searchable: true
+                    },
+                    {
+                        data: 'modification_author',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'modification_description',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'request_path',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'request_ip_addr',
+                        orderable: true,
+                        searchable: true,
+                    }
+                ],
+                order: [[ 0, "desc" ]],
+                ajax: "{% url 'web.api.export_log' %}"
             });
         });
     </script>

smash/web/templates/provenance/list.html

@@ -31,22 +31,10 @@
                 <th>New Value</th>
                 <th>Description</th>
                 <th>Request Path</th>
+                <th>Request IP</th>
             </tr>
             </thead>
             <tbody>
-            {% for provenance in provenances %}
-                <tr>
-                    <td>{{ provenance.modified_table }}</td>
-                    <td>{{ provenance.modified_table_id }}</td>
-                    <td data-sort="{{ provenance.modification_date | timestamp }}">{{ provenance.modification_date }}</td>
-                    <td>{{ provenance.modification_author }}</td>
-                    <td>{{ provenance.modified_field }}</td>
-                    <td>{{ provenance.previous_value }}</td>
-                    <td>{{ provenance.new_value }}</td>
-                    <td>{{ provenance.modification_description }}</td>
-                    <td>{{ provenance.request_path }}</td>
-                </tr>
-            {% endfor %}
             </tbody>
         </table>
     </div>
@@ -61,13 +49,68 @@
     <script>
         $(function () {
             $('#table').DataTable({
-                "paging": true,
-                "lengthChange": false,
-                "searching": true,
-                "ordering": true,
-                "order": [[ 2, "desc" ]],
-                "info": true,
-                "autoWidth": false
+                paging: true,
+                lengthChange: false,
+                searching: true,
+                processing: true,
+                serverSide: true,
+                ordering: true,
+                order: [[ 2, "desc" ]],
+                info: true,
+                autoWidth: false,
+                columns: [
+                    {
+                        data: 'modified_table',
+                        orderable: true,
+                        searchable: true
+                    },
+                    {
+                        data: 'modified_table_id',
+                        orderable: true,
+                        searchable: true
+                    },
+                    {
+                        data: 'modification_date',
+                        orderable: true,
+                        searchable: true
+                    },
+                    {
+                        data: 'modification_author',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'modified_field',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'previous_value',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'new_value',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'modification_description',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'request_path',
+                        orderable: true,
+                        searchable: true,
+                    },
+                    {
+                        data: 'request_ip_addr',
+                        orderable: true,
+                        searchable: true,
+                    }
+                ],
+                ajax: "{% url 'web.api.provenances' %}"
             });
         });
     </script>

smash/web/utils.py

@@ -5,6 +5,13 @@ from datetime import timedelta
 
 from web.algorithm import VerhoeffAlgorithm, LuhnAlgorithm
 
+def get_client_ip(request):
+    x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
+    if x_forwarded_for:
+        ip = x_forwarded_for.split(',')[0]
+    else:
+        ip = request.META.get('REMOTE_ADDR')
+    return ip
 
 def is_valid_social_security_number(number):
     if number is not None and number != '':