old_console/networkmap/api.py

import json
import logging
import re
from datetime import datetime
from distutils.util import strtobool
from ipaddress import ip_address

from django.conf import settings
from django.contrib.auth.decorators import login_required, permission_required
from django.db.models import Q
from django.http import JsonResponse, HttpResponse
from django.template.context_processors import csrf
from elasticsearch import Elasticsearch
from rest_framework import status
from rest_framework.decorators import action, api_view
from rest_framework.mixins import ListModelMixin, UpdateModelMixin, RetrieveModelMixin
from rest_framework.viewsets import GenericViewSet

from assets.models.assets import Asset, AssetListGroup
from console.conslog import object_create_log
from console.models import Connection
from core.decorators import log_url
from core.mixins import ApiPermissionCheckMixin
from core.utils import dtnow
from networkmap.services import parse_endpoint_celery_done_events
from events.constants import ELK_HOST, ELK_PORT, ELK_LOGIN, ELK_PASS
from incident.models import Incident
from networkmap.models import NetworkMap, NetworkMapBackgroundImage
from networkmap.serializers import (UserMapSerializer, NetmapElementsSerializer, NetmapGroupSerializer,
                                    AssetDangerSerializer, UserMapNamesSerializer, AutoNetmapElementsSerializer,
                                    AutoNetmapConnectionsSerializer, NetworkMapBackgroundImagesSerializer)
from perms.models import Perm

_log = logging.getLogger(__name__)

ELK_SEARCH_INDEX = getattr(settings, 'ELK_AGGREGATED_INDEX', 'aggregated-*')
# TODO: rename "Firerwall" to "Firewall" when vector variable will be fixed
# Yes its a mistake and yes it is correct for now
ELK_FIREWALL_DEVICE_PRODUCT = getattr(settings, 'ELK_FIREWALL_PRODUCT_NAME', 'Industrial Firerwall')
ELK_ENDPOINT_DEVICE_PRODUCT = getattr(settings, 'ELK_ENDPOINT_PRODUCT_NAME', 'Industrial Endpoint')

DANGER_STATUSES = [
    Incident.Status.NOT_ASSIGNED,
    Incident.Status.ASSIGNED,
    Incident.Status.DELAYED,
    Incident.Status.FALSE_ALARM,
]


class UserMapViewSet(ApiPermissionCheckMixin,
                     ListModelMixin,
                     UpdateModelMixin,
                     RetrieveModelMixin,
                     GenericViewSet):
    console_permissions = [Perm.can_view_network]
    message = 'Cannot access user network map API. Permission denied'

    serializer_class = UserMapSerializer
    queryset = NetworkMap.objects.all()


class NetworkMapBackgroundImagesViewSet(ApiPermissionCheckMixin,
                                        ListModelMixin,
                                        UpdateModelMixin,
                                        RetrieveModelMixin,
                                        GenericViewSet):
    """ ViewSet for handling the networkmap background image data manipulation """
    console_permissions = [Perm.can_view_network]
    message = 'Cannot access user network map API. Permission denied'
    serializer_class = NetworkMapBackgroundImagesSerializer

    def get_queryset(self):
        """ This method override is used for including linked to background image network map data """
        network_map_id = self.request.query_params.get('current_map_id')
        queryset = NetworkMapBackgroundImage.objects.filter(network_map=network_map_id)
        return queryset

    def try_to_get_background_image(self, map_pk, image_data, full_image_data=True):
        """ Helper method for trying to get an image from DB
        :param map_pk: linked for current background image network map pk
        :param image_data: image data, which is needed for request
        :param full_image_data: flag to show if image data that are being provided for request is:
            if True - image data provided in cytoscape module for handling background images format - support images
            if False - provided data will not be checked in this helper function, thus needed to be validated in further
                actions
        :return: Two variables:
            1. True if image exists ind DB, False otherwise
            2. If first variable is True - requested image instance,
               otherwise - JsonResponse with error status and message
        """
        if map_pk is None or image_data is None:
            return False, JsonResponse({'status': 'err', 'error_message': 'Some mandatory arguments are missing'})
        # Check if provided image data is valid
        if full_image_data:
            try:
                background_image_data = json.loads(image_data)
            except json.JSONDecodeError:
                _log.error(f"Failed to parse background image data for network map {map_pk}")
                return False, JsonResponse({'status': 'err', 'error_message': 'Image data is not valid'})
        else:
            background_image_data = image_data
        # Get requested image
        try:
            if full_image_data:
                target_image = NetworkMapBackgroundImage.objects.get(network_map=map_pk,
                                                                     name=background_image_data['name'])
            else:
                target_image = NetworkMapBackgroundImage.objects.get(pk=background_image_data)
        except NetworkMapBackgroundImage.DoesNotExist:
            return False, JsonResponse({'status': 'err', 'error_message': 'Image does not exists in database'})

        return True, target_image

    @action(detail=False, methods=["POST"], name="update_image_data")
    def update_image_data(self, request):
        """ Method for handling <router-url>-update-image-data request for updating background image data fields value
        in Database. Request always should contain 'current_map_id' and 'image_data' fields.
        :param request: request instance that must contains 'current_map_id' and 'image_data' fields. Image_data should
            be in support images format (cytoscape framework that is used for managing network map background images)
        :return: JsonResponse with 'ok' status if everything works as expected, or with error message and 'err' status
            otherwise
        """
        # Get necessary arguments for updating image data
        try:
            network_map_id = int(self.request.POST.get('current_map_id'))
            background_image_data = self.request.POST.get('image_data')
        except TypeError:
            return JsonResponse(
                {'status': 'err', 'error_message': 'Necessary data for request not provided or corrupted'})
        success_get_status, image_or_error = self.try_to_get_background_image(network_map_id, background_image_data)
        if success_get_status:
            parsed_image_data = json.loads(background_image_data)
            image_or_error.bounds = parsed_image_data['bounds']
            image_or_error.locked = parsed_image_data['locked']
            image_or_error.visible = parsed_image_data['visible']
            image_or_error.save()
            return JsonResponse({'status': 'ok'})
        else:
            return image_or_error

    @action(detail=False, methods=["POST"], name="delete_background_image")
    def delete_background_image(self, request):
        """ Method for handling <router-url>-delete-background-image request for deleting existing background image
        for current network map. Request always should contain 'current_map_id' and 'image_data' fields.
        :param request: request instance that must contains 'current_map_id' and 'image_data' fields. Image_data must
            contain background image ID
        :return: JsonResponse with 'ok' status if everything works as expected, or with error message and 'err' status
            otherwise
                """
        # Get necessary arguments for updating image data
        try:
            network_map_id = int(self.request.POST.get('current_map_id'))
            background_image_data = int(self.request.POST.get('image_data'))
        except TypeError:
            return JsonResponse(
                {'status': 'err', 'error_message': 'Necessary data for request not provided or corrupted'})
        success_get_status, image_or_error = self.try_to_get_background_image(network_map_id,
                                                                              background_image_data,
                                                                              False)
        if success_get_status:
            image_or_error.delete()
            return JsonResponse({'status': 'ok'})
        else:
            return image_or_error


class NetmapElementsViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet):
    console_permissions = [Perm.can_view_network]
    pagination_class = None
    serializer_class = NetmapElementsSerializer
    queryset = Asset.objects.all()


class AutoNetmapElementsViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet):
    pagination_class = None
    serializer_class = AutoNetmapElementsSerializer
    console_permissions = [Perm.can_view_network]
    queryset = Asset.objects.all()


class AutoNetmapConnectionsViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet):
    pagination_class = None
    serializer_class = AutoNetmapConnectionsSerializer
    console_permissions = [Perm.can_view_network]
    queryset = Connection.objects.all()


class NetmapGroupsViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet):
    console_permissions = [Perm.can_view_network]
    pagination_class = None
    serializer_class = NetmapGroupSerializer
    queryset = AssetListGroup.objects.filter(asset__isnull=False).distinct()


class AssetDangerViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet):
    console_permissions = [Perm.can_view_network]
    pagination_class = None
    serializer_class = AssetDangerSerializer

    def get_queryset(self):
        queryset = Asset.objects.filter(incidents__status__in=DANGER_STATUSES).distinct()
        return queryset


# def add_new_user_map(request):
#     if request.method == 'POST':
#         form = AddNewUserMapForm(request.POST, request.FILES, user=request.user)
#         if form.is_valid():
#             new_map = form.save()
#             serializer = UserMapSerializer(new_map)
#             return JsonResponse(serializer.data, safe=False)
#     else:
#         form = AddNewUserMapForm()
#     ctx = {}
#     ctx.update(csrf(request))
#     form = render_crispy_form(form, context=ctx)
#     return JsonResponse({'status': 'err', 'form_html': form})


# def add_new_background_image(request, current_netmap_id):
#     """ API for managing the new network map background image form
#     :param request: request instance
#     :param current_netmap_id: current network map id, on which is performing the background image addition
#     :return: JsonResponse with 'ok' status if everything performed as expected and new background image is being added,
#         JsonResponse with 'err' status and HTML rendered form with errors otherwise
#     """
#     try:
#         netmap = NetworkMap.objects.get(pk=current_netmap_id)
#     except NetworkMap.DoesNotExist:
#         return JsonResponse({'status': 'err', 'error_message': 'Unknown networkmap selected'})
#     if request.method == 'POST':
#         form = AddNewBackgroundImageForm(request.POST, request.FILES,
#                                          networkmap=netmap)
#         if form.is_valid():
#             form.save()
#             return JsonResponse({'status': 'ok'})
#         else:
#             _log.error(form.errors)
#     else:
#         form = AddNewBackgroundImageForm()
#     ctx = {}
#     ctx.update(csrf(request))
#     form = render_crispy_form(form, context=ctx)
#     return JsonResponse({'status': 'err', 'form_html': form})


@api_view(['GET'])
@permission_required(Perm.perm_req(Perm.can_view_network), raise_exception=True)
def get_user_maps(request):
    """ API for getting network maps for specific user
    :param request: request object
    :return: list of group names for current user
    """
    all_maps = NetworkMap.objects.filter(Q(user=request.user) | Q(shared_map=True))
    serializer = UserMapNamesSerializer(all_maps, many=True)
    return JsonResponse(serializer.data, safe=False)


@api_view(['GET'])
@permission_required(Perm.perm_req(Perm.can_view_network), raise_exception=True)
def get_map_background_image(request, background_pk):
    try:
        bimage = NetworkMapBackgroundImage.objects.get(pk=background_pk)
    except NetworkMapBackgroundImage.DoesNotExist:
        return JsonResponse({'status': 'err', 'error_message': 'Background image does not exists'})
    if not bimage.url:
        return JsonResponse({'status': 'err', 'error_message': 'No link for selected background image'})
    else:
        return HttpResponse(bimage.url, content_type='image/png')


@api_view(['DELETE'])
@permission_required(Perm.perm_req(Perm.can_view_network), raise_exception=True)
def delete_user_map(request, pk):
    try:
        target_netmap = NetworkMap.objects.get(pk=pk)
    except NetworkMap.DoesNotExist:
        return JsonResponse({'message': 'The networkmap does not exist'}, status=status.HTTP_404_NOT_FOUND)

    target_netmap.delete()
    return JsonResponse({'message': 'Networkmap was deleted successfully!'}, status=status.HTTP_200_OK)


def auto_netmap_assets_filter(form_assets_data, include_neighbours, assets_queryset=None, connections_queryset=None):
    """ Asset filtering function for auto network map. Function returns list of two querysets [Filtered_assets, Filtered_connections]
        which can be used in further filtering or send to frontend for rendering the map
    :param form_assets_data: data, provided by user
    :param assets_queryset: assets queryset from previous filters, if exists
    :param connections_queryset: connection queryset from previous filters, if exists
    :param include_neighbours: argument to tell if assets, which have connections with selected assets, should be displayed
    :return: list with assets and connections filtered querysets
    """
    chosen_assets_queryset = Asset.objects.filter(pk__in=form_assets_data)
    if not connections_queryset:
        connections_queryset = Connection.objects.all()
    # Get connections for selected assets
    edge_assets = []
    if include_neighbours:
        filtered_connections_queryset = connections_queryset.filter(
            Q(src_asset__in=chosen_assets_queryset) | Q(dst_asset__in=chosen_assets_queryset))
        for asset in chosen_assets_queryset:
            src_connections = filtered_connections_queryset.filter(src_asset=asset)
            dst_connections = filtered_connections_queryset.filter(dst_asset=asset)
            for connection in src_connections:
                edge_assets.append(connection.dst_asset.pk)
            for connection in dst_connections:
                edge_assets.append(connection.src_asset.pk)
    else:
        filtered_connections_queryset = connections_queryset.filter(
            Q(src_asset__in=chosen_assets_queryset) & Q(dst_asset__in=chosen_assets_queryset))
    edged_assets = Asset.objects.filter(pk__in=edge_assets)
    filtered_assets_queryset = chosen_assets_queryset.union(edged_assets)
    return [filtered_assets_queryset, filtered_connections_queryset]


def auto_netmap_time_filter(form_time_data, connections_queryset='no_data'):
    """ Time filtering function for auto network map. Function returns list of two querysets [Filtered_assets, Filtered_connections]
        which can be used in further filtering or send to frontend for rendering the map
    :param form_time_data: data, provided by user
    :param connections_queryset: connection queryset from previous filters, if exists
    :return: list with assets and connections filtered querysets
    """
    splitted_range = form_time_data.split(' - ')
    formatted_range = []
    for timestamp in splitted_range:
        if '.' in timestamp:
            formatted_range.append(datetime.strptime(timestamp, '%d.%m.%Y %H:%M:%S'))
        else:
            formatted_range.append(datetime.fromisoformat(timestamp))

    if connections_queryset == 'no_data':
        connections_queryset = Connection.objects.all()
    filtered_connections = connections_queryset.filter(updated__range=formatted_range)
    assets_set = []
    for connection in filtered_connections:
        assets_set.append(connection.src_asset.pk)
        assets_set.append(connection.dst_asset.pk)
    filtered_assets = Asset.objects.filter(pk__in=list(assets_set))
    return [filtered_assets, filtered_connections]


def auto_netmap_protocol_filter(form_protocol_data, connections_queryset='no_data'):
    """ Protocol filtering function for auto network map. Function returns list of two querysets [Filtered_assets, Filtered_connections]
        which can be used in further filtering or send to frontend for rendering the map
    :param form_time_data: data, provided by user
    :param connections_queryset: connection queryset from previous filters, if exists
    :return: list with assets and connections filtered querysets
    """
    if connections_queryset == 'no_data':
        connections_queryset = Connection.objects.all()

    pattern = re.compile(r'^\w+$')  # allow any word character (equivalent to [a-zA-Z0-9_])
    form_protocol_data = [protocol for protocol in form_protocol_data if pattern.match(protocol)]

    filtered_connections = connections_queryset.filter(connection_protocol__in=form_protocol_data)
    assets_set = []
    for connection in filtered_connections:
        assets_set.append(connection.src_asset.pk)
        assets_set.append(connection.dst_asset.pk)
    filtered_assets = Asset.objects.filter(pk__in=list(assets_set))
    return [filtered_assets, filtered_connections]


def apply_auto_netmap_filters(parsed_filters):
    # Next two 'if's are made for parsing JSON converting issues from ajax
    if 'assets_filter[assets][]' in parsed_filters:
        parsed_filters['assets_filter'] = {'assets': parsed_filters.pop('assets_filter[assets][]'),
                                           'include_neighbours': bool(
                                               strtobool(parsed_filters.pop('assets_filter[include_neighbours]')[0]))}
    if 'protocol_filter[]' in parsed_filters:
        parsed_filters['protocol_filter'] = parsed_filters.pop('protocol_filter[]')
    filters_querydicts = [Asset.objects.all(), Connection.objects.all()]
    if 'assets_filter' in parsed_filters:
        chosen_assets = parsed_filters.get('assets_filter').get('assets')
        include_neighbours = parsed_filters.get('assets_filter').get('include_neighbours')
        filters_querydicts = auto_netmap_assets_filter(chosen_assets, include_neighbours)
    if 'time_filter' in parsed_filters:
        filters_querydicts = auto_netmap_time_filter(parsed_filters.get('time_filter'), filters_querydicts[1])
    if 'protocol_filter' in parsed_filters:
        filters_querydicts = auto_netmap_protocol_filter(parsed_filters.get('protocol_filter'), filters_querydicts[1])
    serialized_assets = AutoNetmapElementsSerializer(filters_querydicts[0], many=True)
    serialized_connections = AutoNetmapConnectionsSerializer(filters_querydicts[1], many=True)
    context = {
        'edges': serialized_connections.data,
        'nodes': serialized_assets.data,
    }
    return context


# @log_url
# @login_required
# @permission_required(Perm.perm_req(Perm.can_view_network), raise_exception=True)
# def handle_filter_forms(request, form_type):
#     if form_type == 'assets':
#         filter_form = FilterByAssetForm
#     elif form_type == 'time':
#         filter_form = FilterByTimeForm
#     elif form_type == 'protocol':
#         filter_form = FilterByProtocolForm
#     elif form_type == 'reset':
#         return JsonResponse(apply_auto_netmap_filters(request.GET.copy()), safe=False)
#     else:
#         _log.error(f"Unknown filter type for auto network map: {form_type}")
#
#     if request.method == 'POST':
#         form = filter_form(request.POST)
#         if form.is_valid():
#             current_auto_netmap_filters = form.cleaned_data['current_filters']
#             context = apply_auto_netmap_filters(current_auto_netmap_filters)
#             return JsonResponse(context, safe=False)
#         else:
#             return JsonResponse({'status': 'err', 'error_message': 'no_entries_selected'})
#     else:
#         form = filter_form()
#     ctx = {}
#     ctx.update(csrf(request))
#     form = render_crispy_form(form, context=ctx)
#     return JsonResponse({'status': 'err', 'form_html': form})


def parse_firewall_celery_done_events(hit):
    """ Algorithm for forming connections data for the auto network map """
    if hit['_source']['source_ip'] and hit['_source']['destination_ip']:
        source_ip = hit['_source']['source_ip']
        destination_ip = hit['_source']['destination_ip']
        log_type = hit['_source']['type']
        connection_protocol = hit['_source']['event_protocol']

        try:
            ip_address(source_ip)
            ip_address(destination_ip)
        except ValueError:
            _log.critical(
                f"Error incorrect ip format [event id: {hit.get('_id')} by source_ip {source_ip} \
                  or dest_ip {destination_ip}]")
            return

        # Check if Assets with corresponding IP's presented in database. If not - add them
        if not Asset.objects.filter(ip=source_ip, sensor=log_type).exists():
            source_asset = Asset.objects.create(name=source_ip, ip=source_ip, sensor=log_type)
            object_create_log(source_asset.name, Asset.Meta.__name__)
        if not Asset.objects.filter(ip=destination_ip, sensor=log_type).exists():
            dest_asset = Asset.objects.create(name=destination_ip, ip=destination_ip, sensor=log_type)
            object_create_log(dest_asset.name, Asset.Meta.__name__)
        # ip_list = [source_ip, destination_ip]
        # # Checking if there is a connection in db with the same IP's as in the event
        # if Connection.objects.filter(src_asset__ip__in=ip_list, dst_asset__ip__in=ip_list).exists():
        #     # If there is a match during filtering process - update field Connection.updated
        #     try:
        #         connection_from_db = Connection.objects.get(src_asset__ip=source_ip,
        #                                                     src_asset__sensor=log_type,
        #                                                     dst_asset__ip=destination_ip,
        #                                                     dst_asset__sensor=log_type)
        #         # Add updated connection to the network map data
        #         connection_from_db.updated = dtnow()
        #         connection_from_db.save()
        #     except Connection.DoesNotExist:
        #         connection_from_db = Connection.objects.get(dst_asset__ip=source_ip,
        #                                                     dst_asset__sensor=log_type,
        #                                                     src_asset__ip=destination_ip,
        #                                                     src_asset__sensor=log_type)
        #         # Add updated connection to the network map data
        #         connection_from_db.updated = dtnow()
        #         connection_from_db.save()
        # else:
        #     # Creating a new connection if there is no in DB
        #     received_protocol = connection_protocol.upper()
        #     if received_protocol in Connection.ProtocolType:
        #         Connection.objects.create(src_asset=Asset.objects.get(ip=source_ip, sensor=log_type),
        #                                   dst_asset=Asset.objects.get(ip=destination_ip, sensor=log_type),
        #                                   connection_protocol=received_protocol)
        #     else:
        #         Connection.objects.create(src_asset=Asset.objects.get(ip=source_ip, sensor=log_type),
        #                                   dst_asset=Asset.objects.get(ip=destination_ip, sensor=log_type))


# TODO: rename to summary celery_done corresponding name
def update_connections_model_data():
    """ Algorithm for forming connections data for the auto network map """
    # Receiving data from Elasticsearch
    es = Elasticsearch([{'host': ELK_HOST, 'port': ELK_PORT}], http_auth=(ELK_LOGIN, ELK_PASS))
    search_body = {
        "query": {
            "match": {
                "celery_done": "false"
            }
        }
    }
    es_search = es.search(index=ELK_SEARCH_INDEX, body=search_body, filter_path=['hits.hits._*'], size=1000)
    if 'hits' in es_search:
        for hit in es_search['hits']['hits']:
            if hit['_source']['device_product'] == ELK_ENDPOINT_DEVICE_PRODUCT:
                parse_endpoint_celery_done_events(hit)
            elif hit['_source']['device_product'] == ELK_FIREWALL_DEVICE_PRODUCT:
                parse_firewall_celery_done_events(hit)
            # Updating 'celery_done field
            hit_id = hit.get('_id')
            update_body = {
                "script": {
                    "source": "ctx._source.celery_done=true",
                    "lang": "painless"
                },
                "query": {
                    "match": {
                        "_id": hit_id
                    }
                }
            }
            # Updating ELK event data
            es.update_by_query(index=ELK_SEARCH_INDEX, body=update_body)