import json import logging import re from datetime import datetime from distutils.util import strtobool from ipaddress import ip_address from django.conf import settings from django.contrib.auth.decorators import login_required, permission_required from django.db.models import Q from django.http import JsonResponse, HttpResponse from django.template.context_processors import csrf from elasticsearch import Elasticsearch from rest_framework import status from rest_framework.decorators import action, api_view from rest_framework.mixins import ListModelMixin, UpdateModelMixin, RetrieveModelMixin from rest_framework.viewsets import GenericViewSet from assets.models.assets import Asset, AssetListGroup from console.conslog import object_create_log from console.models import Connection from core.decorators import log_url from core.mixins import ApiPermissionCheckMixin from core.utils import dtnow from networkmap.services import parse_endpoint_celery_done_events from events.constants import ELK_HOST, ELK_PORT, ELK_LOGIN, ELK_PASS from incident.models import Incident from networkmap.models import NetworkMap, NetworkMapBackgroundImage from networkmap.serializers import (UserMapSerializer, NetmapElementsSerializer, NetmapGroupSerializer, AssetDangerSerializer, UserMapNamesSerializer, AutoNetmapElementsSerializer, AutoNetmapConnectionsSerializer, NetworkMapBackgroundImagesSerializer) from perms.models import Perm _log = logging.getLogger(__name__) ELK_SEARCH_INDEX = getattr(settings, 'ELK_AGGREGATED_INDEX', 'aggregated-*') # TODO: rename "Firerwall" to "Firewall" when vector variable will be fixed # Yes its a mistake and yes it is correct for now ELK_FIREWALL_DEVICE_PRODUCT = getattr(settings, 'ELK_FIREWALL_PRODUCT_NAME', 'Industrial Firerwall') ELK_ENDPOINT_DEVICE_PRODUCT = getattr(settings, 'ELK_ENDPOINT_PRODUCT_NAME', 'Industrial Endpoint') DANGER_STATUSES = [ Incident.Status.NOT_ASSIGNED, Incident.Status.ASSIGNED, Incident.Status.DELAYED, Incident.Status.FALSE_ALARM, ] class UserMapViewSet(ApiPermissionCheckMixin, ListModelMixin, UpdateModelMixin, RetrieveModelMixin, GenericViewSet): console_permissions = [Perm.can_view_network] message = 'Cannot access user network map API. Permission denied' serializer_class = UserMapSerializer queryset = NetworkMap.objects.all() class NetworkMapBackgroundImagesViewSet(ApiPermissionCheckMixin, ListModelMixin, UpdateModelMixin, RetrieveModelMixin, GenericViewSet): """ ViewSet for handling the networkmap background image data manipulation """ console_permissions = [Perm.can_view_network] message = 'Cannot access user network map API. Permission denied' serializer_class = NetworkMapBackgroundImagesSerializer def get_queryset(self): """ This method override is used for including linked to background image network map data """ network_map_id = self.request.query_params.get('current_map_id') queryset = NetworkMapBackgroundImage.objects.filter(network_map=network_map_id) return queryset def try_to_get_background_image(self, map_pk, image_data, full_image_data=True): """ Helper method for trying to get an image from DB :param map_pk: linked for current background image network map pk :param image_data: image data, which is needed for request :param full_image_data: flag to show if image data that are being provided for request is: if True - image data provided in cytoscape module for handling background images format - support images if False - provided data will not be checked in this helper function, thus needed to be validated in further actions :return: Two variables: 1. True if image exists ind DB, False otherwise 2. If first variable is True - requested image instance, otherwise - JsonResponse with error status and message """ if map_pk is None or image_data is None: return False, JsonResponse({'status': 'err', 'error_message': 'Some mandatory arguments are missing'}) # Check if provided image data is valid if full_image_data: try: background_image_data = json.loads(image_data) except json.JSONDecodeError: _log.error(f"Failed to parse background image data for network map {map_pk}") return False, JsonResponse({'status': 'err', 'error_message': 'Image data is not valid'}) else: background_image_data = image_data # Get requested image try: if full_image_data: target_image = NetworkMapBackgroundImage.objects.get(network_map=map_pk, name=background_image_data['name']) else: target_image = NetworkMapBackgroundImage.objects.get(pk=background_image_data) except NetworkMapBackgroundImage.DoesNotExist: return False, JsonResponse({'status': 'err', 'error_message': 'Image does not exists in database'}) return True, target_image @action(detail=False, methods=["POST"], name="update_image_data") def update_image_data(self, request): """ Method for handling -update-image-data request for updating background image data fields value in Database. Request always should contain 'current_map_id' and 'image_data' fields. :param request: request instance that must contains 'current_map_id' and 'image_data' fields. Image_data should be in support images format (cytoscape framework that is used for managing network map background images) :return: JsonResponse with 'ok' status if everything works as expected, or with error message and 'err' status otherwise """ # Get necessary arguments for updating image data try: network_map_id = int(self.request.POST.get('current_map_id')) background_image_data = self.request.POST.get('image_data') except TypeError: return JsonResponse( {'status': 'err', 'error_message': 'Necessary data for request not provided or corrupted'}) success_get_status, image_or_error = self.try_to_get_background_image(network_map_id, background_image_data) if success_get_status: parsed_image_data = json.loads(background_image_data) image_or_error.bounds = parsed_image_data['bounds'] image_or_error.locked = parsed_image_data['locked'] image_or_error.visible = parsed_image_data['visible'] image_or_error.save() return JsonResponse({'status': 'ok'}) else: return image_or_error @action(detail=False, methods=["POST"], name="delete_background_image") def delete_background_image(self, request): """ Method for handling -delete-background-image request for deleting existing background image for current network map. Request always should contain 'current_map_id' and 'image_data' fields. :param request: request instance that must contains 'current_map_id' and 'image_data' fields. Image_data must contain background image ID :return: JsonResponse with 'ok' status if everything works as expected, or with error message and 'err' status otherwise """ # Get necessary arguments for updating image data try: network_map_id = int(self.request.POST.get('current_map_id')) background_image_data = int(self.request.POST.get('image_data')) except TypeError: return JsonResponse( {'status': 'err', 'error_message': 'Necessary data for request not provided or corrupted'}) success_get_status, image_or_error = self.try_to_get_background_image(network_map_id, background_image_data, False) if success_get_status: image_or_error.delete() return JsonResponse({'status': 'ok'}) else: return image_or_error class NetmapElementsViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet): console_permissions = [Perm.can_view_network] pagination_class = None serializer_class = NetmapElementsSerializer queryset = Asset.objects.all() class AutoNetmapElementsViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet): pagination_class = None serializer_class = AutoNetmapElementsSerializer console_permissions = [Perm.can_view_network] queryset = Asset.objects.all() class AutoNetmapConnectionsViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet): pagination_class = None serializer_class = AutoNetmapConnectionsSerializer console_permissions = [Perm.can_view_network] queryset = Connection.objects.all() class NetmapGroupsViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet): console_permissions = [Perm.can_view_network] pagination_class = None serializer_class = NetmapGroupSerializer queryset = AssetListGroup.objects.filter(asset__isnull=False).distinct() class AssetDangerViewSet(ApiPermissionCheckMixin, ListModelMixin, GenericViewSet): console_permissions = [Perm.can_view_network] pagination_class = None serializer_class = AssetDangerSerializer def get_queryset(self): queryset = Asset.objects.filter(incidents__status__in=DANGER_STATUSES).distinct() return queryset # def add_new_user_map(request): # if request.method == 'POST': # form = AddNewUserMapForm(request.POST, request.FILES, user=request.user) # if form.is_valid(): # new_map = form.save() # serializer = UserMapSerializer(new_map) # return JsonResponse(serializer.data, safe=False) # else: # form = AddNewUserMapForm() # ctx = {} # ctx.update(csrf(request)) # form = render_crispy_form(form, context=ctx) # return JsonResponse({'status': 'err', 'form_html': form}) # def add_new_background_image(request, current_netmap_id): # """ API for managing the new network map background image form # :param request: request instance # :param current_netmap_id: current network map id, on which is performing the background image addition # :return: JsonResponse with 'ok' status if everything performed as expected and new background image is being added, # JsonResponse with 'err' status and HTML rendered form with errors otherwise # """ # try: # netmap = NetworkMap.objects.get(pk=current_netmap_id) # except NetworkMap.DoesNotExist: # return JsonResponse({'status': 'err', 'error_message': 'Unknown networkmap selected'}) # if request.method == 'POST': # form = AddNewBackgroundImageForm(request.POST, request.FILES, # networkmap=netmap) # if form.is_valid(): # form.save() # return JsonResponse({'status': 'ok'}) # else: # _log.error(form.errors) # else: # form = AddNewBackgroundImageForm() # ctx = {} # ctx.update(csrf(request)) # form = render_crispy_form(form, context=ctx) # return JsonResponse({'status': 'err', 'form_html': form}) @api_view(['GET']) @permission_required(Perm.perm_req(Perm.can_view_network), raise_exception=True) def get_user_maps(request): """ API for getting network maps for specific user :param request: request object :return: list of group names for current user """ all_maps = NetworkMap.objects.filter(Q(user=request.user) | Q(shared_map=True)) serializer = UserMapNamesSerializer(all_maps, many=True) return JsonResponse(serializer.data, safe=False) @api_view(['GET']) @permission_required(Perm.perm_req(Perm.can_view_network), raise_exception=True) def get_map_background_image(request, background_pk): try: bimage = NetworkMapBackgroundImage.objects.get(pk=background_pk) except NetworkMapBackgroundImage.DoesNotExist: return JsonResponse({'status': 'err', 'error_message': 'Background image does not exists'}) if not bimage.url: return JsonResponse({'status': 'err', 'error_message': 'No link for selected background image'}) else: return HttpResponse(bimage.url, content_type='image/png') @api_view(['DELETE']) @permission_required(Perm.perm_req(Perm.can_view_network), raise_exception=True) def delete_user_map(request, pk): try: target_netmap = NetworkMap.objects.get(pk=pk) except NetworkMap.DoesNotExist: return JsonResponse({'message': 'The networkmap does not exist'}, status=status.HTTP_404_NOT_FOUND) target_netmap.delete() return JsonResponse({'message': 'Networkmap was deleted successfully!'}, status=status.HTTP_200_OK) def auto_netmap_assets_filter(form_assets_data, include_neighbours, assets_queryset=None, connections_queryset=None): """ Asset filtering function for auto network map. Function returns list of two querysets [Filtered_assets, Filtered_connections] which can be used in further filtering or send to frontend for rendering the map :param form_assets_data: data, provided by user :param assets_queryset: assets queryset from previous filters, if exists :param connections_queryset: connection queryset from previous filters, if exists :param include_neighbours: argument to tell if assets, which have connections with selected assets, should be displayed :return: list with assets and connections filtered querysets """ chosen_assets_queryset = Asset.objects.filter(pk__in=form_assets_data) if not connections_queryset: connections_queryset = Connection.objects.all() # Get connections for selected assets edge_assets = [] if include_neighbours: filtered_connections_queryset = connections_queryset.filter( Q(src_asset__in=chosen_assets_queryset) | Q(dst_asset__in=chosen_assets_queryset)) for asset in chosen_assets_queryset: src_connections = filtered_connections_queryset.filter(src_asset=asset) dst_connections = filtered_connections_queryset.filter(dst_asset=asset) for connection in src_connections: edge_assets.append(connection.dst_asset.pk) for connection in dst_connections: edge_assets.append(connection.src_asset.pk) else: filtered_connections_queryset = connections_queryset.filter( Q(src_asset__in=chosen_assets_queryset) & Q(dst_asset__in=chosen_assets_queryset)) edged_assets = Asset.objects.filter(pk__in=edge_assets) filtered_assets_queryset = chosen_assets_queryset.union(edged_assets) return [filtered_assets_queryset, filtered_connections_queryset] def auto_netmap_time_filter(form_time_data, connections_queryset='no_data'): """ Time filtering function for auto network map. Function returns list of two querysets [Filtered_assets, Filtered_connections] which can be used in further filtering or send to frontend for rendering the map :param form_time_data: data, provided by user :param connections_queryset: connection queryset from previous filters, if exists :return: list with assets and connections filtered querysets """ splitted_range = form_time_data.split(' - ') formatted_range = [] for timestamp in splitted_range: if '.' in timestamp: formatted_range.append(datetime.strptime(timestamp, '%d.%m.%Y %H:%M:%S')) else: formatted_range.append(datetime.fromisoformat(timestamp)) if connections_queryset == 'no_data': connections_queryset = Connection.objects.all() filtered_connections = connections_queryset.filter(updated__range=formatted_range) assets_set = [] for connection in filtered_connections: assets_set.append(connection.src_asset.pk) assets_set.append(connection.dst_asset.pk) filtered_assets = Asset.objects.filter(pk__in=list(assets_set)) return [filtered_assets, filtered_connections] def auto_netmap_protocol_filter(form_protocol_data, connections_queryset='no_data'): """ Protocol filtering function for auto network map. Function returns list of two querysets [Filtered_assets, Filtered_connections] which can be used in further filtering or send to frontend for rendering the map :param form_time_data: data, provided by user :param connections_queryset: connection queryset from previous filters, if exists :return: list with assets and connections filtered querysets """ if connections_queryset == 'no_data': connections_queryset = Connection.objects.all() pattern = re.compile(r'^\w+$') # allow any word character (equivalent to [a-zA-Z0-9_]) form_protocol_data = [protocol for protocol in form_protocol_data if pattern.match(protocol)] filtered_connections = connections_queryset.filter(connection_protocol__in=form_protocol_data) assets_set = [] for connection in filtered_connections: assets_set.append(connection.src_asset.pk) assets_set.append(connection.dst_asset.pk) filtered_assets = Asset.objects.filter(pk__in=list(assets_set)) return [filtered_assets, filtered_connections] def apply_auto_netmap_filters(parsed_filters): # Next two 'if's are made for parsing JSON converting issues from ajax if 'assets_filter[assets][]' in parsed_filters: parsed_filters['assets_filter'] = {'assets': parsed_filters.pop('assets_filter[assets][]'), 'include_neighbours': bool( strtobool(parsed_filters.pop('assets_filter[include_neighbours]')[0]))} if 'protocol_filter[]' in parsed_filters: parsed_filters['protocol_filter'] = parsed_filters.pop('protocol_filter[]') filters_querydicts = [Asset.objects.all(), Connection.objects.all()] if 'assets_filter' in parsed_filters: chosen_assets = parsed_filters.get('assets_filter').get('assets') include_neighbours = parsed_filters.get('assets_filter').get('include_neighbours') filters_querydicts = auto_netmap_assets_filter(chosen_assets, include_neighbours) if 'time_filter' in parsed_filters: filters_querydicts = auto_netmap_time_filter(parsed_filters.get('time_filter'), filters_querydicts[1]) if 'protocol_filter' in parsed_filters: filters_querydicts = auto_netmap_protocol_filter(parsed_filters.get('protocol_filter'), filters_querydicts[1]) serialized_assets = AutoNetmapElementsSerializer(filters_querydicts[0], many=True) serialized_connections = AutoNetmapConnectionsSerializer(filters_querydicts[1], many=True) context = { 'edges': serialized_connections.data, 'nodes': serialized_assets.data, } return context # @log_url # @login_required # @permission_required(Perm.perm_req(Perm.can_view_network), raise_exception=True) # def handle_filter_forms(request, form_type): # if form_type == 'assets': # filter_form = FilterByAssetForm # elif form_type == 'time': # filter_form = FilterByTimeForm # elif form_type == 'protocol': # filter_form = FilterByProtocolForm # elif form_type == 'reset': # return JsonResponse(apply_auto_netmap_filters(request.GET.copy()), safe=False) # else: # _log.error(f"Unknown filter type for auto network map: {form_type}") # # if request.method == 'POST': # form = filter_form(request.POST) # if form.is_valid(): # current_auto_netmap_filters = form.cleaned_data['current_filters'] # context = apply_auto_netmap_filters(current_auto_netmap_filters) # return JsonResponse(context, safe=False) # else: # return JsonResponse({'status': 'err', 'error_message': 'no_entries_selected'}) # else: # form = filter_form() # ctx = {} # ctx.update(csrf(request)) # form = render_crispy_form(form, context=ctx) # return JsonResponse({'status': 'err', 'form_html': form}) def parse_firewall_celery_done_events(hit): """ Algorithm for forming connections data for the auto network map """ if hit['_source']['source_ip'] and hit['_source']['destination_ip']: source_ip = hit['_source']['source_ip'] destination_ip = hit['_source']['destination_ip'] log_type = hit['_source']['type'] connection_protocol = hit['_source']['event_protocol'] try: ip_address(source_ip) ip_address(destination_ip) except ValueError: _log.critical( f"Error incorrect ip format [event id: {hit.get('_id')} by source_ip {source_ip} \ or dest_ip {destination_ip}]") return # Check if Assets with corresponding IP's presented in database. If not - add them if not Asset.objects.filter(ip=source_ip, sensor=log_type).exists(): source_asset = Asset.objects.create(name=source_ip, ip=source_ip, sensor=log_type) object_create_log(source_asset.name, Asset.Meta.__name__) if not Asset.objects.filter(ip=destination_ip, sensor=log_type).exists(): dest_asset = Asset.objects.create(name=destination_ip, ip=destination_ip, sensor=log_type) object_create_log(dest_asset.name, Asset.Meta.__name__) # ip_list = [source_ip, destination_ip] # # Checking if there is a connection in db with the same IP's as in the event # if Connection.objects.filter(src_asset__ip__in=ip_list, dst_asset__ip__in=ip_list).exists(): # # If there is a match during filtering process - update field Connection.updated # try: # connection_from_db = Connection.objects.get(src_asset__ip=source_ip, # src_asset__sensor=log_type, # dst_asset__ip=destination_ip, # dst_asset__sensor=log_type) # # Add updated connection to the network map data # connection_from_db.updated = dtnow() # connection_from_db.save() # except Connection.DoesNotExist: # connection_from_db = Connection.objects.get(dst_asset__ip=source_ip, # dst_asset__sensor=log_type, # src_asset__ip=destination_ip, # src_asset__sensor=log_type) # # Add updated connection to the network map data # connection_from_db.updated = dtnow() # connection_from_db.save() # else: # # Creating a new connection if there is no in DB # received_protocol = connection_protocol.upper() # if received_protocol in Connection.ProtocolType: # Connection.objects.create(src_asset=Asset.objects.get(ip=source_ip, sensor=log_type), # dst_asset=Asset.objects.get(ip=destination_ip, sensor=log_type), # connection_protocol=received_protocol) # else: # Connection.objects.create(src_asset=Asset.objects.get(ip=source_ip, sensor=log_type), # dst_asset=Asset.objects.get(ip=destination_ip, sensor=log_type)) # TODO: rename to summary celery_done corresponding name def update_connections_model_data(): """ Algorithm for forming connections data for the auto network map """ # Receiving data from Elasticsearch es = Elasticsearch([{'host': ELK_HOST, 'port': ELK_PORT}], http_auth=(ELK_LOGIN, ELK_PASS)) search_body = { "query": { "match": { "celery_done": "false" } } } es_search = es.search(index=ELK_SEARCH_INDEX, body=search_body, filter_path=['hits.hits._*'], size=1000) if 'hits' in es_search: for hit in es_search['hits']['hits']: if hit['_source']['device_product'] == ELK_ENDPOINT_DEVICE_PRODUCT: parse_endpoint_celery_done_events(hit) elif hit['_source']['device_product'] == ELK_FIREWALL_DEVICE_PRODUCT: parse_firewall_celery_done_events(hit) # Updating 'celery_done field hit_id = hit.get('_id') update_body = { "script": { "source": "ctx._source.celery_done=true", "lang": "painless" }, "query": { "match": { "_id": hit_id } } } # Updating ELK event data es.update_by_query(index=ELK_SEARCH_INDEX, body=update_body)