596 lines
14 KiB
Go
596 lines
14 KiB
Go
package rules
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"github.com/google/uuid"
|
|
log "github.com/sirupsen/logrus"
|
|
"github.com/spf13/viper"
|
|
"iwarma.ru/console/correlator/config"
|
|
"iwarma.ru/console/correlator/es"
|
|
"iwarma.ru/console/correlator/events"
|
|
"iwarma.ru/console/correlator/util"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/olivere/elastic/v7"
|
|
)
|
|
|
|
func FillAggregatedEvents(index string, count int, el *es.Elastic, firstEvent time.Time, lastEvent time.Time) error {
|
|
bulk := el.NewBulkRequest()
|
|
|
|
for i := 0; i < count; i++ {
|
|
event := events.Event{
|
|
events.Hash: fmt.Sprintf("%v", i),
|
|
events.FirstEvent: firstEvent,
|
|
events.LastEvent: lastEvent,
|
|
events.EventCount: 0,
|
|
events.Created: firstEvent,
|
|
events.Tags: nil,
|
|
events.AggregatedId: fmt.Sprintf("%v", i),
|
|
events.CeleryDone: false,
|
|
"timestamp": time.Time{},
|
|
"type": "test",
|
|
"event_timestamp": time.Time{},
|
|
"event_id": uuid.NewString(),
|
|
"event_severity": uint8(i),
|
|
"event_src_msg": fmt.Sprintf("Test message %v", i),
|
|
"event_protocol": "TCP",
|
|
"device_vendor": "TestDevice",
|
|
"device_product": "TestProduct",
|
|
"device_version": "1.0",
|
|
"device_action": "Test",
|
|
"sign_id": fmt.Sprintf("%v", i),
|
|
"app_category": "Test",
|
|
"app_subcategory": "Test",
|
|
"app_name": "Test",
|
|
"source_ip": "127.0.0.1",
|
|
"source_mac": "00:50:56:c0:00:08",
|
|
"source_host": "localhost",
|
|
"source_port": uint32(i),
|
|
"source_user": "root",
|
|
"destination_ip": "127.0.0.1",
|
|
"destination_host": "localhost",
|
|
"destination_port": uint32(i),
|
|
"destination_user": "user",
|
|
}
|
|
|
|
bulk = bulk.Add(elastic.NewBulkIndexRequest().Index(index).Id(event.GetString(events.EventID)).Doc(event))
|
|
}
|
|
|
|
bulkResponse, err := el.ExecuteBulk(bulk)
|
|
if err != nil {
|
|
log.Errorf("Can't index documents: %v", err)
|
|
return err
|
|
}
|
|
|
|
if bulkResponse.Errors {
|
|
log.Errorf("Got errors from bulk requset: %v", bulkResponse.Failed())
|
|
return fmt.Errorf("bulk error")
|
|
}
|
|
|
|
if len(bulkResponse.Indexed()) != count {
|
|
log.Errorf("Bad bulk index count. Got %v, expect %v", len(bulkResponse.Indexed()), count)
|
|
return fmt.Errorf("bad bulk count")
|
|
}
|
|
|
|
// wait until elastic is ready
|
|
time.Sleep(time.Second)
|
|
|
|
return nil
|
|
}
|
|
|
|
func TestGetRangeQueryBadDuration(t *testing.T) {
|
|
var rule Rule
|
|
|
|
query, err := rule.GetRangeQuery()
|
|
if err == nil {
|
|
t.Error("Get duration from empty field")
|
|
}
|
|
|
|
if query != nil {
|
|
t.Error("Got result where error occurs")
|
|
}
|
|
}
|
|
|
|
func TestGetRangeQuery(t *testing.T) {
|
|
rule := Rule{Depth: time.Minute}
|
|
|
|
query, err := rule.GetRangeQuery()
|
|
if err != nil {
|
|
t.Error("Can't parse rule duration")
|
|
return
|
|
}
|
|
|
|
source, err := query.Source()
|
|
if err != nil {
|
|
t.Error("Can't get query source")
|
|
}
|
|
|
|
if source == nil {
|
|
t.Error("Got nil source for query")
|
|
}
|
|
}
|
|
|
|
func TestRuleMarshall(t *testing.T) {
|
|
actions := make([]Action, 2)
|
|
actions[0] = &HttpAction{Url: "http://localhost:3456", Template: "{{.EventSrcMsg}}", ContentType: "text/plain"}
|
|
actions[1] = &SyslogAction{Host: "localhost", Port: 514, Proto: SyslogProtoUdp, Name: "My Tester", Template: "{{.EventSrcMsg}}"}
|
|
|
|
predicate := NewPredicate("", "event_severity:>=6")
|
|
|
|
rule := Rule{Actions: actions, Depth: time.Minute * 5, Name: "My test rule", Predicate: predicate, Id: "a716da55-66c5-478b-80fb-2c2a7e517a01"}
|
|
|
|
bytes, err := json.Marshal(rule)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
if string(bytes) != `{"id":"a716da55-66c5-478b-80fb-2c2a7e517a01","name":"My test rule","description":"","depth":"5m0s","predicat":{"field":"","operands":["event_severity:\u003e=6"]},"multi":false,"actions":[{"type":"http","url":"http://localhost:3456","template":"{{.EventSrcMsg}}","content_type":"text/plain"},{"type":"syslog","host":"localhost","port":"514","protocol":"udp","name":"My Tester","format":"","template":"{{.EventSrcMsg}}"}]}` {
|
|
t.Errorf("Got bad json: %v", string(bytes))
|
|
}
|
|
}
|
|
|
|
func TestRuleMarshall2(t *testing.T) {
|
|
actions := make([]Action, 2)
|
|
actions[0] = &HttpAction{Url: "http://localhost:3456", Template: "{{.EventSrcMsg}}", ContentType: "application/json"}
|
|
actions[1] = &SyslogAction{Host: "localhost", Port: 514, Proto: SyslogProtoUdp, Name: "My Tester", Template: "{{.EventSrcMsg}}"}
|
|
|
|
rule := Rule{Actions: actions, Depth: time.Minute * 5, Name: "My test rule", Id: "a716da55-66c5-478b-80fb-2c2a7e517a01"}
|
|
|
|
bytes, err := json.Marshal(rule)
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
if string(bytes) != `{"id":"a716da55-66c5-478b-80fb-2c2a7e517a01","name":"My test rule","description":"","depth":"5m0s","predicat":{"field":"","operands":null},"multi":false,"actions":[{"type":"http","url":"http://localhost:3456","template":"{{.EventSrcMsg}}","content_type":"application/json"},{"type":"syslog","host":"localhost","port":"514","protocol":"udp","name":"My Tester","format":"","template":"{{.EventSrcMsg}}"}]}` {
|
|
t.Errorf("Got bad json: %v", string(bytes))
|
|
}
|
|
}
|
|
|
|
func TestRuleUnmarshall(t *testing.T) {
|
|
str := `{
|
|
"id": "2",
|
|
"group": "Preset",
|
|
"actions":
|
|
[{
|
|
"type": "incident",
|
|
"title": "{{.SignName}}",
|
|
"comment": "",
|
|
"category": "",
|
|
"importance": "50",
|
|
"assigned_to": "",
|
|
"description": "{{.EventSrcMsg}}"}],
|
|
"name": "Serious event",
|
|
"description": "Long description of rule",
|
|
"multi": false,
|
|
"type": 0,
|
|
"status": true,
|
|
"depth": "10m",
|
|
"created": "2021-11-25 15:35:45",
|
|
"updated": "2021-11-25 15:35:46",
|
|
"predicat":
|
|
{
|
|
"type": "query_string",
|
|
"field": "",
|
|
"operands": "event_severity:>=6"
|
|
},
|
|
"rev": 1,
|
|
"sid": 2,
|
|
"is_active": true
|
|
}`
|
|
var rule Rule
|
|
err := json.Unmarshal([]byte(str), &rule)
|
|
if err != nil {
|
|
t.Error(err)
|
|
return
|
|
}
|
|
|
|
if rule.Id != "2" {
|
|
t.Errorf("Got bad id: %v", rule.Id)
|
|
}
|
|
|
|
if rule.Name != "Serious event" {
|
|
t.Errorf("Got bad name: %v", rule.Name)
|
|
}
|
|
|
|
if rule.Description != "Long description of rule" {
|
|
t.Errorf("Got bad description: %v", rule.Description)
|
|
}
|
|
|
|
if rule.Depth != time.Minute*10 {
|
|
t.Errorf("Got bad depth: %v", rule.Depth)
|
|
}
|
|
|
|
if rule.Predicate.Field != "" {
|
|
t.Errorf("Got bad predicat field: %v", rule.Predicate.Field)
|
|
}
|
|
|
|
if len(rule.Predicate.Operands) != 1 {
|
|
t.Errorf("Bad operands length: %v", len(rule.Predicate.Operands))
|
|
}
|
|
|
|
if rule.Predicate.Operands[0].(string) != "event_severity:>=6" {
|
|
t.Errorf("Got bad operand calue: %v of type %T", rule.Predicate.Operands[0], rule.Predicate.Operands[0])
|
|
}
|
|
|
|
// Check actions
|
|
if len(rule.Actions) != 1 {
|
|
t.Errorf("Got bad actions size: %v", len(rule.Actions))
|
|
}
|
|
|
|
action1 := rule.Actions[0]
|
|
if action1.GetType() != "incident" {
|
|
t.Errorf("Got bad action #0 type: %v", action1.GetType())
|
|
}
|
|
|
|
}
|
|
|
|
func TestRuleUnmarshallBadNotJson(t *testing.T) {
|
|
text := "Some text"
|
|
|
|
rule := &Rule{}
|
|
|
|
err := rule.UnmarshalJSON([]byte(text))
|
|
if err == nil {
|
|
t.Error("No error with not json")
|
|
}
|
|
}
|
|
|
|
func TestRuleUnmarshallBadDepth(t *testing.T) {
|
|
text := `{"depth":"ass"}`
|
|
|
|
rule := &Rule{}
|
|
|
|
err := rule.UnmarshalJSON([]byte(text))
|
|
if err == nil {
|
|
t.Error("No error with bad depth")
|
|
}
|
|
}
|
|
|
|
func TestRuleUnmarshallBadActionType(t *testing.T) {
|
|
text := `{"depth":"1m","actions":[{"type":"ass"}]}`
|
|
|
|
rule := &Rule{}
|
|
|
|
err := rule.UnmarshalJSON([]byte(text))
|
|
if err == nil {
|
|
t.Error("No error with bad action type")
|
|
}
|
|
}
|
|
|
|
func TestRuleUnmarshallBadActionContent(t *testing.T) {
|
|
text := `{"depth":"1m","actions":[{"type":"http"}]}`
|
|
|
|
rule := &Rule{}
|
|
|
|
err := rule.UnmarshalJSON([]byte(text))
|
|
if err == nil {
|
|
t.Error("No error with bad action type")
|
|
}
|
|
}
|
|
|
|
// Check that we get somethings from elastic
|
|
func TestRuleDoWorks(t *testing.T) {
|
|
util.SetupTest(t)
|
|
defer util.TearDownTest(t)
|
|
|
|
client, err := es.NewElastic()
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
err = events.ClearIndex(client, events.GetAggregatedIndexName())
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
depth := time.Minute * 50
|
|
N := 10
|
|
err = FillAggregatedEvents(events.GetAggregatedIndexName(), N, client, time.Now().UTC().Add(-depth), time.Now().UTC().Add(depth))
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
actions := make([]Action, 1)
|
|
actions[0] = &TestAction{}
|
|
|
|
// Rule
|
|
rule := Rule{
|
|
Id: "Rule",
|
|
Name: "My rule",
|
|
Depth: depth,
|
|
Multi: true,
|
|
Predicate: NewPredicate("", "device_vendor:TestDevice"),
|
|
Actions: actions,
|
|
}
|
|
|
|
resultEvents, err := rule.Do(client)
|
|
if err != nil {
|
|
t.Error(err)
|
|
return
|
|
}
|
|
|
|
if len(*resultEvents) != N {
|
|
t.Errorf("Bad events count, got %v need %v", len(*resultEvents), N)
|
|
}
|
|
|
|
err = events.ClearIndex(client, events.GetAggregatedIndexName())
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Check that if aggregated events miss rule depth we don't have any
|
|
func TestRuleDoWithBadDepth(t *testing.T) {
|
|
util.SetupTest(t)
|
|
defer util.TearDownTest(t)
|
|
|
|
client, err := es.NewElastic()
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
err = events.ClearIndex(client, events.GetAggregatedIndexName())
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
depth := time.Minute * 5
|
|
N := 10
|
|
err = FillAggregatedEvents(events.GetAggregatedIndexName(), N, client, time.Now().UTC().Add(-depth*2), time.Now().UTC().Add(-depth*2))
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
actions := make([]Action, 1)
|
|
actions[0] = &TestAction{}
|
|
|
|
// Rule
|
|
rule := Rule{
|
|
Id: "Rule",
|
|
Name: "My rule",
|
|
Depth: time.Duration(0),
|
|
Multi: true,
|
|
Predicate: NewPredicate("", "device_vendor:TestDevice"),
|
|
Actions: actions,
|
|
}
|
|
|
|
resultEvents, err := rule.Do(client)
|
|
if err == nil {
|
|
t.Error("Can work with bad duration")
|
|
}
|
|
|
|
if resultEvents != nil {
|
|
t.Error("Got some events, when must be nil")
|
|
}
|
|
}
|
|
|
|
// Check that we can detect errors while fetching aggregated events
|
|
func TestRuleDoWithGetterErrors(t *testing.T) {
|
|
util.SetupTest(t)
|
|
defer util.TearDownTest(t)
|
|
|
|
client, err := es.NewElastic()
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
// Change index name to something that doesn't exist
|
|
viper.Set(config.ElasticAggregatedIndexName, "asss450")
|
|
|
|
depth := time.Minute * 5
|
|
|
|
actions := make([]Action, 1)
|
|
actions[0] = &TestAction{}
|
|
|
|
// Rule
|
|
rule := Rule{
|
|
Id: "Rule",
|
|
Name: "My rule",
|
|
Depth: depth,
|
|
Multi: true,
|
|
Predicate: NewPredicate("", "device_vendor:TestDevice"),
|
|
Actions: actions,
|
|
}
|
|
|
|
resultEvents, err := rule.Do(client)
|
|
if err == nil {
|
|
t.Error("No errors when we need some")
|
|
return
|
|
}
|
|
|
|
if resultEvents != nil {
|
|
t.Error("Got some events when must be nil")
|
|
}
|
|
}
|
|
|
|
// Check that we can detect action errors
|
|
func TestRuleDoWithActionError(t *testing.T) {
|
|
util.SetupTest(t)
|
|
defer util.TearDownTest(t)
|
|
|
|
client, err := es.NewElastic()
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
err = events.ClearIndex(client, events.GetAggregatedIndexName())
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
depth := time.Minute * 5
|
|
N := 10
|
|
err = FillAggregatedEvents(events.GetAggregatedIndexName(), N, client, time.Now().UTC().Add(-depth), time.Now().UTC().Add(depth))
|
|
if err != nil {
|
|
t.Errorf("%v", err)
|
|
return
|
|
}
|
|
|
|
actions := make([]Action, 1)
|
|
actions[0] = &TestAction{PerformError: true}
|
|
|
|
// Rule
|
|
rule := Rule{
|
|
Id: "Rule",
|
|
Name: "My rule",
|
|
Depth: depth,
|
|
Multi: true,
|
|
Predicate: NewPredicate("", "device_vendor:TestDevice"),
|
|
Actions: actions,
|
|
}
|
|
|
|
resultEvents, err := rule.Do(client)
|
|
if err == nil {
|
|
t.Error("No error when getter trows one")
|
|
}
|
|
|
|
if resultEvents != nil {
|
|
t.Error("Got events when must be nil")
|
|
}
|
|
}
|
|
|
|
func TestDjangoRuleUnmarshall(t *testing.T) {
|
|
message := `{
|
|
"name":"Test",
|
|
"depth":"600s",
|
|
"id":"1",
|
|
"predicat":{
|
|
"type":"query_string",
|
|
"field":"",
|
|
"operands" : "source_host:localhost"
|
|
|
|
},
|
|
"actions":[
|
|
{
|
|
"host":"localhost",
|
|
"port":"514",
|
|
"protocol":"udp",
|
|
"name":"test",
|
|
"template":"{{.EventSrcMsg}}",
|
|
"type":"syslog"
|
|
}
|
|
]
|
|
}`
|
|
|
|
var rule Rule
|
|
err := json.Unmarshal([]byte(message), &rule)
|
|
if err != nil {
|
|
t.Errorf("Json error: %v", err)
|
|
}
|
|
|
|
if rule.Name != "Test" {
|
|
t.Errorf("Got bad rule name: %v", rule.Name)
|
|
}
|
|
|
|
if rule.Id != "1" {
|
|
t.Errorf("Got bad rule id: %v", rule.Id)
|
|
}
|
|
|
|
if rule.Depth != time.Minute*10 {
|
|
t.Errorf("Got bad rule depth: %v", rule.Depth)
|
|
}
|
|
|
|
predicat := rule.Predicate
|
|
|
|
if len(predicat.Operands) != 1 {
|
|
t.Errorf("Got bad predicat operands length: %v", len(predicat.Operands))
|
|
}
|
|
|
|
value := predicat.Operands[0].(string)
|
|
if value != "source_host:localhost" {
|
|
t.Errorf("Got bad predicat value: %v", value)
|
|
}
|
|
|
|
if len(rule.Actions) != 1 {
|
|
t.Errorf("Got bad actions length: %v", len(rule.Actions))
|
|
}
|
|
|
|
if rule.Actions[0].GetType() != SyslogActionType {
|
|
t.Errorf("Got bad action type: %v", rule.Actions[0].GetType())
|
|
}
|
|
|
|
bytes, err := json.Marshal(rule.Actions[0])
|
|
if err != nil {
|
|
t.Errorf("Can't marshall action")
|
|
}
|
|
|
|
if string(bytes) != `{"type":"syslog","host":"localhost","port":"514","protocol":"udp","name":"test","format":"","template":"{{.EventSrcMsg}}"}` {
|
|
t.Errorf(fmt.Sprintf("Got bad action: %v", string(bytes)))
|
|
}
|
|
}
|
|
|
|
func TestDjangoRuleUnmarshall2(t *testing.T) {
|
|
message := `{
|
|
"name":"Test",
|
|
"depth":"600s",
|
|
"id":"1",
|
|
"predicat":{
|
|
"type":"query_string",
|
|
"field":"",
|
|
"operands":"source_port:514"
|
|
},
|
|
"actions":[
|
|
{
|
|
"host":"localhsot",
|
|
"port":"514",
|
|
"protocol":"udp",
|
|
"name":"test",
|
|
"template":"{{.EventSrcMsg}}",
|
|
"type":"syslog"
|
|
}
|
|
]
|
|
}`
|
|
|
|
var rule Rule
|
|
err := json.Unmarshal([]byte(message), &rule)
|
|
if err != nil {
|
|
t.Errorf("Json error: %v", err)
|
|
}
|
|
|
|
if rule.Name != "Test" {
|
|
t.Errorf("Got bad rule name: %v", rule.Name)
|
|
}
|
|
|
|
if rule.Id != "1" {
|
|
t.Errorf("Got bad rule id: %v", rule.Id)
|
|
}
|
|
|
|
if rule.Depth != time.Minute*10 {
|
|
t.Errorf("Got bad rule depth: %v", rule.Depth)
|
|
}
|
|
|
|
predicat := rule.Predicate
|
|
|
|
if len(predicat.Operands) != 1 {
|
|
t.Errorf("Got bad predicat operands length: %v", len(predicat.Operands))
|
|
}
|
|
|
|
if len(rule.Actions) != 1 {
|
|
t.Errorf("Got bad actions length: %v", len(rule.Actions))
|
|
}
|
|
|
|
if rule.Actions[0].GetType() != SyslogActionType {
|
|
t.Errorf("Got bad action type: %v", rule.Actions[0].GetType())
|
|
}
|
|
|
|
bytes, err := json.Marshal(rule.Actions[0])
|
|
if err != nil {
|
|
t.Errorf("Can't marshall action")
|
|
}
|
|
|
|
if string(bytes) != `{"type":"syslog","host":"localhsot","port":"514","protocol":"udp","name":"test","format":"","template":"{{.EventSrcMsg}}"}` {
|
|
t.Errorf(fmt.Sprintf("Got bad action: %v", string(bytes)))
|
|
}
|
|
}
|