package rules import ( "encoding/json" "fmt" "github.com/google/uuid" log "github.com/sirupsen/logrus" "github.com/spf13/viper" "iwarma.ru/console/correlator/config" "iwarma.ru/console/correlator/es" "iwarma.ru/console/correlator/events" "iwarma.ru/console/correlator/util" "testing" "time" "github.com/olivere/elastic/v7" ) func FillAggregatedEvents(index string, count int, el *es.Elastic, firstEvent time.Time, lastEvent time.Time) error { bulk := el.NewBulkRequest() for i := 0; i < count; i++ { event := events.Event{ events.Hash: fmt.Sprintf("%v", i), events.FirstEvent: firstEvent, events.LastEvent: lastEvent, events.EventCount: 0, events.Created: firstEvent, events.Tags: nil, events.AggregatedId: fmt.Sprintf("%v", i), events.CeleryDone: false, "timestamp": time.Time{}, "type": "test", "event_timestamp": time.Time{}, "event_id": uuid.NewString(), "event_severity": uint8(i), "event_src_msg": fmt.Sprintf("Test message %v", i), "event_protocol": "TCP", "device_vendor": "TestDevice", "device_product": "TestProduct", "device_version": "1.0", "device_action": "Test", "sign_id": fmt.Sprintf("%v", i), "app_category": "Test", "app_subcategory": "Test", "app_name": "Test", "source_ip": "127.0.0.1", "source_mac": "00:50:56:c0:00:08", "source_host": "localhost", "source_port": uint32(i), "source_user": "root", "destination_ip": "127.0.0.1", "destination_host": "localhost", "destination_port": uint32(i), "destination_user": "user", } bulk = bulk.Add(elastic.NewBulkIndexRequest().Index(index).Id(event.GetString(events.EventID)).Doc(event)) } bulkResponse, err := el.ExecuteBulk(bulk) if err != nil { log.Errorf("Can't index documents: %v", err) return err } if bulkResponse.Errors { log.Errorf("Got errors from bulk requset: %v", bulkResponse.Failed()) return fmt.Errorf("bulk error") } if len(bulkResponse.Indexed()) != count { log.Errorf("Bad bulk index count. Got %v, expect %v", len(bulkResponse.Indexed()), count) return fmt.Errorf("bad bulk count") } // wait until elastic is ready time.Sleep(time.Second) return nil } func TestGetRangeQueryBadDuration(t *testing.T) { var rule Rule query, err := rule.GetRangeQuery() if err == nil { t.Error("Get duration from empty field") } if query != nil { t.Error("Got result where error occurs") } } func TestGetRangeQuery(t *testing.T) { rule := Rule{Depth: time.Minute} query, err := rule.GetRangeQuery() if err != nil { t.Error("Can't parse rule duration") return } source, err := query.Source() if err != nil { t.Error("Can't get query source") } if source == nil { t.Error("Got nil source for query") } } func TestRuleMarshall(t *testing.T) { actions := make([]Action, 2) actions[0] = &HttpAction{Url: "http://localhost:3456", Template: "{{.EventSrcMsg}}", ContentType: "text/plain"} actions[1] = &SyslogAction{Host: "localhost", Port: 514, Proto: SyslogProtoUdp, Name: "My Tester", Template: "{{.EventSrcMsg}}"} predicate := NewPredicate("", "event_severity:>=6") rule := Rule{Actions: actions, Depth: time.Minute * 5, Name: "My test rule", Predicate: predicate, Id: "a716da55-66c5-478b-80fb-2c2a7e517a01"} bytes, err := json.Marshal(rule) if err != nil { t.Error(err) } if string(bytes) != `{"id":"a716da55-66c5-478b-80fb-2c2a7e517a01","name":"My test rule","description":"","depth":"5m0s","predicat":{"field":"","operands":["event_severity:\u003e=6"]},"multi":false,"actions":[{"type":"http","url":"http://localhost:3456","template":"{{.EventSrcMsg}}","content_type":"text/plain"},{"type":"syslog","host":"localhost","port":"514","protocol":"udp","name":"My Tester","format":"","template":"{{.EventSrcMsg}}"}]}` { t.Errorf("Got bad json: %v", string(bytes)) } } func TestRuleMarshall2(t *testing.T) { actions := make([]Action, 2) actions[0] = &HttpAction{Url: "http://localhost:3456", Template: "{{.EventSrcMsg}}", ContentType: "application/json"} actions[1] = &SyslogAction{Host: "localhost", Port: 514, Proto: SyslogProtoUdp, Name: "My Tester", Template: "{{.EventSrcMsg}}"} rule := Rule{Actions: actions, Depth: time.Minute * 5, Name: "My test rule", Id: "a716da55-66c5-478b-80fb-2c2a7e517a01"} bytes, err := json.Marshal(rule) if err != nil { t.Error(err) } if string(bytes) != `{"id":"a716da55-66c5-478b-80fb-2c2a7e517a01","name":"My test rule","description":"","depth":"5m0s","predicat":{"field":"","operands":null},"multi":false,"actions":[{"type":"http","url":"http://localhost:3456","template":"{{.EventSrcMsg}}","content_type":"application/json"},{"type":"syslog","host":"localhost","port":"514","protocol":"udp","name":"My Tester","format":"","template":"{{.EventSrcMsg}}"}]}` { t.Errorf("Got bad json: %v", string(bytes)) } } func TestRuleUnmarshall(t *testing.T) { str := `{ "id": "2", "group": "Preset", "actions": [{ "type": "incident", "title": "{{.SignName}}", "comment": "", "category": "", "importance": "50", "assigned_to": "", "description": "{{.EventSrcMsg}}"}], "name": "Serious event", "description": "Long description of rule", "multi": false, "type": 0, "status": true, "depth": "10m", "created": "2021-11-25 15:35:45", "updated": "2021-11-25 15:35:46", "predicat": { "type": "query_string", "field": "", "operands": "event_severity:>=6" }, "rev": 1, "sid": 2, "is_active": true }` var rule Rule err := json.Unmarshal([]byte(str), &rule) if err != nil { t.Error(err) return } if rule.Id != "2" { t.Errorf("Got bad id: %v", rule.Id) } if rule.Name != "Serious event" { t.Errorf("Got bad name: %v", rule.Name) } if rule.Description != "Long description of rule" { t.Errorf("Got bad description: %v", rule.Description) } if rule.Depth != time.Minute*10 { t.Errorf("Got bad depth: %v", rule.Depth) } if rule.Predicate.Field != "" { t.Errorf("Got bad predicat field: %v", rule.Predicate.Field) } if len(rule.Predicate.Operands) != 1 { t.Errorf("Bad operands length: %v", len(rule.Predicate.Operands)) } if rule.Predicate.Operands[0].(string) != "event_severity:>=6" { t.Errorf("Got bad operand calue: %v of type %T", rule.Predicate.Operands[0], rule.Predicate.Operands[0]) } // Check actions if len(rule.Actions) != 1 { t.Errorf("Got bad actions size: %v", len(rule.Actions)) } action1 := rule.Actions[0] if action1.GetType() != "incident" { t.Errorf("Got bad action #0 type: %v", action1.GetType()) } } func TestRuleUnmarshallBadNotJson(t *testing.T) { text := "Some text" rule := &Rule{} err := rule.UnmarshalJSON([]byte(text)) if err == nil { t.Error("No error with not json") } } func TestRuleUnmarshallBadDepth(t *testing.T) { text := `{"depth":"ass"}` rule := &Rule{} err := rule.UnmarshalJSON([]byte(text)) if err == nil { t.Error("No error with bad depth") } } func TestRuleUnmarshallBadActionType(t *testing.T) { text := `{"depth":"1m","actions":[{"type":"ass"}]}` rule := &Rule{} err := rule.UnmarshalJSON([]byte(text)) if err == nil { t.Error("No error with bad action type") } } func TestRuleUnmarshallBadActionContent(t *testing.T) { text := `{"depth":"1m","actions":[{"type":"http"}]}` rule := &Rule{} err := rule.UnmarshalJSON([]byte(text)) if err == nil { t.Error("No error with bad action type") } } // Check that we get somethings from elastic func TestRuleDoWorks(t *testing.T) { util.SetupTest(t) defer util.TearDownTest(t) client, err := es.NewElastic() if err != nil { t.Errorf("%v", err) return } err = events.ClearIndex(client, events.GetAggregatedIndexName()) if err != nil { t.Errorf("%v", err) return } depth := time.Minute * 50 N := 10 err = FillAggregatedEvents(events.GetAggregatedIndexName(), N, client, time.Now().UTC().Add(-depth), time.Now().UTC().Add(depth)) if err != nil { t.Errorf("%v", err) return } actions := make([]Action, 1) actions[0] = &TestAction{} // Rule rule := Rule{ Id: "Rule", Name: "My rule", Depth: depth, Multi: true, Predicate: NewPredicate("", "device_vendor:TestDevice"), Actions: actions, } resultEvents, err := rule.Do(client) if err != nil { t.Error(err) return } if len(*resultEvents) != N { t.Errorf("Bad events count, got %v need %v", len(*resultEvents), N) } err = events.ClearIndex(client, events.GetAggregatedIndexName()) if err != nil { t.Errorf("%v", err) return } } // Check that if aggregated events miss rule depth we don't have any func TestRuleDoWithBadDepth(t *testing.T) { util.SetupTest(t) defer util.TearDownTest(t) client, err := es.NewElastic() if err != nil { t.Errorf("%v", err) return } err = events.ClearIndex(client, events.GetAggregatedIndexName()) if err != nil { t.Errorf("%v", err) return } depth := time.Minute * 5 N := 10 err = FillAggregatedEvents(events.GetAggregatedIndexName(), N, client, time.Now().UTC().Add(-depth*2), time.Now().UTC().Add(-depth*2)) if err != nil { t.Errorf("%v", err) return } actions := make([]Action, 1) actions[0] = &TestAction{} // Rule rule := Rule{ Id: "Rule", Name: "My rule", Depth: time.Duration(0), Multi: true, Predicate: NewPredicate("", "device_vendor:TestDevice"), Actions: actions, } resultEvents, err := rule.Do(client) if err == nil { t.Error("Can work with bad duration") } if resultEvents != nil { t.Error("Got some events, when must be nil") } } // Check that we can detect errors while fetching aggregated events func TestRuleDoWithGetterErrors(t *testing.T) { util.SetupTest(t) defer util.TearDownTest(t) client, err := es.NewElastic() if err != nil { t.Errorf("%v", err) return } // Change index name to something that doesn't exist viper.Set(config.ElasticAggregatedIndexName, "asss450") depth := time.Minute * 5 actions := make([]Action, 1) actions[0] = &TestAction{} // Rule rule := Rule{ Id: "Rule", Name: "My rule", Depth: depth, Multi: true, Predicate: NewPredicate("", "device_vendor:TestDevice"), Actions: actions, } resultEvents, err := rule.Do(client) if err == nil { t.Error("No errors when we need some") return } if resultEvents != nil { t.Error("Got some events when must be nil") } } // Check that we can detect action errors func TestRuleDoWithActionError(t *testing.T) { util.SetupTest(t) defer util.TearDownTest(t) client, err := es.NewElastic() if err != nil { t.Errorf("%v", err) return } err = events.ClearIndex(client, events.GetAggregatedIndexName()) if err != nil { t.Errorf("%v", err) return } depth := time.Minute * 5 N := 10 err = FillAggregatedEvents(events.GetAggregatedIndexName(), N, client, time.Now().UTC().Add(-depth), time.Now().UTC().Add(depth)) if err != nil { t.Errorf("%v", err) return } actions := make([]Action, 1) actions[0] = &TestAction{PerformError: true} // Rule rule := Rule{ Id: "Rule", Name: "My rule", Depth: depth, Multi: true, Predicate: NewPredicate("", "device_vendor:TestDevice"), Actions: actions, } resultEvents, err := rule.Do(client) if err == nil { t.Error("No error when getter trows one") } if resultEvents != nil { t.Error("Got events when must be nil") } } func TestDjangoRuleUnmarshall(t *testing.T) { message := `{ "name":"Test", "depth":"600s", "id":"1", "predicat":{ "type":"query_string", "field":"", "operands" : "source_host:localhost" }, "actions":[ { "host":"localhost", "port":"514", "protocol":"udp", "name":"test", "template":"{{.EventSrcMsg}}", "type":"syslog" } ] }` var rule Rule err := json.Unmarshal([]byte(message), &rule) if err != nil { t.Errorf("Json error: %v", err) } if rule.Name != "Test" { t.Errorf("Got bad rule name: %v", rule.Name) } if rule.Id != "1" { t.Errorf("Got bad rule id: %v", rule.Id) } if rule.Depth != time.Minute*10 { t.Errorf("Got bad rule depth: %v", rule.Depth) } predicat := rule.Predicate if len(predicat.Operands) != 1 { t.Errorf("Got bad predicat operands length: %v", len(predicat.Operands)) } value := predicat.Operands[0].(string) if value != "source_host:localhost" { t.Errorf("Got bad predicat value: %v", value) } if len(rule.Actions) != 1 { t.Errorf("Got bad actions length: %v", len(rule.Actions)) } if rule.Actions[0].GetType() != SyslogActionType { t.Errorf("Got bad action type: %v", rule.Actions[0].GetType()) } bytes, err := json.Marshal(rule.Actions[0]) if err != nil { t.Errorf("Can't marshall action") } if string(bytes) != `{"type":"syslog","host":"localhost","port":"514","protocol":"udp","name":"test","format":"","template":"{{.EventSrcMsg}}"}` { t.Errorf(fmt.Sprintf("Got bad action: %v", string(bytes))) } } func TestDjangoRuleUnmarshall2(t *testing.T) { message := `{ "name":"Test", "depth":"600s", "id":"1", "predicat":{ "type":"query_string", "field":"", "operands":"source_port:514" }, "actions":[ { "host":"localhsot", "port":"514", "protocol":"udp", "name":"test", "template":"{{.EventSrcMsg}}", "type":"syslog" } ] }` var rule Rule err := json.Unmarshal([]byte(message), &rule) if err != nil { t.Errorf("Json error: %v", err) } if rule.Name != "Test" { t.Errorf("Got bad rule name: %v", rule.Name) } if rule.Id != "1" { t.Errorf("Got bad rule id: %v", rule.Id) } if rule.Depth != time.Minute*10 { t.Errorf("Got bad rule depth: %v", rule.Depth) } predicat := rule.Predicate if len(predicat.Operands) != 1 { t.Errorf("Got bad predicat operands length: %v", len(predicat.Operands)) } if len(rule.Actions) != 1 { t.Errorf("Got bad actions length: %v", len(rule.Actions)) } if rule.Actions[0].GetType() != SyslogActionType { t.Errorf("Got bad action type: %v", rule.Actions[0].GetType()) } bytes, err := json.Marshal(rule.Actions[0]) if err != nil { t.Errorf("Can't marshall action") } if string(bytes) != `{"type":"syslog","host":"localhsot","port":"514","protocol":"udp","name":"test","format":"","template":"{{.EventSrcMsg}}"}` { t.Errorf(fmt.Sprintf("Got bad action: %v", string(bytes))) } }