package service import ( "encoding/json" "fmt" "io" "net/http" "strconv" "strings" "time" "github.com/hashicorp/golang-lru/v2/expirable" "gitlab.com/security-rd/go-pkg/logging" "gitlab.com/tensorsecurity-rd/waf-console/internal/model" ) var fakeData = ` { "data": { "87.236.176.199": { "samples": [], "tags_classes": [], "judgments": [ "Scanner", "Zombie", "Dynamic IP", "Spam" ], "intelligences": { "threatbook_lab": [ { "source": "ThreatBook Labs", "confidence": 75, "expired": false, "intel_tags": [], "find_time": "2023-02-27 12:52:17", "intel_types": [ "Spam" ], "update_time": "2024-08-18 17:04:49" }, { "source": "ThreatBook Labs", "confidence": 80, "expired": false, "intel_tags": [], "find_time": "2023-02-27 12:14:00", "intel_types": [ "Dynamic IP" ], "update_time": "2023-02-27 12:14:00" }, { "source": "ThreatBook Labs", "confidence": 75, "expired": false, "intel_tags": [], "find_time": "2022-09-19 03:38:38", "intel_types": [ "Scanner" ], "update_time": "2024-08-22 10:40:46" }, { "source": "ThreatBook Labs", "confidence": 85, "expired": false, "intel_tags": [], "find_time": "2022-09-17 09:00:13", "intel_types": [ "Spam" ], "update_time": "2024-08-15 09:00:12" }, { "source": "ThreatBook Labs", "confidence": 75, "expired": false, "intel_tags": [], "find_time": "2022-09-16 11:16:41", "intel_types": [ "Zombie" ], "update_time": "2023-07-05 12:34:57" }, { "source": "ThreatBook Labs", "confidence": 75, "expired": true, "intel_tags": [], "find_time": "2023-05-10 19:58:16", "intel_types": [ "Scanner" ], "update_time": "2023-06-21 03:23:22" }, { "source": "ThreatBook Labs", "confidence": 85, "expired": true, "intel_tags": [], "find_time": "2023-02-27 12:23:46", "intel_types": [ "Spam", "Zombie" ], "update_time": "2024-07-20 18:35:32" }, { "source": "ThreatBook Labs", "confidence": 75, "expired": true, "intel_tags": [], "find_time": "2022-10-15 12:42:35", "intel_types": [ "Scanner" ], "update_time": "2022-10-22 09:11:17" } ], "x_reward": [], "open_source": [ { "source": "Open Source ", "confidence": 50, "expired": false, "intel_tags": [], "find_time": "2024-06-05 15:47:36", "intel_types": [ "Malware" ], "update_time": "2024-06-06 09:23:14" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2024-03-05 06:00:51", "intel_types": [ "Suspicious" ], "update_time": "2024-08-14 04:17:28" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2023-12-03 09:31:12", "intel_types": [ "Suspicious" ], "update_time": "2024-08-23 02:52:17" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2023-12-02 11:11:32", "intel_types": [ "Suspicious" ], "update_time": "2024-08-22 03:06:20" }, { "source": "Open Source ", "confidence": 70, "expired": false, "intel_tags": [], "find_time": "2023-11-29 09:39:12", "intel_types": [ "Scanner" ], "update_time": "2024-07-01 06:24:26" }, { "source": "cinsscore.com", "confidence": 50, "expired": false, "intel_tags": [], "find_time": "2023-11-11 02:40:16", "intel_types": [ "Suspicious" ], "update_time": "2024-08-10 01:30:20" }, { "source": "binarydefense.com", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2023-03-17 01:45:16", "intel_types": [ "Malware" ], "update_time": "2024-08-22 01:17:11" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2023-01-08 19:58:32", "intel_types": [ "Suspicious" ], "update_time": "2024-06-11 10:43:24" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2022-09-30 02:55:49", "intel_types": [ "Suspicious" ], "update_time": "2024-05-18 03:38:29" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2022-09-30 00:20:35", "intel_types": [ "Suspicious" ], "update_time": "2024-08-05 04:13:24" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2022-09-30 00:04:41", "intel_types": [ "Suspicious" ], "update_time": "2022-10-05 00:24:23" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2022-09-23 18:04:19", "intel_types": [ "Suspicious" ], "update_time": "2024-08-06 06:07:27" }, { "source": "Open Source ", "confidence": 65, "expired": false, "intel_tags": [], "find_time": "2022-09-23 18:00:31", "intel_types": [ "Suspicious" ], "update_time": "2024-08-06 06:08:33" }, { "source": "blocklist.de", "confidence": 55, "expired": false, "intel_tags": [], "find_time": "2022-09-22 15:57:53", "intel_types": [ "Scanner" ], "update_time": "2024-08-06 03:25:56" }, { "source": "Open Source ", "confidence": 55, "expired": false, "intel_tags": [], "find_time": "2022-09-16 14:51:57", "intel_types": [ "Malware" ], "update_time": "2024-08-23 15:00:39" }, { "source": "Open Source ", "confidence": 50, "expired": true, "intel_tags": [], "find_time": "2024-01-20 01:33:37", "intel_types": [ "Scanner" ], "update_time": "2024-08-22 01:18:11" } ] }, "scene": "", "basic": { "carrier": "Constantine Cybersecurity Ltd.", "location": { "country": "United Kingdom", "province": "England", "city": "Leeds", "lng": "-1.549557", "lat": "53.800302", "country_code": "GB" } }, "asn": { "rank": 4, "info": "ITECOM-ASN, NL", "number": 29529 }, "ports": [ { "port": 80, "module": "http", "product": "", "version": "", "detail": "" }, { "port": 443, "module": "https", "product": "", "version": "", "detail": "" } ], "cas": [ { "protocol": "https", "port": 443, "digital_certificate": { "subject": "CloudFlare Origin Certificate", "issuer": "", "fingerprint": "857b7363019f1bf550375d56fa1f483ffb72bdef", "purpose": "SSL client|SSL server|Netscape SSL server|Any Purpose|Any Purpose CA|OCSP helper", "verify": "SHA256withECDSA", "status": "0", "revoked": false, "begin": "2021-07-19 18:53:00", "end": "2036-07-15 18:53:00", "status_desc": "Valid", "serial_number": "738a81454f135c291ee8b3264243964d29dba125", "revoked_time": "" } } ], "update_time": "2024-08-22 10:40:46", "rdns_list": [ { "rdns": "keen.monitoring.internet-measurement.com", "get_time": "2023-12-12 00:00:00" } ], "sum_cur_domains": "3" } }, "response_code": 0, "verbose_msg": "OK" }` const ipReputation = "https://api.threatbook.cn/v3/scene/ip_reputation" const ipQuery = "https://api.threatbook.cn/v3/ip/query" type ipService struct { URL string ApiKey string ipQueryUrl string ipReputationUrl string // ipInfoMap map[string]IpInfo useCachedIPInfo bool ipInfoCache *expirable.LRU[string, IpInfo] } // func NewIpService(url, apiKey string, useCachedIPInfo bool) Service { // return &ipService{ // URL: url, // ApiKey: apiKey, // ipQueryUrl: fmt.Sprintf("%s?apikey=%s&resource=", ipQuery, apiKey), // ipReputationUrl: fmt.Sprintf("%s?apikey=%s&resource=", ipReputation, apiKey), // // ipInfoMap: make(map[string]IpInfo), // ipInfoCache: expirable.NewLRU[string, IpInfo](2048, nil, time.Minute*30), // useCachedIPInfo: useCachedIPInfo, // } // } func toModel(info *IpInfo) *model.IPInfo { if info == nil { return nil } sumCurDomains, err := strconv.Atoi(info.SumCurDomains) if err != nil { logging.Get().Err(err).Msgf("parse SumCurDomains: %s", info.SumCurDomains) } lon, err := strconv.ParseFloat(info.Basic.Location.Longitude, 32) if err != nil { logging.Get().Err(err).Msgf("parse Longitude: %s", info.Basic.Location.Longitude) } lat, err := strconv.ParseFloat(info.Basic.Location.Latitude, 32) if err != nil { logging.Get().Err(err).Msgf("parse Longitude: %s", info.Basic.Location.Latitude) } ipInfo := &model.IPInfo{ Carrier: info.Basic.Carrier, Location: model.Location{ Country: info.Basic.Location.Country, CountryCode: info.Basic.Location.CountryCode, Province: info.Basic.Location.Province, City: info.Basic.Location.City, Longitude: lon, Latitude: lat, }, OSSIntelligences: []model.Intelligence{}, OnlineIntelligences: []model.Intelligence{}, Ports: []model.Port{}, Samples: []model.Sample{}, Cas: []model.CAS{}, Rdnses: []model.RDNS{}, Scene: info.Scene, SumCurDomains: sumCurDomains, TagsClasses: []model.TagsClass{}, Asn: model.ASN{ Number: info.ASN.Number, Info: info.ASN.Info, RiskRank: info.ASN.Rank, }, Judgments: []string{}, } if len(info.Judgments) > 0 { ipInfo.Judgments = append(ipInfo.Judgments, info.Judgments...) } for _, intell := range info.Intelligences.OpenSource { findTime, err := time.Parse("2006-01-02 15:04:05", intell.FindTime) if err != nil { logging.Get().Err(err).Msgf("FindTime :%s", intell.FindTime) } updateTime, err := time.Parse("2006-01-02 15:04:05", intell.UpdateTime) if err != nil { logging.Get().Err(err).Msgf("UpdateTime :%s", intell.UpdateTime) } intelTags := []model.TagsClass{} for _, tag := range intell.IntelTags { intelTags = append(intelTags, model.TagsClass{ TagsType: tag.TagsType, Tags: strings.Join(tag.Tags, ","), }) } ipInfo.OSSIntelligences = append(ipInfo.OSSIntelligences, model.Intelligence{ Source: intell.Source, FindTime: findTime.UnixMilli(), UpdateTime: updateTime.UnixMilli(), Confidence: intell.Confidence, Expired: intell.Expired, IntelTypes: intell.IntelTypes, IntelTags: intelTags, }) } for _, intell := range info.Intelligences.ThreatbookLab { findTime, err := time.Parse("2006-01-02 15:04:05", intell.FindTime) if err != nil { logging.Get().Err(err).Msgf("FindTime :%s", intell.FindTime) } updateTime, err := time.Parse("2006-01-02 15:04:05", intell.UpdateTime) if err != nil { logging.Get().Err(err).Msgf("UpdateTime :%s", intell.UpdateTime) } intelTags := []model.TagsClass{} for _, tag := range intell.IntelTags { intelTags = append(intelTags, model.TagsClass{ TagsType: tag.TagsType, Tags: strings.Join(tag.Tags, ","), }) } ipInfo.OnlineIntelligences = append(ipInfo.OnlineIntelligences, model.Intelligence{ Source: intell.Source, FindTime: findTime.UnixMilli(), UpdateTime: updateTime.UnixMilli(), Confidence: intell.Confidence, Expired: intell.Expired, IntelTypes: intell.IntelTypes, IntelTags: intelTags, }) } for _, port := range info.Ports { ipInfo.Ports = append(ipInfo.Ports, model.Port{ Port: int16(port.Port), Module: port.Module, Product: port.Product, Version: port.Version, Detail: port.Detail, }) } for _, sample := range info.Samples { scanTime, err := time.Parse("2006-01-02 15:04:05", sample.ScanTime) if err != nil { logging.Get().Err(err).Msgf("scanTime :%s", sample.ScanTime) } ipInfo.Samples = append(ipInfo.Samples, model.Sample{ Hash: sample.Hash, ScanTime: scanTime.UnixMilli(), Ratio: sample.Ratio, MalwareType: sample.MalwareType, MalwareFamily: sample.MalwareFamily, }) } for _, cas := range info.CAS { ipInfo.Cas = append(ipInfo.Cas, model.CAS{ Protocol: cas.Protocol, Port: cas.Port, Period: cas.Period, DigitalCertificate: model.Certificate{ Subject: cas.DigitalCertificate.Subject, Issuer: cas.DigitalCertificate.Issuer, Fingerprint: cas.DigitalCertificate.Fingerprint, Purpose: cas.DigitalCertificate.Purpose, Verify: cas.DigitalCertificate.Verify, Status: cas.DigitalCertificate.Status, Revoked: cas.DigitalCertificate.Revoked, Begin: cas.DigitalCertificate.Begin, End: cas.DigitalCertificate.End, StatusDesc: cas.DigitalCertificate.StatusDesc, SerialNumber: cas.DigitalCertificate.SerialNumber, RevokedTime: cas.DigitalCertificate.RevokedTime, }, }) } for _, dns := range info.RDNSList { ipInfo.Rdnses = append(ipInfo.Rdnses, model.RDNS{ RDNS: dns.RDNS, GetTime: dns.GetTime, }) } for _, tagclass := range info.TagsClasses { ipInfo.TagsClasses = append(ipInfo.TagsClasses, model.TagsClass{ TagsType: tagclass.TagsType, Tags: strings.Join(tagclass.Tags, ","), }) } return ipInfo } func (s *ipService) QueryIP(ip string) (*model.IPInfo, error) { ipInfo := IpInfo{} // ipI, ok := s.ipInfoMap[ip] ipI, ok := s.ipInfoCache.Get(ip) if ok && s.useCachedIPInfo { logging.Get().Info().Msg("hit ipinfo cache") ipInfo = ipI ipQueryInfo := toModel(&ipInfo) // isMalicious, _ := s.isMalicious(ip) // ipQueryInfo.IsMalicious = isMalicious return ipQueryInfo, nil } else { resp, err := http.DefaultClient.Get(s.ipQueryUrl + ip) if err != nil { return nil, err } data, err := io.ReadAll(resp.Body) if err != nil { return nil, err } logging.Get().Info().Msg(string(data)) var respData RespData err = json.Unmarshal([]byte(data), &respData) if err != nil { return nil, err } if respData.ResponseCode != 0 { return nil, fmt.Errorf("error occurs: %s", respData.VerboseMsg) } if info, ok := respData.Data[ip]; ok { d, err := info.MarshalJSON() if err != nil { return nil, err } err = json.Unmarshal(d, &ipInfo) if err != nil { return nil, err } if s.useCachedIPInfo { // s.ipInfoMap[ip] = ipInfo s.ipInfoCache.Add(ip, ipInfo) } ipQueryInfo := toModel(&ipInfo) isMalicious, _ := s.isMalicious(ip) ipQueryInfo.IsMalicious = isMalicious return ipQueryInfo, nil } } return &model.IPInfo{}, nil } func (s *ipService) isMalicious(ip string) (bool, error) { resp, err := http.DefaultClient.Get(s.ipReputationUrl + ip) if err != nil { return false, err } data, err := io.ReadAll(resp.Body) if err != nil { return false, err } logging.Get().Info().Msg(string(data)) // fmt.Println(string(data)) var respData RespData err = json.Unmarshal([]byte(data), &respData) if err != nil { return false, err } if reputation, ok := respData.Data[ip]; ok { var rep Reputation d, err := reputation.MarshalJSON() if err != nil { return false, err } err = json.Unmarshal(d, &rep) if err != nil { return false, err } return rep.IsMalicious, nil } return false, nil }