diff --git a/README.md b/README.md index 46be0fdf..562ed13a 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ For further Information see the [license file](https://misp.github.io/misp-modul * [CVE Advanced Lookup](https://misp.github.io/misp-modules/expansion/#cve-advanced-lookup) - An expansion module to query the CIRCL CVE search API for more information about a vulnerability (CVE). * [Cytomic Orion Lookup](https://misp.github.io/misp-modules/expansion/#cytomic-orion-lookup) - An expansion module to enrich attributes in MISP by quering the Cytomic Orion API * [DBL Spamhaus Lookup](https://misp.github.io/misp-modules/expansion/#dbl-spamhaus-lookup) - Checks Spamhaus DBL for a domain name. +* [DNS Record Enumeration](https://misp.github.io/misp-modules/expansion/#dns-record-enumeration) - Query DNS data types including SOA and TXT, preserve complete resource records, and extract linked domain, IP, email and URL pivots. [JSON documentation](documentation/expansion/dns_query.json). * [DNS Resolver](https://misp.github.io/misp-modules/expansion/#dns-resolver) - Simple DNS expansion service to resolve IP address from MISP attributes * [DOCX Enrich](https://misp.github.io/misp-modules/expansion/#docx-enrich) - Module to extract freetext from a .docx document. * [DomainTools Lookup](https://misp.github.io/misp-modules/expansion/#domaintools-lookup) - DomainTools MISP expansion module. diff --git a/documentation/expansion/dns_query.json b/documentation/expansion/dns_query.json new file mode 100644 index 00000000..18036086 --- /dev/null +++ b/documentation/expansion/dns_query.json @@ -0,0 +1,119 @@ +{ + "name": "DNS Record Enumeration", + "module": "dns_query", + "module-type": [ + "expansion" + ], + "description": "Actively query DNS data types and collect complete resource records and typed correlation pivots in MISP.", + "requirements": [ + "dnspython", + "PyMISP", + "DNS access to a recursive resolver" + ], + "features": [ + "Query every data type known to the installed dnspython version explicitly; dnspython 2.8.0 enumerates 74 types. Common pivot-rich types are scheduled first. Results depend on resolver support and the time budget.", + "Include SOA, TXT, SPF, MX, NS, CNAME, DNAME, PTR, SRV, NAPTR, CAA, HTTPS/SVCB, DNSSEC and legacy data types. record_types can select a comma-separated subset or private types such as TYPE65400.", + "Use absolute DNS names with resolver search disabled. Normalize IDNA and case. Domains, hostnames, domain|ip, hostname|port, URLs, email/email-src/email-dst/dns-soa-email and ip-src/ip-dst (including |port) are supported. IPs and URL IP literals use the reverse DNS name.", + "Retain each resource record's complete presentation-format RDATA, actual owner, class and first observed TTL in a dns-resource-record object. Object first_seen and last_seen mark this run's observation time.", + "Keep records from answer, authority and additional sections, including SOA and glue in negative replies. queried-name and section preserve context; an authority SOA describes its own zone, not a positive SOA answer at the input name.", + "Deduplicate by owner, class, type and RDATA across queries; preserve all query names and response sections on the object. Aliases and canonical answers retain separate owner names.", + "Extract domain targets, IPs, MX/SRV/URI priorities, SRV weights and ports, SOA timings and responsible mailbox, RP mailbox, HTTPS/SVCB address hints and ports, and URLs from URI/CAA/TXT data. All other data remains available in rdata.", + "Concatenate TXT/SPF chunks for reading while retaining quoting and chunk boundaries in rdata. Parse explicit SPF include/redirect/a/mx/exists domains and IPv4/IPv6 networks. Extract email addresses and HTTP(S) reporting URLs; no web request is made.", + "Query TXT at _dmarc, _mta-sts and _smtp._tls under the input name by default. One bounded follow-up stage queries A and AAAA for up to 16 discovered domain targets. Follow-ups share the original time budget and are not recursively expanded.", + "Eight worker threads share immutable resolver configuration and a total DNS time budget. Preserve successful results when other queries time out, are refused, return NXDOMAIN, or cannot start within the budget. Report limitations in the first object's comment. No returned records produces an error.", + "Set EDNS DNSSEC OK to request available DNSSEC data. This module does not independently validate DNSSEC signatures.", + "Return the original attribute with its UUID and metadata; every record object is derived-from that UUID. All generated attributes use to_ids=false. The bundled template provides strict validation even before a PyMISP template update." + ], + "input": { + "format": "misp_standard", + "types": [ + "domain", + "hostname", + "domain|ip", + "hostname|port", + "url", + "email", + "email-src", + "email-dst", + "dns-soa-email", + "ip-src", + "ip-dst", + "ip-src|port", + "ip-dst|port" + ], + "required_attribute_fields": [ + "type", + "value", + "uuid" + ] + }, + "output": { + "format": "misp_standard", + "objects": [ + "dns-resource-record" + ], + "max_records": 1000, + "errors": "Malformed requests, unsupported attributes, invalid values/configuration or no resource records return an error. Individual DNS failures preserve successful records with diagnostic comments." + }, + "config": { + "nameserver": { + "default": "System resolver", + "description": "Optional recursive DNS server IP address (IPv4 or IPv6)." + }, + "record_types": { + "default": "all", + "description": "All dnspython-known data types or a comma-separated list such as A,AAAA,SOA,TXT,MX,NS,CAA,HTTPS. Up to 128 explicit types. Excludes ANY, AXFR, IXFR, OPT, TKEY, TSIG, MAILA, MAILB and reserved values. A/AAAA follow-ups and optional mail-policy TXT queries are controlled separately." + }, + "query_timeout": { + "default": 2, + "range": "0.1–10 seconds", + "description": "Per-query lifetime, including resolver retries, capped by the remaining total budget." + }, + "max_time": { + "default": 30, + "range": "1–120 seconds", + "description": "Total budget shared by all initial DNS queries and address follow-ups. Serialization adds some processing time." + }, + "max_pivots": { + "default": 16, + "range": "0–64 names", + "description": "Maximum discovered domain names to resolve in a single A/AAAA follow-up stage. Set 0 to disable." + }, + "mail_policy": { + "default": true, + "description": "Boolean or true/false/1/0 string. Query _dmarc, _mta-sts and _smtp._tls TXT names for non-IP input. This uses the exact input domain, without guessing an organizational domain." + } + }, + "limitations": [ + "This enumerates record types at known names, not all records in a zone. DNS has no general query that reveals every owner name. It does not guess subdomains, DKIM selectors or SRV service labels; supply those names explicitly.", + "It does not send ANY, perform zone transfers, scan all 65,536 numeric type codes, walk NSEC chains, or recursively expand newly found names. Private numeric types can be selected explicitly.", + "A recursive resolver may filter types, return cached answers or omit records. No claim of complete zone coverage or DNSSEC authentication is made. A TTL is the resolver-returned remaining TTL.", + "Some obsolete or specialized types have no dnspython-specific decoder; full generic RDATA is still retained, while extracted pivot fields depend on the decoder.", + "Queries go to the configured recursive resolver. Eight workers and bounded time, result count and follow-up count limit each expansion's workload." + ], + "installation": [ + "Install the module through the normal misp-modules package deployment; dnspython is already in the minimal/all dependency extras and PyMISP is a core dependency.", + "Restart misp-modules and enable dns_query in MISP Enrichment plugin settings. No API key is required.", + "Import/update the dns-resource-record template in MISP from MISP/misp-objects. Until available upstream, the identical definition is bundled at misp_modules/data/dns-resource-record/definition.json; import it into the MISP object-template store with your normal administrator workflow.", + "Expand an existing supported attribute using its original UUID. Both input and results use misp_standard format." + ], + "verification": { + "command": "python -m pytest -q -o log_cli=false tests/test_dns_query.py tests/test_dns.py tests/test_url_dns.py", + "coverage": "Record types, SOA/TXT and policy pivots, unknown types, aliases and deduplication, negative authority/glue records, supported inputs, configuration, partial failures, budgets, limits, metadata and strict object validation. A local UDP DNS server exercises real dnspython wire parsing and EDNS.", + "limitations": "Public DNS and deployment into a running MISP instance have not been tested." + }, + "example": { + "module": "dns_query", + "attribute": { + "type": "domain", + "value": "example.org", + "uuid": "5b582d80-7a7e-4b6a-9f22-77656e72bb3b" + }, + "config": { + "record_types": "all", + "max_time": 30, + "max_pivots": 16, + "mail_policy": true + } + } +} diff --git a/misp_modules/data/dns-resource-record/definition.json b/misp_modules/data/dns-resource-record/definition.json new file mode 100644 index 00000000..6588317b --- /dev/null +++ b/misp_modules/data/dns-resource-record/definition.json @@ -0,0 +1,166 @@ +{ + "attributes": { + "domain": { + "description": "Domain name extracted from the resource record data.", + "misp-attribute": "domain", + "multiple": true, + "to_ids": false, + "ui-priority": 1 + }, + "email": { + "description": "Email address extracted from resource record data or mail policy text.", + "misp-attribute": "email", + "multiple": true, + "to_ids": false, + "ui-priority": 1 + }, + "expire": { + "description": "SOA expiry interval in seconds.", + "disable_correlation": true, + "misp-attribute": "counter", + "to_ids": false, + "ui-priority": 0 + }, + "ip": { + "description": "IP address or network extracted from address records, SPF or service binding hints.", + "misp-attribute": "ip-dst", + "multiple": true, + "to_ids": false, + "ui-priority": 1 + }, + "minimum": { + "description": "SOA minimum negative-cache TTL in seconds.", + "disable_correlation": true, + "misp-attribute": "counter", + "to_ids": false, + "ui-priority": 0 + }, + "port": { + "description": "Service port extracted from SRV or service binding data.", + "disable_correlation": true, + "misp-attribute": "port", + "multiple": true, + "to_ids": false, + "ui-priority": 0 + }, + "priority": { + "description": "MX preference, SRV/URI priority, or service binding priority.", + "disable_correlation": true, + "misp-attribute": "counter", + "to_ids": false, + "ui-priority": 0 + }, + "queried-name": { + "description": "Absolute DNS query name, without the final root dot. May differ from rrname when aliases are followed.", + "misp-attribute": "domain", + "multiple": true, + "to_ids": false, + "ui-priority": 1 + }, + "rdata": { + "description": "Complete resource record data in DNS presentation format; preserves TXT quoting, SOA fields and unknown types.", + "misp-attribute": "text", + "to_ids": false, + "ui-priority": 1 + }, + "refresh": { + "description": "SOA secondary refresh interval in seconds.", + "disable_correlation": true, + "misp-attribute": "counter", + "to_ids": false, + "ui-priority": 0 + }, + "retry": { + "description": "SOA retry interval in seconds.", + "disable_correlation": true, + "misp-attribute": "counter", + "to_ids": false, + "ui-priority": 0 + }, + "rrclass": { + "description": "DNS resource record class, normally IN.", + "disable_correlation": true, + "misp-attribute": "text", + "to_ids": false, + "ui-priority": 0 + }, + "rrname": { + "description": "Owner name in DNS presentation form, without the final root dot except for the DNS root itself (.). Text permits root, wildcard and escaped owner names.", + "misp-attribute": "text", + "to_ids": false, + "ui-priority": 1 + }, + "rrtype": { + "description": "DNS resource record type mnemonic, or TYPE followed by its numeric code.", + "disable_correlation": true, + "misp-attribute": "text", + "to_ids": false, + "ui-priority": 1 + }, + "section": { + "description": "DNS response section where this record was observed. Authority and additional records describe their own rrname, not necessarily the queried name.", + "disable_correlation": true, + "misp-attribute": "text", + "multiple": true, + "to_ids": false, + "ui-priority": 0, + "values_list": [ + "answer", + "authority", + "additional" + ] + }, + "serial": { + "description": "SOA zone serial number.", + "disable_correlation": true, + "misp-attribute": "counter", + "to_ids": false, + "ui-priority": 0 + }, + "soa-email": { + "description": "SOA responsible mailbox converted from DNS RNAME notation to an email address.", + "misp-attribute": "dns-soa-email", + "to_ids": false, + "ui-priority": 1 + }, + "text": { + "description": "Decoded TXT/SPF character strings concatenated in order; rdata preserves original chunk boundaries.", + "misp-attribute": "text", + "multiple": true, + "to_ids": false, + "ui-priority": 0 + }, + "ttl": { + "description": "Remaining time to live in seconds as returned by the resolver.", + "disable_correlation": true, + "misp-attribute": "counter", + "to_ids": false, + "ui-priority": 0 + }, + "url": { + "description": "URL extracted from resource record data; no HTTP request is implied.", + "misp-attribute": "url", + "multiple": true, + "to_ids": false, + "ui-priority": 1 + }, + "weight": { + "description": "SRV or URI relative weight.", + "disable_correlation": true, + "misp-attribute": "counter", + "to_ids": false, + "ui-priority": 0 + } + }, + "description": "A single DNS resource record observed through active resolution, preserving its query name, owner, type, class, TTL and complete RDATA alongside extracted correlation pivots. Object first_seen and last_seen record observation time, not zone history.", + "meta-category": "network", + "name": "dns-resource-record", + "required": [ + "queried-name", + "rrname", + "rrtype", + "rdata" + ], + "uuid": "c6138676-09a5-4c3d-b73e-654df15e3242", + "version": 1 +} \ No newline at end of file diff --git a/misp_modules/modules/expansion/dns_query.py b/misp_modules/modules/expansion/dns_query.py new file mode 100644 index 00000000..994f88dc --- /dev/null +++ b/misp_modules/modules/expansion/dns_query.py @@ -0,0 +1,419 @@ +"""Enumerate DNS data types and retain complete records and correlation pivots.""" + +import ipaddress +import json +import math +import re +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from functools import lru_cache +from importlib import resources +from urllib.parse import urlsplit +from uuid import UUID + +import dns.exception +import dns.flags +import dns.name +import dns.rdataclass +import dns.rdatatype +import dns.resolver +from pymisp import MISPAttribute, MISPEvent, MISPObject +from pymisp.exceptions import PyMISPError + +from . import check_input_attribute, standard_error_message + +mispattributes = { + "input": [ + "domain", + "hostname", + "domain|ip", + "hostname|port", + "url", + "email", + "email-src", + "email-dst", + "dns-soa-email", + "ip-src", + "ip-dst", + "ip-src|port", + "ip-dst|port", + ], + "format": "misp_standard", +} +moduleinfo = { + "version": "1.0", + "author": "MISP", + "description": "Enumerate DNS record types and extract domain, IP, email and URL pivots.", + "module-type": ["expansion"], + "name": "DNS Record Enumeration", + "logo": "", + "requirements": ["dnspython", "PyMISP"], + "features": ( + "Query each dnspython-known DNS data type explicitly, including SOA, TXT, MX, NS, SRV, CAA and DNSSEC. " + "Return complete dns-resource-record objects with owner, type, class, TTL, RDATA and typed pivots. " + "Accept DNS names, URLs, email addresses and IPs (reverse DNS). Query common mail-policy TXT names and " + "optionally resolve discovered names to A/AAAA. Eight workers share a total time budget; failures retain " + "successful results. See documentation/expansion/dns_query.json for configuration and limitations." + ), + "references": ["https://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml"], + "input": "A supported MISP attribute containing type, value and UUID.", + "output": "Linked dns-resource-record objects; newly generated attributes have to_ids=false.", +} +moduleconfig = ["nameserver", "record_types", "query_timeout", "max_time", "max_pivots", "mail_policy"] + +# These are reserved values, query/meta types, authentication or EDNS machinery, +# rather than ordinary resource data to enumerate. ANY is deliberately excluded. +EXCLUDED_TYPES = {0, 41, 249, 250, 251, 252, 253, 254, 255, 65535} +DEFAULT_TYPES = tuple(t.name.replace("_", "-") for t in dns.rdatatype.RdataType if t.value not in EXCLUDED_TYPES) +COMMON_TYPES = ("A", "AAAA", "CNAME", "SOA", "NS", "MX", "TXT", "SRV", "CAA", "HTTPS", "SVCB", "PTR") +MAX_RECORDS = 1000 + + +def _hostname(value): + value = value.removesuffix(".").encode("idna").decode("ascii").lower() + if ( + not value + or len(value) > 253 + or any(not re.fullmatch(r"[a-z0-9_](?:[a-z0-9_-]{0,61}[a-z0-9_])?", label) for label in value.split(".")) + ): + raise ValueError("Invalid DNS name.") + return value + + +def _query_name(attribute): + value = attribute["value"] + if not isinstance(value, str) or not value or any(c.isspace() or ord(c) < 32 for c in value): + raise ValueError("The input must be a non-empty string without whitespace or control characters.") + kind = attribute["type"] + if "|" in kind: + value, separator, suffix = value.partition("|") + if not separator or not suffix: + raise ValueError("Invalid composite attribute.") + if kind.endswith("|port") and (not suffix.isascii() or not suffix.isdigit() or not 1 <= int(suffix) <= 65535): + raise ValueError("Invalid port in composite attribute.") + if kind == "domain|ip": + ipaddress.ip_address(suffix) + if kind == "url": + parsed = urlsplit(value) + if not parsed.scheme or not parsed.netloc or not parsed.hostname or "%" in parsed.hostname: + raise ValueError("The URL must include a scheme and valid hostname.") + parsed.port # Validate an explicit port even though DNS does not use it. + value = parsed.hostname + elif kind in {"email", "email-src", "email-dst", "dns-soa-email"}: + local, separator, value = value.rpartition("@") + if not separator or not local: + raise ValueError("The email address must contain a local part and domain.") + try: + address = ipaddress.ip_address(value) + except ValueError: + if kind.startswith("ip-"): + raise ValueError("Invalid IP address.") + return _hostname(value) + ".", False + return address.reverse_pointer + ".", True + + +def _number(config, key, default, minimum, maximum, integer=False): + raw = config.get(key, default) + if raw is None or raw == "": + raw = default + if isinstance(raw, bool): + raise ValueError(f"Invalid {key}.") + value = float(raw) + if not math.isfinite(value) or not minimum <= value <= maximum or (integer and not value.is_integer()): + raise ValueError(f"{key} must be between {minimum} and {maximum}.") + return int(value) if integer else value + + +def _settings(config): + if not isinstance(config, dict): + raise ValueError("The module config must be a JSON object.") + record_types = config.get("record_types", "all") + if record_types is None or record_types == "": + record_types = "all" + if not isinstance(record_types, str): + raise ValueError("record_types must be 'all' or a comma-separated list.") + if record_types.strip().lower() == "all": + types = set(DEFAULT_TYPES) + else: + types = set() + for item in record_types.split(","): + rdtype = dns.rdatatype.from_text(item.strip().upper()) + if int(rdtype) in EXCLUDED_TYPES: + raise ValueError("record_types cannot include reserved, meta, transfer or authentication types.") + types.add(dns.rdatatype.to_text(rdtype)) + if len(types) > 128: + raise ValueError("At most 128 record types can be specified.") + types = [t for t in COMMON_TYPES if t in types] + sorted(types - set(COMMON_TYPES)) + mail_policy = config.get("mail_policy", "true") + if mail_policy is None or mail_policy == "": + mail_policy = "true" + if str(mail_policy).lower() not in {"true", "false", "1", "0"}: + raise ValueError("mail_policy must be true or false.") + return ( + types, + _number(config, "query_timeout", 2, 0.1, 10), + _number(config, "max_time", 30, 1, 120), + _number(config, "max_pivots", 16, 0, 64, integer=True), + str(mail_policy).lower() in {"true", "1"}, + ) + + +@lru_cache(maxsize=1) +def _object_template(): + # Keep this identical to MISP/misp-objects/objects/dns-resource-record. + return json.loads( + resources.files("misp_modules").joinpath("data", "dns-resource-record", "definition.json").read_text() + ) + + +def _text_pivots(text): + """Extract explicit policy mechanisms, addresses and URLs, not arbitrary words.""" + for match in re.finditer(r"https?://[^\s\"<>;]+", text): + yield "url", match.group() + for match in re.finditer(r"(?:mailto:)?([a-zA-Z0-9_.+%-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})", text): + yield "email", match[1] + if text.lower().startswith("v=spf1"): + for token in text.split()[1:]: + token = token.lstrip("+-~?") + match = re.fullmatch(r"ip[46]:(.+)", token, re.IGNORECASE) + if match: + try: + network = ipaddress.ip_network(match[1], strict=False) + if network.version == (4 if token.lower().startswith("ip4:") else 6): + yield "ip", str(network.network_address) if network.num_addresses == 1 else str(network) + except ValueError: + pass + match = re.fullmatch(r"(?:include:|redirect=|a:|mx:|exists:)([^/]+)(?:/.*)?", token, re.IGNORECASE) + if match and "%" not in match[1]: # SPF macros require a mail transaction context. + try: + yield "domain", _hostname(match[1]) + except ValueError: + pass + + +def _pivots(record): + kind = dns.rdatatype.to_text(record.rdtype) + for field in ( + "target", + "exchange", + "mname", + "replacement", + "next", + "signer", + "intermediate", + "map822", + "mapx400", + "fqdn", + "gateway", + ): + value = getattr(record, field, None) + if isinstance(value, dns.name.Name) and value != dns.name.root: + yield "domain", value.to_text().rstrip(".").lower() + if kind in {"A", "AAAA"}: + yield "ip", record.address + if kind == "SOA": + if len(record.rname.labels) > 2: + local = record.rname.labels[0].decode("utf-8", errors="replace") + domain = dns.name.Name(record.rname.labels[1:]).to_text().rstrip(".").lower() + yield "soa-email", f"{local}@{domain}" + for field in ("serial", "refresh", "retry", "expire", "minimum"): + yield field, str(getattr(record, field)) + if kind == "RP": + mailbox = record.mbox + if len(mailbox.labels) > 2: + yield "email", mailbox.labels[0].decode("utf-8", errors="replace") + "@" + dns.name.Name( + mailbox.labels[1:] + ).to_text().rstrip(".") + if record.txt != dns.name.root: + yield "domain", record.txt.to_text().rstrip(".").lower() + for field, relation in ( + ("preference", "priority"), + ("priority", "priority"), + ("weight", "weight"), + ("port", "port"), + ): + value = getattr(record, field, None) + if value is not None and (relation != "port" or value): + yield relation, str(value) + if kind in {"SVCB", "HTTPS"}: + for key, param in record.params.items(): + if int(key) in {4, 6}: + for address in param.addresses: + yield "ip", address + elif int(key) == 3: + yield "port", str(param.port) + if kind in {"TXT", "SPF"}: + text = b"".join(record.strings).decode("utf-8", errors="replace") + yield "text", text + yield from _text_pivots(text) + if kind == "URI": + target = record.target.decode("utf-8", errors="replace") + if target.startswith(("http://", "https://")): + yield "url", target + if kind == "CAA": + tag = record.tag.decode("ascii", errors="replace").lower() + value = record.value.decode("utf-8", errors="replace") + if tag in {"issue", "issuewild"} and value.split(";", 1)[0].strip(): + try: + yield "domain", _hostname(value.split(";", 1)[0].strip()) + except ValueError: + pass + yield from _text_pivots(value) + + +def _sections(response): + if response is None: + return [] + return [ + (section, rrset) + for section in ("answer", "authority", "additional") + for rrset in getattr(response, section, []) + if int(rrset.rdtype) not in EXCLUDED_TYPES + ] + + +def _lookup(resolver, name, kind, deadline, timeout): + remaining = deadline - time.monotonic() + if remaining <= 0: + return name, [], f"{name} {kind}: total time budget exhausted" + try: + answer = resolver.resolve(name, kind, search=False, lifetime=min(timeout, remaining)) + # Retain alias RRsets with their own owners, instead of attributing the + # canonical target's data to the original input name. + return name, _sections(answer.response), None + except dns.resolver.NoAnswer as error: + response = error.kwargs.get("response") + return name, _sections(response), None + except dns.resolver.NXDOMAIN as error: + rrsets = [entry for response in error.kwargs.get("responses", {}).values() for entry in _sections(response)] + return name, rrsets, f"{name} {kind}: NXDOMAIN" + except dns.exception.DNSException as error: + return name, [], f"{name} {kind}: {type(error).__name__}" + + +def _enumerate(resolver, queries, deadline, timeout): + with ThreadPoolExecutor(max_workers=8) as executor: + # Resolver configuration stays immutable while resolve() runs concurrently. + futures = [executor.submit(_lookup, resolver, name, kind, deadline, timeout) for name, kind in queries] + return [future.result() for future in futures] + + +def handler(q=False): + if q is False: + return False + try: + request = json.loads(q) + if not isinstance(request, dict): + raise ValueError("The request must be a JSON object.") + attribute = request.get("attribute") + if not isinstance(attribute, dict) or not check_input_attribute(attribute): + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an uuid."} + if attribute["type"] not in mispattributes["input"]: + raise ValueError("Unsupported input attribute type.") + UUID(attribute["uuid"]) + name, reverse = _query_name(attribute) + config = request.get("config") + if config is None: + config = {} + types, timeout, max_time, max_pivots, mail_policy = _settings(config) + resolver = dns.resolver.Resolver() + resolver.timeout = timeout + resolver.lifetime = timeout + resolver.use_edns(edns=0, ednsflags=dns.flags.DO, payload=1232) + if config.get("nameserver"): + resolver.nameservers = [str(ipaddress.ip_address(config["nameserver"]))] + deadline = time.monotonic() + max_time + queries = [(name, kind) for kind in types] + if mail_policy and not reverse: + queries.extend((f"{prefix}.{name}", "TXT") for prefix in ("_dmarc", "_mta-sts", "_smtp._tls")) + + event = MISPEvent() + source = MISPAttribute() + source.from_dict(**attribute) + event.add_attribute(**source) + objects = {} + discovered = set() + warnings = [] + observed = datetime.now(timezone.utc) + + def collect(responses): + for queried, rrsets, warning in responses: + if warning: + warnings.append(warning) + for section, rrset in rrsets: + owner = rrset.name.to_text().lower() + if owner != ".": + owner = owner.rstrip(".") + kind = dns.rdatatype.to_text(rrset.rdtype) + for record in rrset: + key = (owner, rrset.rdclass, kind, record.to_text()) + if key in objects: + obj = objects[key] + if queried.rstrip(".") not in { + a.value for a in obj.get_attributes_by_relation("queried-name") + }: + obj.add_attribute("queried-name", queried.rstrip("."), to_ids=False) + if section not in {a.value for a in obj.get_attributes_by_relation("section")}: + obj.add_attribute("section", section, to_ids=False) + continue + if len(objects) >= MAX_RECORDS: + warnings.append(f"Result limit of {MAX_RECORDS} records reached.") + return + obj = MISPObject( + "dns-resource-record", strict=True, misp_objects_template_custom=_object_template() + ) + for relation, value in ( + ("queried-name", queried.rstrip(".")), + ("rrname", owner), + ("rrtype", kind), + ("rrclass", dns.rdataclass.to_text(rrset.rdclass)), + ("section", section), + ("ttl", str(rrset.ttl)), + ("rdata", record.to_text()), + ): + obj.add_attribute(relation, value, to_ids=False) + seen = set() + for relation, value in _pivots(record): + if not value or (relation, value) in seen: + continue + seen.add((relation, value)) + obj.add_attribute(relation, value, to_ids=False) + if relation == "domain": + discovered.add(value) + obj.first_seen = observed + obj.last_seen = observed + obj.add_reference(source.uuid, "derived-from") + objects[key] = obj + + collect(_enumerate(resolver, queries, deadline, timeout)) + # One bounded follow-up stage; discovered data never causes recursive crawling. + pivots = sorted(discovered - {name.rstrip(".")}) + if max_pivots and len(pivots) > max_pivots: + warnings.append(f"Address follow-ups limited to {max_pivots} of {len(pivots)} discovered names.") + followups = [] + if max_pivots and len(objects) < MAX_RECORDS: + for pivot in pivots[:max_pivots]: + try: + followups.extend((_hostname(pivot) + ".", kind) for kind in ("A", "AAAA")) + except ValueError: + continue + collect(_enumerate(resolver, followups, deadline, timeout)) + if not objects: + return {"error": f"No DNS records returned for {name} " + " ".join(dict.fromkeys(warnings))} + if warnings: + next(iter(objects.values())).comment = "Partial enumeration: " + " ".join(dict.fromkeys(warnings)) + for obj in objects.values(): + event.add_object(obj) + data = json.loads(event.to_json()) + return {"results": {key: data[key] for key in ("Attribute", "Object")}} + except (ValueError, TypeError, AttributeError, dns.exception.DNSException, PyMISPError) as error: + return {"error": f"DNS enumeration failed: {error}"} + + +def introspection(): + return mispattributes + + +def version(): + return dict(moduleinfo, config=moduleconfig) diff --git a/tests/test_dns_query.py b/tests/test_dns_query.py new file mode 100644 index 00000000..2c33bade --- /dev/null +++ b/tests/test_dns_query.py @@ -0,0 +1,424 @@ +import json +import socketserver +import threading +from types import SimpleNamespace +from unittest.mock import MagicMock + +import dns.exception +import dns.message +import dns.rdatatype +import dns.resolver +import dns.rrset +import pytest +from pymisp import MISPObject + +from misp_modules.modules.expansion import dns_query + +UUID = "5b582d80-7a7e-4b6a-9f22-77656e72bb3b" + + +@pytest.fixture +def resolver(monkeypatch): + mock = MagicMock() + mock.resolve.side_effect = dns.resolver.NoAnswer + monkeypatch.setattr(dns_query.dns.resolver, "Resolver", lambda: mock) + return mock + + +def _answers(resolver, answers): + def resolve(name, kind, *, search, lifetime): + assert search is False + assert 0 < lifetime <= 10 + answer = answers.get((name, kind), dns.resolver.NoAnswer()) + if isinstance(answer, Exception): + raise answer + if isinstance(answer, list): + rrsets = answer + else: + rrsets = [dns.rrset.from_text(name, 300, "IN", kind, *answer)] + return SimpleNamespace(response=SimpleNamespace(answer=rrsets)) + + resolver.resolve.side_effect = resolve + + +def _run(value="example.org", kind="domain", config=None, **metadata): + source = {"type": kind, "value": value, "uuid": UUID, **metadata} + return dns_query.handler(json.dumps({"attribute": source, "config": config})) + + +def _objects(result, kind=None): + objects = result["results"]["Object"] + return objects if kind is None else [obj for obj in objects if _values(obj, "rrtype") == [kind]] + + +def _values(obj, relation): + return [str(a["value"]) for a in obj["Attribute"] if a["object_relation"] == relation] + + +def test_all_types_are_queried_explicitly_and_templates_validate(resolver): + _answers(resolver, {("example.org.", "A"): ("192.0.2.1", "192.0.2.2")}) + result = _run(comment="Original source", to_ids=True, Tag=[{"name": "tlp:clear"}]) + queried = {(c.args[0], c.args[1]) for c in resolver.resolve.call_args_list} + assert {kind for name, kind in queried if name == "example.org."} == set(dns_query.DEFAULT_TYPES) + assert {"SOA", "TXT", "CAA", "TLSA", "DNSKEY", "SVCB", "HTTPS"} <= set(dns_query.DEFAULT_TYPES) + assert not set(dns_query.DEFAULT_TYPES) & {"ANY", "AXFR", "IXFR", "OPT", "TKEY", "TSIG", "TYPE0"} + assert {(f"{prefix}.example.org.", "TXT") for prefix in ("_dmarc", "_mta-sts", "_smtp._tls")} <= queried + assert len(_objects(result)) == 2 + source = result["results"]["Attribute"][0] + assert (source["uuid"], source["comment"], source["to_ids"]) == (UUID, "Original source", True) + assert source["Tag"][0]["name"] == "tlp:clear" + for obj in _objects(result): + strict = MISPObject( + "dns-resource-record", strict=True, misp_objects_template_custom=dns_query._object_template() + ) + strict.from_dict(**obj) + strict._validate() + assert _values(obj, "ttl") == ["300"] + assert _values(obj, "rrclass") == ["IN"] + assert obj["first_seen"] == obj["last_seen"] + assert all(a["to_ids"] is False for a in obj["Attribute"]) + assert [(r["referenced_uuid"], r["relationship_type"]) for r in obj["ObjectReference"]] == [ + (UUID, "derived-from") + ] + + +def test_soa_preserves_full_rdata_and_decodes_mailbox(resolver): + soa = r"ns.example.net. first\.last.example.org. 2026101701 3600 600 86400 300" + _answers(resolver, {("example.org.", "SOA"): (soa,)}) + obj = _objects(_run(config={"max_pivots": 0}), "SOA")[0] + assert _values(obj, "rdata") == [soa] + assert _values(obj, "domain") == ["ns.example.net"] + assert _values(obj, "soa-email") == ["first.last@example.org"] + assert [_values(obj, field)[0] for field in ("serial", "refresh", "retry", "expire", "minimum")] == [ + "2026101701", + "3600", + "600", + "86400", + "300", + ] + + +@pytest.mark.parametrize( + "kind,data,expected", + [ + ("MX", "10 mail.example.net.", {"domain": ["mail.example.net"], "priority": ["10"]}), + ( + "SRV", + "1 20 443 service.example.net.", + {"domain": ["service.example.net"], "port": ["443"], "weight": ["20"]}, + ), + ("NS", "ns.example.net.", {"domain": ["ns.example.net"]}), + ("PTR", "host.example.net.", {"domain": ["host.example.net"]}), + ("DNAME", "other.example.net.", {"domain": ["other.example.net"]}), + ("NAPTR", '1 1 "S" "SIP+D2U" "" _sip._udp.example.net.', {"domain": ["_sip._udp.example.net"]}), + ("CAA", '0 issue "ca.example.net; accounturi=123"', {"domain": ["ca.example.net"]}), + ("CAA", '0 iodef "mailto:security@example.net"', {"email": ["security@example.net"]}), + ("URI", '1 2 "https://service.example.net/path"', {"url": ["https://service.example.net/path"]}), + ( + "RP", + "security.example.net. info.example.net.", + {"email": ["security@example.net"], "domain": ["info.example.net"]}, + ), + ( + "HTTPS", + "1 svc.example.net. port=8443 ipv4hint=192.0.2.1 ipv6hint=2001:db8::1", + {"domain": ["svc.example.net"], "port": ["8443"], "ip": ["192.0.2.1", "2001:db8::1"]}, + ), + ("TLSA", "3 1 1 aabbccdd", {}), + ("TYPE65400", r"\# 4 aabbccdd", {}), + ], +) +def test_record_pivots_and_uncommon_records_are_retained(resolver, kind, data, expected): + _answers(resolver, {("example.org.", kind): (data,)}) + obj = _objects(_run(config={"record_types": kind, "mail_policy": False, "max_pivots": 0}), kind)[0] + record = dns.rrset.from_text("example.org.", 300, "IN", kind, data)[0] + assert _values(obj, "rdata") == [record.to_text()] + for relation, values in expected.items(): + assert _values(obj, relation) == values + + +def test_txt_chunks_spf_networks_and_mail_policy_pivots(resolver): + text = '"v=spf1 ip4:192.0.2.0/24 " "ip6:2001:db8::/32 include:mail.example.net redirect=other.example.net -all"' + _answers( + resolver, + { + ("example.org.", "TXT"): (text,), + ("_dmarc.example.org.", "TXT"): ('"v=DMARC1; p=reject; rua=mailto:reports@example.net"',), + ("_smtp._tls.example.org.", "TXT"): ('"v=TLSRPTv1; rua=https://reports.example.net/tls"',), + }, + ) + result = _run(config={"record_types": "TXT", "max_pivots": 0}) + obj = next(o for o in _objects(result) if _values(o, "rrname") == ["example.org"]) + assert _values(obj, "rdata") == [text] + assert _values(obj, "text") == [ + "v=spf1 ip4:192.0.2.0/24 ip6:2001:db8::/32 include:mail.example.net redirect=other.example.net -all" + ] + assert _values(obj, "ip") == ["192.0.2.0/24", "2001:db8::/32"] + assert set(_values(obj, "domain")) == {"mail.example.net", "other.example.net"} + assert any(_values(o, "email") == ["reports@example.net"] for o in _objects(result)) + assert any(_values(o, "url") == ["https://reports.example.net/tls"] for o in _objects(result)) + + +def test_alias_owners_deduplication_and_bounded_followups(resolver): + cname = dns.rrset.from_text("example.org.", 60, "IN", "CNAME", "target.example.net.") + target = dns.rrset.from_text("target.example.net.", 120, "IN", "A", "192.0.2.1") + _answers( + resolver, + { + ("example.org.", "A"): [cname, target], + ("example.org.", "CNAME"): [cname], + ("target.example.net.", "A"): [target], + ("target.example.net.", "AAAA"): [ + dns.rrset.from_text("target.example.net.", 120, "IN", "CNAME", "third.example.net.") + ], + }, + ) + result = _run(config={"record_types": "A,CNAME", "mail_policy": False}) + assert len(_objects(result, "A")) == 1 + obj = _objects(result, "A")[0] + assert _values(obj, "rrname") == ["target.example.net"] + assert _values(obj, "queried-name") == ["example.org", "target.example.net"] + queries = {(c.args[0], c.args[1]) for c in resolver.resolve.call_args_list} + assert ("target.example.net.", "AAAA") in queries + assert not any(name == "third.example.net." for name, _ in queries) + + +@pytest.mark.parametrize( + "kind,value,name", + [ + ("domain|ip", "Example.org|192.0.2.1", "example.org."), + ("hostname|port", "_sip._tcp.Example.org|443", "_sip._tcp.example.org."), + ("url", "https://Example.org:443/path", "example.org."), + ("email-src", "alice@Example.org", "example.org."), + ("email-dst", "bob@example.org", "example.org."), + ("dns-soa-email", "hostmaster@example.org", "example.org."), + ("hostname", "BÜCHER.example.", "xn--bcher-kva.example."), + ("ip-src", "192.0.2.1", "1.2.0.192.in-addr.arpa."), + ( + "ip-dst|port", + "2001:db8::1|443", + ipaddress_name := "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa.", + ), + ("url", "https://[2001:db8::1]/", ipaddress_name), + ], +) +def test_inputs_normalize_to_absolute_dns_names(resolver, kind, value, name): + _answers(resolver, {(name, "TXT"): ('"ok"',)}) + assert "results" in _run(value, kind, config={"record_types": "TXT", "mail_policy": False}) + assert resolver.resolve.call_args.args == (name, "TXT") + + +@pytest.mark.parametrize( + "kind,value", + [ + ("md5", "a" * 32), + ("ip-src", "not-ip"), + ("domain", "bad..example.org"), + ("hostname", "-bad.example"), + ("domain", "bad name"), + ("domain", ""), + ("domain", None), + ("domain", 1), + ("domain", "example.org\n"), + ("url", "example.org/path"), + ("url", "http://example.org:invalid/"), + ("url", "http://%65xample.org/"), + ("domain|ip", "example.org|invalid"), + ("hostname|port", "example.org|65536"), + ("email", "missing-at"), + ], +) +def test_invalid_inputs_do_not_send_queries(resolver, kind, value): + assert "error" in _run(value, kind) + resolver.resolve.assert_not_called() + + +@pytest.mark.parametrize( + "config", + [ + [], + {"record_types": []}, + {"record_types": "ANY"}, + {"record_types": "AXFR"}, + {"record_types": "IXFR"}, + {"record_types": "OPT"}, + {"record_types": "TSIG"}, + {"record_types": "TYPE0"}, + {"record_types": "TYPO"}, + {"nameserver": "invalid"}, + {"max_time": 0}, + {"max_time": 121}, + {"max_time": "nan"}, + {"query_timeout": "inf"}, + {"max_pivots": -1}, + {"max_pivots": 1.5}, + {"mail_policy": "maybe"}, + ], +) +def test_invalid_config_does_not_send_queries(resolver, config): + assert "error" in _run(config=config) + resolver.resolve.assert_not_called() + + +def test_partial_failures_preserve_records_and_resolver_configuration(resolver): + _answers( + resolver, + { + ("example.org.", "A"): ("192.0.2.1",), + ("example.org.", "AAAA"): dns.exception.Timeout(), + ("example.org.", "TXT"): dns.resolver.NoNameservers(), + }, + ) + result = _run( + config={"record_types": "A,AAAA,TXT", "nameserver": "2001:db8::53", "query_timeout": "1", "mail_policy": False} + ) + assert len(_objects(result)) == 1 + assert "AAAA: Timeout" in _objects(result)[0]["comment"] + assert "TXT: NoNameservers" in _objects(result)[0]["comment"] + assert resolver.nameservers == ["2001:db8::53"] + assert resolver.timeout == 1 + + +@pytest.mark.parametrize("error", [dns.resolver.NXDOMAIN(), dns.resolver.NoAnswer(), dns.exception.Timeout()]) +def test_no_results_has_clear_error(resolver, error): + resolver.resolve.side_effect = error + result = _run(config={"record_types": "A", "mail_policy": False}) + assert "No DNS records returned" in result["error"] + if not isinstance(error, dns.resolver.NoAnswer): + assert type(error).__name__ in result["error"] + + +def test_expired_deadline_sends_no_query(resolver, monkeypatch): + monkeypatch.setattr(dns_query.time, "monotonic", lambda: 100) + _, rrsets, warning = dns_query._lookup(resolver, "example.org.", "TXT", 99, 2) + assert not rrsets and "budget exhausted" in warning + resolver.resolve.assert_not_called() + + +def test_query_lifetime_is_limited_to_remaining_budget(resolver, monkeypatch): + monkeypatch.setattr(dns_query.time, "monotonic", lambda: 99.5) + dns_query._lookup(resolver, "example.org.", "TXT", 100, 2) + assert resolver.resolve.call_args.kwargs["lifetime"] == 0.5 + + +def test_record_and_pivot_limits_are_reported(resolver, monkeypatch): + _answers(resolver, {("example.org.", "NS"): ("ns1.example.net.", "ns2.example.net.", "ns3.example.net.")}) + result = _run(config={"record_types": "NS", "max_pivots": 1, "mail_policy": False}) + assert "limited to 1 of 3" in _objects(result)[0]["comment"] + assert len(resolver.resolve.call_args_list) == 3 # NS plus one A/AAAA pair. + monkeypatch.setattr(dns_query, "MAX_RECORDS", 2) + result = _run(config={"record_types": "NS", "mail_policy": False}) + assert len(_objects(result)) == 2 + assert "Result limit of 2" in _objects(result)[0]["comment"] + + +@pytest.mark.parametrize( + "payload", ["{", "null", "[]", "1", "{}", '{"attribute": []}', '{"attribute": {"type": "domain"}}'] +) +def test_malformed_requests(resolver, payload): + assert "error" in dns_query.handler(payload) + resolver.resolve.assert_not_called() + + +def test_invalid_uuid_and_metadata(resolver): + assert "error" in _run(uuid="invalid") + assert "error" in _run(uuid=None) + resolver.resolve.assert_not_called() + + +def test_introspection_and_version(): + assert dns_query.handler() is False + assert dns_query.introspection()["format"] == "misp_standard" + assert dns_query.version()["config"] == dns_query.moduleconfig + assert "config" not in dns_query.moduleinfo + + +def test_empty_misp_config_fields_use_defaults(): + assert dns_query._settings({key: "" for key in dns_query.moduleconfig}) == dns_query._settings({}) + + +def test_alias_in_noanswer_is_retained(resolver): + response = dns.message.make_response(dns.message.make_query("example.org.", "A")) + response.answer.append(dns.rrset.from_text("example.org.", 60, "IN", "CNAME", "target.example.net.")) + resolver.resolve.side_effect = dns.resolver.NoAnswer(response=response) + obj = _objects(_run(config={"record_types": "A", "max_pivots": 0, "mail_policy": False}), "CNAME")[0] + assert _values(obj, "domain") == ["target.example.net"] + + +@pytest.mark.parametrize("nxdomain", [False, True]) +def test_negative_responses_preserve_authority_and_glue(resolver, nxdomain): + response = dns.message.make_response(dns.message.make_query("empty.example.org.", "TXT")) + response.authority.append( + dns.rrset.from_text("example.org.", 300, "IN", "SOA", "ns.example.org. admin.example.org. 1 3600 600 86400 300") + ) + response.additional.append(dns.rrset.from_text("ns.example.org.", 300, "IN", "A", "192.0.2.53")) + if nxdomain: + response.set_rcode(3) + name = dns_query.dns.name.from_text("empty.example.org.") + error = dns.resolver.NXDOMAIN(qnames=[name], responses={name: response}) + else: + error = dns.resolver.NoAnswer(response=response) + resolver.resolve.side_effect = error + result = _run("empty.example.org", config={"record_types": "TXT", "mail_policy": False, "max_pivots": 0}) + soa = _objects(result, "SOA")[0] + glue = _objects(result, "A")[0] + assert _values(soa, "rrname") == ["example.org"] + assert _values(soa, "queried-name") == ["empty.example.org"] + assert _values(soa, "section") == ["authority"] + assert _values(glue, "section") == ["additional"] + assert _values(glue, "ip") == ["192.0.2.53"] + + +def test_root_authority_owner_is_preserved(resolver): + response = dns.message.make_response(dns.message.make_query("missing.invalid.", "TXT")) + response.authority.append( + dns.rrset.from_text(".", 300, "IN", "SOA", "a.root-servers.net. nstld.verisign-grs.com. 1 3600 600 86400 300") + ) + resolver.resolve.side_effect = dns.resolver.NoAnswer(response=response) + obj = _objects(_run("missing.invalid", config={"record_types": "TXT", "mail_policy": False, "max_pivots": 0}))[0] + assert _values(obj, "rrname") == ["."] + + +def test_invalid_repeated_root_dot_sends_no_query(resolver): + assert "error" in _run("example.org..") + resolver.resolve.assert_not_called() + + +def test_real_dns_wire_responses_over_local_udp(monkeypatch): + """Exercise dnspython Answer/RRset parsing and EDNS without public DNS.""" + requests = [] + + class DNSHandler(socketserver.BaseRequestHandler): + def handle(self): + wire, sock = self.request + query = dns.message.from_wire(wire) + requests.append(query) + response = dns.message.make_response(query) + question = query.question[0] + kind = dns.rdatatype.to_text(question.rdtype) + records = { + "A": "192.0.2.1", + "SOA": "ns.example.org. admin.example.org. 1 3600 600 86400 300", + "TXT": '"hello " "world"', + } + response.answer.append(dns.rrset.from_text(question.name, 123, "IN", kind, records[kind])) + sock.sendto(response.to_wire(), self.client_address) + + with socketserver.ThreadingUDPServer(("127.0.0.1", 0), DNSHandler) as server: + thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + real_resolver = dns.resolver.Resolver(configure=False) + real_resolver.nameservers = ["127.0.0.1"] + real_resolver.port = server.server_address[1] + monkeypatch.setattr(dns_query.dns.resolver, "Resolver", lambda: real_resolver) + try: + result = _run(config={"record_types": "A,SOA,TXT", "max_pivots": 0, "mail_policy": False}) + finally: + server.shutdown() + thread.join(timeout=2) + assert len(_objects(result)) == 3 + assert _values(_objects(result, "TXT")[0], "text") == ["hello world"] + assert _values(_objects(result, "SOA")[0], "soa-email") == ["admin@example.org"] + assert all(_values(obj, "ttl") == ["123"] for obj in _objects(result)) + assert len(requests) == 3 + assert all(query.edns == 0 and query.ednsflags & dns_query.dns.flags.DO for query in requests)