diff --git a/.github/workflows/skywalking.yaml b/.github/workflows/skywalking.yaml index 0644e008dca2..3e6a7e455a1d 100644 --- a/.github/workflows/skywalking.yaml +++ b/.github/workflows/skywalking.yaml @@ -303,11 +303,11 @@ jobs: # Slow tests: the `@Tag("slow")` ITs, all on Elasticsearch and OpenSearch containers, on JDK 11. Each matrix entry # builds the whole project once and runs its own ITs: - # - `ElasticSearchIT`, by class: library-elasticsearch-client against Elasticsearch 7, 8 and 9, and OpenSearch 1 - # and 2. - # - Elastic IT, by the `@Tag("elastic-it")` the storage-elasticsearch-plugin's ITs also carry: the Zipkin query DAO - # (`ZipkinQueryEsDAO*IT`) and the span-attached event queries (`SpanAttachedEventQuery*IT`), each on - # Elasticsearch 8, 9 and 9 sharded, and OpenSearch 2, 3 and 3 sharded. + # - `ElasticSearchIT`, by class: library-elasticsearch-client's requests, on a server of each request format it + # picks by version: Elasticsearch 6.8, 7.7, 7.8, 7.17, 8.19 and 9.3, and OpenSearch 1.3, 2.8 and 3.0. + # - Elastic IT, by the `@Tag("elastic-it")` the storage-elasticsearch-plugin's ITs also carry: every storage DAO, + # as nested ITs of one IT per server (`Elasticsearch*IT`, `OpenSearch*IT`): Elasticsearch 8, 9 and 9 sharded, + # and OpenSearch 1, 2, 3 and 3 sharded. slow-integration-test: if: | ( always() && ! cancelled() ) && @@ -357,8 +357,12 @@ jobs: fail-fast: false matrix: test: - - name: Cluster ZK ES + - name: Cluster ZK ES 8.18.8 config: test/e2e-v2/cases/cluster/zk/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Cluster ZK ES 9.3.0 + config: test/e2e-v2/cases/cluster/zk/es/e2e.yaml + env: ES_VERSION=9.3.0 - name: Cluster ZK BanyanDB config: test/e2e-v2/cases/cluster/zk/banyandb/e2e.yaml @@ -368,10 +372,18 @@ jobs: config: test/e2e-v2/cases/go/e2e.yaml - name: Agent NodeJS Frontend config: test/e2e-v2/cases/browser/e2e.yaml - - name: Agent NodeJS Frontend ES + - name: Agent NodeJS Frontend ES 8.18.8 + config: test/e2e-v2/cases/browser/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Agent NodeJS Frontend ES 9.3.0 config: test/e2e-v2/cases/browser/es/e2e.yaml - - name: Agent NodeJS Frontend ES Sharding + env: ES_VERSION=9.3.0 + - name: Agent NodeJS Frontend ES Sharding 8.18.8 + config: test/e2e-v2/cases/browser/es/es-sharding/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Agent NodeJS Frontend ES Sharding 9.3.0 config: test/e2e-v2/cases/browser/es/es-sharding/e2e.yaml + env: ES_VERSION=9.3.0 - name: Agent PHP config: test/e2e-v2/cases/php/e2e.yaml - name: Agent Python @@ -430,6 +442,9 @@ jobs: - name: Storage ES Sharding 8.18.8 config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml env: ES_VERSION=8.18.8 + - name: Storage ES Sharding 9.3.0 + config: test/e2e-v2/cases/storage/es/es-sharding/e2e.yaml + env: ES_VERSION=9.3.0 - name: Inspect API BanyanDB config: test/e2e-v2/cases/inspect/banyandb/e2e.yaml @@ -438,6 +453,9 @@ jobs: - name: Inspect API Elasticsearch 8.18.8 config: test/e2e-v2/cases/inspect/elasticsearch/e2e.yaml env: ES_VERSION=8.18.8 + - name: Inspect API Elasticsearch 9.3.0 + config: test/e2e-v2/cases/inspect/elasticsearch/e2e.yaml + env: ES_VERSION=9.3.0 - name: Runtime Rule MAL Storage BanyanDB config: test/e2e-v2/cases/runtime-rule/mal-storage/banyandb/e2e.yaml @@ -446,6 +464,9 @@ jobs: - name: Runtime Rule MAL Storage Elasticsearch 8.18.8 config: test/e2e-v2/cases/runtime-rule/mal-storage/elasticsearch/e2e.yaml env: ES_VERSION=8.18.8 + - name: Runtime Rule MAL Storage Elasticsearch 9.3.0 + config: test/e2e-v2/cases/runtime-rule/mal-storage/elasticsearch/e2e.yaml + env: ES_VERSION=9.3.0 - name: Runtime Rule LAL Hot-Update config: test/e2e-v2/cases/runtime-rule/lal/e2e.yaml - name: Runtime Rule Cluster (kind) @@ -462,10 +483,18 @@ jobs: - name: DSL Debug API — LAL statement config: test/e2e-v2/cases/dsl-debugging/lal-statement/e2e.yaml - - name: Alarm ES + - name: Alarm ES 8.18.8 + config: test/e2e-v2/cases/alarm/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Alarm ES 9.3.0 config: test/e2e-v2/cases/alarm/es/e2e.yaml - - name: Alarm ES Sharding + env: ES_VERSION=9.3.0 + - name: Alarm ES Sharding 8.18.8 + config: test/e2e-v2/cases/alarm/es/es-sharding/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Alarm ES Sharding 9.3.0 config: test/e2e-v2/cases/alarm/es/es-sharding/e2e.yaml + env: ES_VERSION=9.3.0 - name: Alarm MySQL config: test/e2e-v2/cases/alarm/mysql/e2e.yaml - name: Alarm PostgreSQL @@ -473,21 +502,36 @@ jobs: - name: Alarm BanyanDB config: test/e2e-v2/cases/alarm/banyandb/e2e.yaml - - name: Baseline-driven Alarm ES + - name: Baseline-driven Alarm ES 8.18.8 config: test/e2e-v2/cases/baseline/es/e2e.yaml - - name: Baseline-driven Alarm ES Sharding + env: ES_VERSION=8.18.8 + - name: Baseline-driven Alarm ES 9.3.0 + config: test/e2e-v2/cases/baseline/es/e2e.yaml + env: ES_VERSION=9.3.0 + - name: Baseline-driven Alarm ES Sharding 8.18.8 + config: test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Baseline-driven Alarm ES Sharding 9.3.0 config: test/e2e-v2/cases/baseline/es/es-sharding/e2e.yaml + env: ES_VERSION=9.3.0 - name: Baseline-driven Alarm BanyanDB config: test/e2e-v2/cases/baseline/banyandb/e2e.yaml - name: TTL ES 8.18.8 config: test/e2e-v2/cases/ttl/es/e2e.yaml env: ES_VERSION=8.18.8 + - name: TTL ES 9.3.0 + config: test/e2e-v2/cases/ttl/es/e2e.yaml + env: ES_VERSION=9.3.0 - name: Event BanyanDB config: test/e2e-v2/cases/event/banyandb/e2e.yaml - - name: Event ES + - name: Event ES 8.18.8 config: test/e2e-v2/cases/event/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Event ES 9.3.0 + config: test/e2e-v2/cases/event/es/e2e.yaml + env: ES_VERSION=9.3.0 - name: Event MySQL config: test/e2e-v2/cases/event/mysql/e2e.yaml @@ -498,9 +542,15 @@ jobs: - name: Log ES 8.18.8 config: test/e2e-v2/cases/log/es/e2e.yaml env: ES_VERSION=8.18.8 + - name: Log ES 9.3.0 + config: test/e2e-v2/cases/log/es/e2e.yaml + env: ES_VERSION=9.3.0 - name: Log ES 8.18.8 Sharding config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml env: ES_VERSION=8.18.8 + - name: Log ES 9.3.0 Sharding + config: test/e2e-v2/cases/log/es/es-sharding/e2e.yaml + env: ES_VERSION=9.3.0 - name: Log BanyanDB config: test/e2e-v2/cases/log/banyandb/e2e.yaml - name: AI Agent Conversations BanyanDB @@ -508,6 +558,9 @@ jobs: - name: AI Agent Conversations ES 8.18.8 config: test/e2e-v2/cases/ai-agent/es/e2e.yaml env: ES_VERSION=8.18.8 + - name: AI Agent Conversations ES 9.3.0 + config: test/e2e-v2/cases/ai-agent/es/e2e.yaml + env: ES_VERSION=9.3.0 - name: AI Agent Conversations MySQL config: test/e2e-v2/cases/ai-agent/mysql/e2e.yaml - name: AI Agent Conversations Postgres @@ -516,13 +569,24 @@ jobs: - name: Log FluentBit ES 8.18.8 config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml env: ES_VERSION=8.18.8 + - name: Log FluentBit ES 9.3.0 + config: test/e2e-v2/cases/log/fluent-bit/e2e.yaml + env: ES_VERSION=9.3.0 - name: Trace Profiling BanyanDB config: test/e2e-v2/cases/profiling/trace/banyandb/e2e.yaml - - name: Trace Profiling ES + - name: Trace Profiling ES 8.18.8 + config: test/e2e-v2/cases/profiling/trace/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Trace Profiling ES 9.3.0 config: test/e2e-v2/cases/profiling/trace/es/e2e.yaml - - name: Trace Profiling ES Sharding + env: ES_VERSION=9.3.0 + - name: Trace Profiling ES Sharding 8.18.8 + config: test/e2e-v2/cases/profiling/trace/es/es-sharding/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Trace Profiling ES Sharding 9.3.0 config: test/e2e-v2/cases/profiling/trace/es/es-sharding/e2e.yaml + env: ES_VERSION=9.3.0 - name: Trace Profiling MySQL config: test/e2e-v2/cases/profiling/trace/mysql/e2e.yaml - name: Trace Profiling Postgres @@ -704,10 +768,18 @@ jobs: - name: MariaDB Prometheus and slowsql config: test/e2e-v2/cases/mariadb/mariadb-slowsql/e2e.yaml - - name: Zipkin ES + - name: Zipkin ES 8.18.8 + config: test/e2e-v2/cases/zipkin/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Zipkin ES 9.3.0 config: test/e2e-v2/cases/zipkin/es/e2e.yaml - - name: Zipkin ES Sharding + env: ES_VERSION=9.3.0 + - name: Zipkin ES Sharding 8.18.8 config: test/e2e-v2/cases/zipkin/es/es-sharding/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Zipkin ES Sharding 9.3.0 + config: test/e2e-v2/cases/zipkin/es/es-sharding/e2e.yaml + env: ES_VERSION=9.3.0 - name: Zipkin OpenSearch 3.0.0 config: test/e2e-v2/cases/zipkin/opensearch/e2e.yaml env: OPENSEARCH_VERSION=3.0.0 @@ -758,8 +830,12 @@ jobs: config: test/e2e-v2/cases/aws/dynamodb/e2e.yaml - name: PromQL Service config: test/e2e-v2/cases/promql/e2e.yaml - - name: LogQL Service + - name: LogQL Service 8.18.8 + config: test/e2e-v2/cases/logql/e2e.yaml + env: ES_VERSION=8.18.8 + - name: LogQL Service 9.3.0 config: test/e2e-v2/cases/logql/e2e.yaml + env: ES_VERSION=9.3.0 - name: TraceQL Service Zipkin config: test/e2e-v2/cases/traceql/zipkin/e2e.yaml - name: TraceQL Service SkyWalking @@ -799,8 +875,15 @@ jobs: config: test/e2e-v2/cases/otlp-traces/e2e.yaml - name: OTLP Trace Native BanyanDB config: test/e2e-v2/cases/otlp-traces/native/banyandb/e2e.yaml - - name: OTLP Trace Native ES + - name: OTLP Trace Native ES 7.15.0 + config: test/e2e-v2/cases/otlp-traces/native/es/e2e.yaml + env: ES_VERSION=7.15.0 + - name: OTLP Trace Native ES 8.18.8 + config: test/e2e-v2/cases/otlp-traces/native/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: OTLP Trace Native ES 9.3.0 config: test/e2e-v2/cases/otlp-traces/native/es/e2e.yaml + env: ES_VERSION=9.3.0 - name: OTLP Trace Native MySQL config: test/e2e-v2/cases/otlp-traces/native/mysql/e2e.yaml @@ -810,15 +893,23 @@ jobs: - name: Cilium Service config: test/e2e-v2/cases/cilium/e2e.yaml - - name: Async Profiler ES + - name: Async Profiler ES 8.18.8 + config: test/e2e-v2/cases/profiling/async-profiler/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Async Profiler ES 9.3.0 config: test/e2e-v2/cases/profiling/async-profiler/es/e2e.yaml + env: ES_VERSION=9.3.0 - name: Async Profiler BanyanDB config: test/e2e-v2/cases/profiling/async-profiler/banyandb/e2e.yaml - name: Async Profiler MySQL config: test/e2e-v2/cases/profiling/async-profiler/mysql/e2e.yaml - - name: Pprof ES + - name: Pprof ES 8.18.8 + config: test/e2e-v2/cases/profiling/pprof/es/e2e.yaml + env: ES_VERSION=8.18.8 + - name: Pprof ES 9.3.0 config: test/e2e-v2/cases/profiling/pprof/es/e2e.yaml + env: ES_VERSION=9.3.0 - name: Pprof BanyanDB config: test/e2e-v2/cases/profiling/pprof/banyandb/e2e.yaml - name: Pprof MySQL diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index 491f684e1118..c056e865dbe2 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -7,6 +7,10 @@ * Fix the flaky Elasticsearch client IT (`ElasticSearchIT`), whose failures stayed hidden until a failed IT failed the build (above). OpenSearch 1.0.0 moves to 1.3.20 and Elasticsearch 8.1.0 to 8.19.25, the latest of each line. On 8 of the 19 nightly runs since 2026-10-20 an OpenSearch 1.0.0 container never started, and the test waited out its five-minute startup timeout; every failed container's only output is its start script stopping OpenSearch. The same output was reported upstream on 1.0.1, and its reporter saw it go away with the 1.1 image (opensearch-project/opensearch-build#504). Elasticsearch 8.1.0's bundled JDK 17.0.2 crashed at startup on the runner's cgroup v2 setup (`NullPointerException` in `CgroupV2Subsystem`) on 11 of the 13 nightly runs from 2026-10-07 to 2026-10-27, failing all eight of its tests each time. The OpenSearch 1.3.20 and 2.8.0 containers now run without the Performance Analyzer agent: the image starts its JVM beside the launcher's, a JVM that starts beside another can print an hsperfdata lock warning, and the launcher takes that warning into its JVM options, so OpenSearch fails with `Could not find or load main class Cannot` (opensearch-project/opensearch-build#3821). The tested OpenSearch 1.x version in the storage docs is now 1.3.20. * Test Zipkin and the span-attached events of eBPF profiling on Elasticsearch and OpenSearch, and the DAOs of the BanyanDB storage. The Zipkin e2e case on OpenSearch runs in CI again, now on OpenSearch 3.0.0: it pinned OpenSearch 1.2.0, so it lost its CI job when #13715 dropped the OpenSearch 1.x e2e tests, and had not run in CI since. New slow ITs write through OAP's own write path and read back through the Elasticsearch storage's DAOs and the trace and Zipkin query services, on Elasticsearch 8.18.8 and 9.3.0 and OpenSearch 2.8.0 and 3.0.0 with OAP's default storage settings, and on Elasticsearch 9.3.0 and OpenSearch 3.0.0 with logic sharding and custom routing. For Zipkin they check every search filter against a near miss, spans on and just outside each time bound, traces spread over two daily indices, the traces a limit keeps, reads by trace ID, and the service, remote service and span name lists; for span-attached events, every event of the traces asked for across scroll pages, in start-time order, and each event on its own span in the trace and Zipkin query results. The BanyanDB storage ITs now cover every DAO the plugin registers except the history delete, a no-op because BanyanDB applies its own TTL, with new ITs for the span-attached event query, traces read with their events (through the DAOs and the trace and Zipkin query services), the batch DAO that routes each write to its bulk, and dropping a measure, an index-mode measure, a stream, a trace and a property, and a unit test for the TTL status query. * Move the BanyanDB that the e2e and integration tests run on to `8efdf0ca`, which carries apache/skywalking-banyandb#1479 and apache/skywalking-banyandb#1481, and run the three BanyanDB ITs that waited for them again. Before it, an index-mode measure ordered by time came back in index order, so `findEndpoint` with a limit did not return the newest endpoints (#14218), and a pre-built TopN query applied `AGGREGATE BY` after keeping one value per entity, so the `MEAN` OAP asks for returned each entity's best value in the range instead of its mean (#14217). +* Test every DAO of the JDBC and Elasticsearch storages, as the BanyanDB DAOs are: each IT installs the schema the way OAP does at boot, writes through OAP's write path, reads through the DAO the storage provider registers, and compares whole results, with near misses, rows on and just outside the time bounds, paging and order. The JDBC ITs run on PostgreSQL 13 and MariaDB 11.4, with rows on both sides of midnight so a read crosses two day tables. The Elasticsearch ITs run on Elasticsearch 8.18.8 and 9.3.0 and OpenSearch 1.3.20, 2.8.0 and 3.0.0, and on Elasticsearch 9.3.0 and OpenSearch 3.0.0 with logic sharding and custom routing. They are nested ITs of one IT per server, so each server starts once, and every model they install must be found installed again, as on OAP's next boot. +* Test every request the Elasticsearch client sends for OAP on a server of each request format it picks by version: Elasticsearch 6.8.23, 7.7.1 (legacy index templates), 7.8.0 and 7.17.29 (composable index templates), 8.19.25 and 9.3.0, and OpenSearch 1.3.20, 2.8.0 and 3.0.0. Elasticsearch 7.4.2, 7.15.0 and 8.9.0 and OpenSearch 2.4.0 leave the client IT, whose formats the versions above cover. Each server starts once for all its tests, so the job takes about 3 minutes instead of 20. +* Run every e2e case that stores in Elasticsearch on Elasticsearch 8.18.8 and 9.3.0; only the storage case ran on 9.3.0 before. The OTLP native trace case also keeps Elasticsearch 7.15.0. Outside the CI matrix, these cases default to 8.18.8. +* Fix the mini-program e2e log check failing about 1 run in 4: a log whose content held a `'` broke the YAML of the expected log. #### OAP Server * Fix the Cilium fetcher dropping or double-reading Cilium nodes. The Peer notify stream, the cluster watcher and the 10-second refresh timer rebuilt the shared node list without a lock, which could open two flow streams for one node or keep a closed node as in use. A node closed when this OAP's share of nodes moved away was reused, still closed, when the share moved back (e.g. after scaling the OAP cluster out and back in), so that node went unread while the assignment lasted. A `Receiver`-role OAP on a ZooKeeper, Consul, Nacos or etcd coordinator never finds itself in the cluster list and, when Cilium nodes and other OAPs were listed, threw on every refresh; it now skips Cilium fetching quietly, as the `Mixed` and `Aggregator` OAPs share all Cilium nodes among themselves. @@ -68,6 +72,22 @@ * Fix creating an async-profiler task for a service while another one runs on it: the check that refuses it never found the running task on the JDBC and Elasticsearch storage. * Fix the async-profiler task OAP sends to an agent whose service has two tasks created within five minutes of each other: it could send the older one. * Fix alarm windows retaining empty layers after an initial metadata lookup miss. Later metrics can supply the missing layers for subsequent alarms, allowing layer-filtered `queryAlarms` queries to match the persisted layer. +* Elasticsearch: `indexTemplateOrder` (`SW_STORAGE_ES_INDEX_TEMPLATE_ORDER`) now applies on every version. Only Elasticsearch 6 received it; a legacy index template now carries it as its `order`, and a composable one, used from Elasticsearch 7.8 and on OpenSearch, as its `priority`. A template stored with another order is updated at boot, so an existing deployment that configured it gets it applied once. A legacy template keeps a negative order; a composable template's priority cannot be negative, so it takes 0. +* Elasticsearch: fix document ids holding `/`, `?`, `#` or `%`, which went into the request path unescaped. Such a document could not be written, such as a runtime rule named like `aws-gateway/gateway-service` or the continuous-profiling policy of a service whose Base64 service id holds `/`, and the server decoded a percent escape in an id, storing `a%20b` as `a b`, which a read by id never found. The id is now escaped in the request path. A document an earlier version stored under a decoded id is not found by this version. A `+` in an id still reaches the server unescaped, which Elasticsearch before 7.4 reads as a space. +* Elasticsearch: fix the topology keeping one component of a service or process relation whose component changed within the time range; every component is returned, as on JDBC and BanyanDB. The process topology also shows every component the client side of a call reported, on every storage. +* Elasticsearch: fix the async-profiler (JFR) and pprof flame graphs being built from at most 10 uploads of a task; every upload is read, one per request, so large uploads never exceed the client's response limit. A pprof read with no instance listed reads every instance, as on JDBC and BanyanDB. +* Elasticsearch: fix process details (`getProcess`) never being found in the default storage mode. Fix tag autocomplete returning fewer values than its limit when a value was seen on several days, returning tags of other days when `dayStep` is above 1, reading every page before it stops, and failing with a limit of 0. +* Elasticsearch: fix an eBPF profiling task query with no target types returning every task of the service. Deleting a runtime rule, a UI template or other management data that does not exist now returns quietly, and a delete the server refuses is reported instead of passing for a deletion. +* Elasticsearch: fix the event query's endpoint filter matching nothing, and events on the duration's first and last instant being left out. +* Elasticsearch: fix the alarms of a page after the 10th losing their recovery time, an instance's hierarchy stopping at 10 relations, and a relation whose component changed more than 10 times within the range losing components. +* Elasticsearch and JDBC: fix listing instances and endpoints by a time range returning rows first seen in the minute after the range, and listing the processes of an instance at the hour or day step returning nothing. +* Elasticsearch and JDBC: a query's end is inclusive. Logs, browser error logs, alarms, GenAI evaluations, AI agent conversation rounds, the trace list and sampled records now read a row stored exactly on the duration's end, and a duration ending in a day's last minute also reads the next day's table or index, where that end lies. Tag autocomplete, stored per day, keeps reading the days of the requested steps only. +* Fix a trace profiling task being refused as overlapping a task of the service that starts later in the minute the new task ends. +* JDBC: fix the Zipkin trace search and the OTLP trace query (latest first) returning the wrong traces when a trace has spans on both sides of midnight: each day table ranked it by its spans in that table only. +* JDBC: fix a metric on the last day the TTL drops being treated as alive by the metrics cache, so late values were accepted into a dropped table. +* Elasticsearch: write a row into the daily index of its time bucket's day, whatever the bucket's precision. Async-profiler, pprof and continuous-profiling tasks, which carry a second-precision bucket, went into an index per hour that the TTL never deleted, and could not be written at all with `dayStep` above 1; the TTL now deletes the hour-named indices (`*-yyyyMMddHH`) earlier versions left, with their day. The copy of an eBPF network profiling task that keeps it alive carried time bucket 0, so it went into an index dated 0 that the next TTL run deleted; it now carries the minute it is written. BanyanDB: a task list now includes the tasks of the range's last time bucket. +* Elasticsearch: the TTL deletion skips a table it already cleaned to the same deadline, instead of checking every table again for each of its models on every run. +* JDBC: fix the network address aliases updated in the minute asked for being left out, eBPF profiling schedules not coming back newest first, and an event on the query duration's end instant being left out. #### UI diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/asyncprofiler/AsyncProfilerMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/asyncprofiler/AsyncProfilerMutationService.java index 700022e526b3..a1c1e7777c2e 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/asyncprofiler/AsyncProfilerMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/asyncprofiler/AsyncProfilerMutationService.java @@ -77,6 +77,8 @@ public AsyncProfilerTaskCreationResult createTask(String serviceId, task.setEventsFromList(rowEvents); task.setCreateTime(createTime); task.setExecArgs(execArgs); + // A second bucket on a minute-downsampled model: the task lists bound tasks by the seconds of a duration, and + // Elasticsearch names the row's daily index by the bucket's own precision. task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); NoneStreamProcessor.getInstance().in(task); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/EBPFProfilingMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/EBPFProfilingMutationService.java index a329e7f37f6b..f82e7238c2e9 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/EBPFProfilingMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/ebpf/EBPFProfilingMutationService.java @@ -203,6 +203,7 @@ public EBPFNetworkKeepProfilingResult keepEBPFNetworkProfiling(String taskId) th record.setTargetType(EBPFProfilingTargetType.NETWORK.value()); record.setCreateTime(now.getTimeInMillis()); record.setLastUpdateTime(now.getTimeInMillis()); + record.setTimeBucket(TimeBucket.getMinuteTimeBucket(now.getTimeInMillis())); record.setExtensionConfigJson(Const.EMPTY_STRING); NoneStreamProcessor.getInstance().in(record); return buildKeepProfilingSuccess(); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java index a220ba29304e..8391338c819e 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationService.java @@ -76,6 +76,8 @@ public PprofTaskCreationResult createTask(String serviceId, task.setEvents(events.toString()); task.setDumpPeriod(dumpPeriod); task.setCreateTime(createTime); + // A second bucket on a minute-downsampled model: the task lists bound tasks by the seconds of a duration, and + // Elasticsearch names the row's daily index by the bucket's own precision. task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); NoneStreamProcessor.getInstance().in(task); return PprofTaskCreationResult.builder() diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskMutationService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskMutationService.java index 1e095a5cb4ad..10a517a66383 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskMutationService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskMutationService.java @@ -138,14 +138,18 @@ private String checkDataSuccess(final String serviceId, return "max sampling count must less than " + ProfileConstants.TASK_MAX_SAMPLING_COUNT; } - // Each service can monitor up to 1 endpoints during the execution of tasks + // Each service can monitor up to 1 endpoints during the execution of tasks. + // The list reads whole minutes, so it also holds a task starting in this task's last minute after this task + // ends. A service's tasks never overlap and last a minute or more, so at most one starts in a minute, and the + // latest two hold the latest task starting before this one ends, the only one that can overlap it. long endTimeBucket = TimeBucket.getMinuteTimeBucket(monitorEndTime); final List alreadyHaveTaskList = getProfileTaskDAO().getTaskList( - serviceId, null, null, endTimeBucket, 1); + serviceId, null, null, endTimeBucket, 2); if (CollectionUtils.isNotEmpty(alreadyHaveTaskList)) { for (ProfileTask profileTask : alreadyHaveTaskList) { - if (profileTask.getStartTime() + TimeUnit.MINUTES.toMillis(profileTask.getDuration()) >= monitorStartTime) { - // if the endTime is greater or equal than the startTime of the newly created task, i.e. there is overlap between two tasks, it is an invalid case + final long existingEndTime = + profileTask.getStartTime() + TimeUnit.MINUTES.toMillis(profileTask.getDuration()); + if (profileTask.getStartTime() <= monitorEndTime && existingEndTime >= monitorStartTime) { return "current service already has monitor task execute at this time"; } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProcessTopologyBuilder.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProcessTopologyBuilder.java index 8cdf6e0cdc1f..55cf5815ad94 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProcessTopologyBuilder.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/ProcessTopologyBuilder.java @@ -114,18 +114,20 @@ ProcessTopology build(List clientCalls, int appendCallCount = 0; for (Call.CallDetail clientCall : clientCalls) { - if (!callMap.containsKey(clientCall.getId())) { - Call call = new Call(); + Call call = callMap.get(clientCall.getId()); + if (call == null) { + call = new Call(); callMap.put(clientCall.getId(), call); call.setSource(clientCall.getSource()); call.setTarget(clientCall.getTarget()); call.setId(clientCall.getId()); call.addDetectPoint(DetectPoint.CLIENT); - call.addSourceComponent(componentLibraryCatalogService.getComponentName(clientCall.getComponentId())); calls.add(call); appendCallCount++; } + // The storage returns one call per component the relation named in the range. + call.addSourceComponent(componentLibraryCatalogService.getComponentName(clientCall.getComponentId())); } // adding server side call diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/asyncprofiler/AsyncProfilerMutationServiceTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/asyncprofiler/AsyncProfilerMutationServiceTest.java index 3de35d213781..448280762a35 100644 --- a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/asyncprofiler/AsyncProfilerMutationServiceTest.java +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/asyncprofiler/AsyncProfilerMutationServiceTest.java @@ -21,6 +21,9 @@ import java.util.Collections; import java.util.List; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskRecord; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerEventType; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTaskCreationResult; @@ -31,12 +34,16 @@ import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; class AsyncProfilerMutationServiceTest { @@ -63,16 +70,43 @@ void refusesASecondTaskForTheServiceWhileTheFirstRuns() throws Exception { return storedTimeBucket <= endTimeBucket ? Collections.singletonList(running) : Collections.emptyList(); }); + + final AsyncProfilerTaskCreationResult result = new AsyncProfilerMutationService(moduleManager(dao)).createTask( + SERVICE_ID, List.of("instance-1"), 60, List.of(AsyncProfilerEventType.CPU), ""); + + assertEquals(AsyncProfilerTaskCreationType.ALREADY_PROFILING_ERROR, result.getCode()); + } + + /** + * The row carries the second bucket of its creation, which the task lists, the creation check and the agent task + * cache bound by second buckets. + */ + @Test + void writesTheTaskWithTheSecondBucketOfItsCreation() throws Exception { + final IAsyncProfilerTaskQueryDAO dao = mock(IAsyncProfilerTaskQueryDAO.class); + when(dao.getTaskList(any(), any(), any(), any())).thenReturn(Collections.emptyList()); + final NoneStreamProcessor processor = mock(NoneStreamProcessor.class); + try (MockedStatic ignored = mockStatic(NoneStreamProcessor.class)) { + ignored.when(NoneStreamProcessor::getInstance).thenReturn(processor); + + final AsyncProfilerTaskCreationResult result = new AsyncProfilerMutationService(moduleManager(dao)) + .createTask(SERVICE_ID, List.of("instance-1"), 60, List.of(AsyncProfilerEventType.CPU), ""); + + assertEquals(AsyncProfilerTaskCreationType.SUCCESS, result.getCode()); + } + final ArgumentCaptor written = ArgumentCaptor.forClass(NoneStream.class); + verify(processor).in(written.capture()); + final AsyncProfilerTaskRecord task = (AsyncProfilerTaskRecord) written.getValue(); + assertEquals(TimeBucket.getRecordTimeBucket(task.getCreateTime()), task.getTimeBucket()); + } + + private static ModuleManager moduleManager(final IAsyncProfilerTaskQueryDAO dao) { final ModuleManager moduleManager = mock(ModuleManager.class); final ModuleProviderHolder providerHolder = mock(ModuleProviderHolder.class); final ModuleServiceHolder serviceHolder = mock(ModuleServiceHolder.class); when(moduleManager.find(StorageModule.NAME)).thenReturn(providerHolder); when(providerHolder.provider()).thenReturn(serviceHolder); when(serviceHolder.getService(IAsyncProfilerTaskQueryDAO.class)).thenReturn(dao); - - final AsyncProfilerTaskCreationResult result = new AsyncProfilerMutationService(moduleManager).createTask( - SERVICE_ID, List.of("instance-1"), 60, List.of(AsyncProfilerEventType.CPU), ""); - - assertEquals(AsyncProfilerTaskCreationType.ALREADY_PROFILING_ERROR, result.getCode()); + return moduleManager; } } diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/ebpf/EBPFProfilingMutationServiceTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/ebpf/EBPFProfilingMutationServiceTest.java new file mode 100644 index 000000000000..044a0db0453f --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/ebpf/EBPFProfilingMutationServiceTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.ebpf; + +import java.util.Collections; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTargetType; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTriggerType; +import org.apache.skywalking.oap.server.core.query.type.EBPFNetworkKeepProfilingResult; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingTaskDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class EBPFProfilingMutationServiceTest { + private static final String INSTANCE_ID = "Y2hlY2tvdXQ=.1_Y2hlY2tvdXQtMQ=="; + + /** + * The copy that extends a network task has the minute it is created in as its time bucket, as every task has: + * the task model has minute downsampling, and Elasticsearch writes a task into the daily index of that minute, + * which the TTL deletes with its day. + */ + @Test + void keepsANetworkTaskAliveWithACopyOfTheMinuteItIsCreatedIn() throws Exception { + // Created to end in thirty seconds, within the minute before its end in which a keep-alive extends it. + final long createTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(600 - 30); + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId("Y2hlY2tvdXQ=.1"); + task.setProcessLabelsJson(Const.EMPTY_STRING); + task.setInstanceId(INSTANCE_ID); + task.setStartTime(createTime); + task.setTriggerType(EBPFProfilingTriggerType.FIXED_TIME.value()); + task.setFixedTriggerDuration(600); + task.setTargetType(EBPFProfilingTargetType.NETWORK.value()); + task.setCreateTime(createTime); + task.setLastUpdateTime(createTime); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(createTime)); + task.generateLogicalId(); + final IEBPFProfilingTaskDAO dao = mock(IEBPFProfilingTaskDAO.class); + when(dao.getTaskRecord(task.getLogicalId())).thenReturn(Collections.singletonList(task)); + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleProviderHolder providerHolder = mock(ModuleProviderHolder.class); + final ModuleServiceHolder serviceHolder = mock(ModuleServiceHolder.class); + when(moduleManager.find(StorageModule.NAME)).thenReturn(providerHolder); + when(providerHolder.provider()).thenReturn(serviceHolder); + when(serviceHolder.getService(IEBPFProfilingTaskDAO.class)).thenReturn(dao); + + final NoneStreamProcessor tasks = mock(NoneStreamProcessor.class); + final EBPFNetworkKeepProfilingResult result; + final long before; + final long after; + try (MockedStatic processor = mockStatic(NoneStreamProcessor.class)) { + processor.when(NoneStreamProcessor::getInstance).thenReturn(tasks); + before = System.currentTimeMillis(); + result = new EBPFProfilingMutationService(moduleManager).keepEBPFNetworkProfiling(task.getLogicalId()); + after = System.currentTimeMillis(); + } + + assertTrue(result.isStatus(), result.getErrorReason()); + final ArgumentCaptor written = ArgumentCaptor.forClass(NoneStream.class); + verify(tasks).in(written.capture()); + final EBPFProfilingTaskRecord copy = (EBPFProfilingTaskRecord) written.getValue(); + assertTrue(copy.getCreateTime() >= before && copy.getCreateTime() <= after, + "created at " + copy.getCreateTime()); + assertEquals(task.getLogicalId(), copy.getLogicalId()); + assertEquals(1200, copy.getFixedTriggerDuration()); + assertEquals(TimeBucket.getMinuteTimeBucket(copy.getCreateTime()), copy.getTimeBucket()); + } +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationServiceTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationServiceTest.java new file mode 100644 index 000000000000..3c3e0f6ea05e --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/pprof/PprofMutationServiceTest.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.pprof; + +import java.util.Collections; +import java.util.List; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationResult; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationType; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class PprofMutationServiceTest { + private static final String SERVICE_ID = "c2VydmljZQ==.1"; + + /** + * A CPU task created ten seconds ago runs for five minutes. Its row carries the second bucket of its creation, as + * {@code createTask} stamps it, and the DAO keeps the rows whose time bucket is not past the bound, as the JDBC + * and Elasticsearch DAOs filter. + */ + @Test + void refusesASecondTaskOfTheEventForTheServiceWhileTheFirstRuns() throws Exception { + final long createTime = System.currentTimeMillis() - 10_000L; + final PprofTask running = PprofTask.builder() + .id(createTime + "_" + SERVICE_ID) + .serviceId(SERVICE_ID) + .events(PprofEventType.CPU) + .createTime(createTime) + .duration(5) + .build(); + final long storedTimeBucket = TimeBucket.getRecordTimeBucket(createTime); + final IPprofTaskQueryDAO dao = mock(IPprofTaskQueryDAO.class); + when(dao.getTaskList(eq(SERVICE_ID), any(), anyLong(), any())).thenAnswer(invocation -> { + final long endTimeBucket = invocation.getArgument(2); + return storedTimeBucket <= endTimeBucket + ? Collections.singletonList(running) : Collections.emptyList(); + }); + + final PprofTaskCreationResult result = new PprofMutationService(moduleManager(dao)).createTask( + SERVICE_ID, List.of("instance-1"), 5, PprofEventType.CPU, 0); + + assertEquals(PprofTaskCreationType.ALREADY_PROFILING_ERROR, result.getCode()); + } + + /** + * The row carries the second bucket of its creation, which the task lists, the creation check and the agent task + * cache bound by second buckets. + */ + @Test + void writesTheTaskWithTheSecondBucketOfItsCreation() throws Exception { + final IPprofTaskQueryDAO dao = mock(IPprofTaskQueryDAO.class); + when(dao.getTaskList(any(), any(), any(), any())).thenReturn(Collections.emptyList()); + final NoneStreamProcessor processor = mock(NoneStreamProcessor.class); + try (MockedStatic ignored = mockStatic(NoneStreamProcessor.class)) { + ignored.when(NoneStreamProcessor::getInstance).thenReturn(processor); + + final PprofTaskCreationResult result = new PprofMutationService(moduleManager(dao)).createTask( + SERVICE_ID, List.of("instance-1"), 5, PprofEventType.CPU, 0); + + assertEquals(PprofTaskCreationType.SUCCESS, result.getCode()); + } + final ArgumentCaptor written = ArgumentCaptor.forClass(NoneStream.class); + verify(processor).in(written.capture()); + final PprofTaskRecord task = (PprofTaskRecord) written.getValue(); + assertEquals(TimeBucket.getRecordTimeBucket(task.getCreateTime()), task.getTimeBucket()); + } + + private static ModuleManager moduleManager(final IPprofTaskQueryDAO dao) { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleProviderHolder providerHolder = mock(ModuleProviderHolder.class); + final ModuleServiceHolder serviceHolder = mock(ModuleServiceHolder.class); + when(moduleManager.find(StorageModule.NAME)).thenReturn(providerHolder); + when(providerHolder.provider()).thenReturn(serviceHolder); + when(serviceHolder.getService(IPprofTaskQueryDAO.class)).thenReturn(dao); + return moduleManager; + } +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskMutationServiceTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskMutationServiceTest.java new file mode 100644 index 000000000000..2ff203b2e390 --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/profiling/trace/ProfileTaskMutationServiceTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.profiling.trace; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Comparator; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.query.type.ProfileTask; +import org.apache.skywalking.oap.server.core.query.type.ProfileTaskCreationResult; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.trace.IProfileTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +class ProfileTaskMutationServiceTest { + private static final String SERVICE_ID = "c2VydmljZQ==.1"; + private static final String CONFLICT = "current service already has monitor task execute at this time"; + + /** + * A new task from 12:00:10 to 12:05:10. The task list reads whole minutes, so it also holds a task starting at + * 12:05:50, in the new task's last minute but after it ends: that task does not overlap it. + */ + @Test + void acceptsATaskEndingBeforeOneThatStartsLaterInItsLastMinute() throws Exception { + final IProfileTaskQueryDAO dao = taskList(task("11:50:00", 5), task("12:05:50", 5)); + + assertNull(create(dao, "12:00:10", 5).getErrorReason()); + } + + /** + * The task overlapping the new one started before the one listed latest, which starts after the new one ends. + */ + @Test + void refusesATaskOverlappingOneThatStartedBeforeTheLatestListed() throws Exception { + final IProfileTaskQueryDAO dao = taskList(task("11:58:00", 3), task("12:05:50", 5)); + + assertEquals(CONFLICT, create(dao, "12:00:10", 5).getErrorReason()); + } + + @Test + void refusesATaskStartingBeforeTheLatestOneEnds() throws Exception { + final IProfileTaskQueryDAO dao = taskList(task("11:58:00", 3)); + + assertEquals(CONFLICT, create(dao, "12:01:00", 5).getErrorReason()); + assertNull(create(dao, "12:01:01", 5).getErrorReason()); + } + + private static ProfileTaskCreationResult create(final IProfileTaskQueryDAO dao, + final String start, + final int minutes) throws Exception { + try (MockedStatic processor = mockStatic(NoneStreamProcessor.class)) { + processor.when(NoneStreamProcessor::getInstance).thenReturn(mock(NoneStreamProcessor.class)); + return new ProfileTaskMutationService(moduleManager(dao)) + .createTask(SERVICE_ID, "/checkout", millis(start), minutes, 0, 10, 5); + } + } + + /** + * The tasks of the service whose minute bucket is not past the end bucket, latest first, up to the limit, as the + * storages list them. + */ + private static IProfileTaskQueryDAO taskList(final ProfileTask... tasks) throws Exception { + final IProfileTaskQueryDAO dao = mock(IProfileTaskQueryDAO.class); + when(dao.getTaskList(eq(SERVICE_ID), any(), any(), any(), any())).thenAnswer(invocation -> { + final Long endTimeBucket = invocation.getArgument(3); + final Integer limit = invocation.getArgument(4); + return Arrays.stream(tasks) + .filter(task -> TimeBucket.getMinuteTimeBucket(task.getStartTime()) <= endTimeBucket) + .sorted(Comparator.comparingLong(ProfileTask::getStartTime).reversed()) + .limit(limit) + .collect(Collectors.toList()); + }); + return dao; + } + + private static ProfileTask task(final String start, final int minutes) { + return ProfileTask.builder().serviceId(SERVICE_ID).startTime(millis(start)).duration(minutes).build(); + } + + private static long millis(final String time) { + return Instant.parse("2026-11-18T" + time + "Z").toEpochMilli(); + } + + private static ModuleManager moduleManager(final IProfileTaskQueryDAO dao) { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleProviderHolder providerHolder = mock(ModuleProviderHolder.class); + final ModuleServiceHolder serviceHolder = mock(ModuleServiceHolder.class); + when(moduleManager.find(StorageModule.NAME)).thenReturn(providerHolder); + when(providerHolder.provider()).thenReturn(serviceHolder); + when(serviceHolder.getService(IProfileTaskQueryDAO.class)).thenReturn(dao); + return moduleManager; + } +} diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java index 614aed441de7..7fc30464d880 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchClient.java @@ -397,11 +397,14 @@ public void forceUpdate(String indexName, String id, Map source) } } - public void deleteById(String indexName, String id) { + /** + * @return {@code false} when there is no such document to delete + */ + public boolean deleteById(String indexName, String id) { try (HistogramMetrics.Timer timer = singleDeleteHistogram.createTimer()) { indexName = indexNameConverter.apply(indexName); Map params = ImmutableMap.of("refresh", "true"); - es.get().documents().deleteById(indexName, TYPE, id, params); + return es.get().documents().deleteById(indexName, TYPE, id, params); } } diff --git a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchScroller.java b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchScroller.java index 7cefe125bb2b..d6913afff8df 100644 --- a/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchScroller.java +++ b/oap-server/server-library/library-client/src/main/java/org/apache/skywalking/oap/server/library/client/elasticsearch/ElasticSearchScroller.java @@ -26,7 +26,9 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.function.Function; @@ -45,12 +47,18 @@ public class ElasticSearchScroller { final String[] indices; @Builder.Default final int queryMaxSize = 0; + /** + * When set, a result equal to one already read is left out, so {@link #queryMaxSize} counts distinct results and + * the scroll stops as soon as that many are read. + */ + @Builder.Default + final boolean distinct = false; @Builder.Default final SearchParams params = new SearchParams(); final Function resultConverter; public List scroll() { - final var results = new ArrayList(); + final Collection results = distinct ? new LinkedHashSet<>() : new ArrayList<>(); final var scrollIds = new HashSet(); params.scroll(SCROLL_CONTEXT_RETENTION); @@ -69,11 +77,11 @@ public List scroll() { for (final var searchHit : response.getHits()) { results.add(resultConverter.apply(searchHit)); if (queryMaxSize > 0 && results.size() >= queryMaxSize) { - return results; + return new ArrayList<>(results); } } if (search.getSize() != null && response.getHits().getHits().size() < search.getSize()) { - return results; + return new ArrayList<>(results); } response = client.scroll(SCROLL_CONTEXT_RETENTION, scrollId); } @@ -81,6 +89,6 @@ public List scroll() { scrollIds.forEach(client::deleteScrollContextQuietly); } - return results; + return new ArrayList<>(results); } } diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessor.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessor.java index b1442e3f96b3..02cb55cee23d 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessor.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessor.java @@ -63,6 +63,10 @@ public final class BulkProcessor { private volatile long lastFlushTS = 0; private final int batchOfBytes; private final HistogramMetrics bulkMetrics; + /** + * Runs the periodical flush. A field, not a local, so the ITs can stop it by reflection; OAP never stops it. + */ + private final ScheduledThreadPoolExecutor scheduler; public static BulkProcessorBuilder builder() { return new BulkProcessorBuilder(); @@ -83,7 +87,7 @@ public static BulkProcessorBuilder builder() { this.requests = new ArrayBlockingQueue<>(bulkActions + 1); this.bulkMetrics = bulkMetrics; - final ScheduledThreadPoolExecutor scheduler = new ScheduledThreadPoolExecutor(1, r -> { + this.scheduler = new ScheduledThreadPoolExecutor(1, r -> { final Thread thread = new Thread(r); thread.setName("ElasticSearch BulkProcessor"); return thread; diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/client/DocumentClient.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/client/DocumentClient.java index 33d6acec3370..77bb5239c767 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/client/DocumentClient.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/client/DocumentClient.java @@ -26,10 +26,12 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.library.elasticsearch.ElasticSearchVersion; +import org.apache.skywalking.library.elasticsearch.exception.ResponseException; import org.apache.skywalking.library.elasticsearch.requests.IndexRequest; import org.apache.skywalking.library.elasticsearch.requests.UpdateRequest; import org.apache.skywalking.library.elasticsearch.response.Document; @@ -159,14 +161,22 @@ public void update(UpdateRequest request, Map params) { } @SneakyThrows - public void deleteById(String index, String type, String id, Map params) { - final CompletableFuture future = version.thenCompose( + /** + * @return {@code false} when there is no such document, or no such index, to delete from + * @throws ResponseException when the server refuses the delete for any other reason + */ + public boolean deleteById(String index, String type, String id, Map params) { + final CompletableFuture future = version.thenCompose( v -> client.execute(v.requestFactory().document().deleteById(index, type, id, params)) - .aggregate().thenAccept(response -> { + .aggregate().thenApply(response -> { final HttpStatus status = response.status(); + if (status == HttpStatus.NOT_FOUND) { + return false; + } if (status != HttpStatus.OK) { - throw new RuntimeException(response.contentUtf8()); + throw new ResponseException(response.contentUtf8(), status.code()); } + return true; })); future.whenComplete((result, exception) -> { if (exception != null) { @@ -177,6 +187,13 @@ public void deleteById(String index, String type, String id, Map log.debug("Succeeded delete doc by id {} in index {}, params: {}", id, params, index); } }); - future.join(); + try { + return future.join(); + } catch (CompletionException e) { + if (e.getCause() instanceof ResponseException) { + throw (ResponseException) e.getCause(); + } + throw e; + } } } diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/PathSegments.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/PathSegments.java new file mode 100644 index 000000000000..0ab939db6665 --- /dev/null +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/PathSegments.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.library.elasticsearch.requests.factory; + +import com.google.common.escape.Escaper; +import com.google.common.net.UrlEscapers; + +/** + * Escapes a value the client puts into a request path as one segment, such as a document id. The HTTP client does + * not escape path params, so an id holding {@code /}, {@code ?}, {@code #} or {@code %} would change the path: a + * runtime rule is named like {@code aws-gateway/gateway-service}, and a service id, which some documents are keyed + * by, is Base64 and can hold {@code /}. + * + *

A {@code +} reaches the server as it is, escaped or not, as the HTTP client decodes {@code %2B} in a path. + * Elasticsearch 7.4 and later and OpenSearch read it as {@code +}; Elasticsearch 6 reads it as a space. + */ +public final class PathSegments { + private static final Escaper ESCAPER = UrlEscapers.urlPathSegmentEscaper(); + + private PathSegments() { + } + + public static String escape(final String segment) { + return ESCAPER.escape(segment); + } +} diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v6/V6DocumentFactory.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v6/V6DocumentFactory.java index 63986e1764c5..a0baec0f322d 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v6/V6DocumentFactory.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v6/V6DocumentFactory.java @@ -32,6 +32,7 @@ import org.apache.skywalking.library.elasticsearch.requests.IndexRequest; import org.apache.skywalking.library.elasticsearch.requests.UpdateRequest; import org.apache.skywalking.library.elasticsearch.requests.factory.DocumentFactory; +import org.apache.skywalking.library.elasticsearch.requests.factory.PathSegments; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import static com.google.common.base.Preconditions.checkArgument; @@ -54,7 +55,7 @@ public HttpRequest exist(String index, String type, String id) { .head("/{index}/{type}/{id}") .pathParam("index", index) .pathParam("type", type) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .build(); } @@ -68,7 +69,7 @@ public HttpRequest get(String index, String type, String id) { .get("/{index}/{type}/{id}") .pathParam("index", index) .pathParam("type", type) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .build(); } @@ -141,7 +142,7 @@ public HttpRequest index(IndexRequest request, Map params) { builder.put("/{index}/{type}/{id}") .pathParam("index", index) .pathParam("type", type) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .content(MediaType.JSON, content); return builder.build(); @@ -171,7 +172,7 @@ public HttpRequest update(UpdateRequest request, Map params) { builder.post("/{index}/{type}/{id}/_update") .pathParam("index", index) .pathParam("type", type) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .content(MediaType.JSON, content); return builder.build(); @@ -214,7 +215,7 @@ public HttpRequest deleteById(final String index, final String type, final Strin .delete("/{index}/{type}/{id}") .pathParam("index", index) .pathParam("type", type) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .build(); } } diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v6/codec/V6IndexTemplatesDeserializer.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v6/codec/V6IndexTemplatesDeserializer.java index ad4d91f18e6e..6b69ec9f851b 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v6/codec/V6IndexTemplatesDeserializer.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v6/codec/V6IndexTemplatesDeserializer.java @@ -40,6 +40,8 @@ public IndexTemplates deserialize(final JsonParser p, if (templates == null) { return new IndexTemplates(Collections.emptyMap()); } + // A legacy template is keyed by its name, which its body does not repeat. + templates.forEach((name, template) -> template.setName(name)); return new IndexTemplates(templates); } } diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V78TemplateFactory.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V78TemplateFactory.java index 885d30ea1f76..7b25648c25e5 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V78TemplateFactory.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V78TemplateFactory.java @@ -65,6 +65,9 @@ public HttpRequest createOrUpdate(String name, Map settings, final Map template = ImmutableMap.builder() .put("index_patterns", patterns) + // A composable template's precedence is its priority, the legacy template's order. A + // priority cannot be negative, while an order can. + .put("priority", Math.max(0, order)) .put( "template", ImmutableMap.builder() diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V7DocumentFactory.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V7DocumentFactory.java index cd8bf763e5e3..01a935ff8bbd 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V7DocumentFactory.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V7DocumentFactory.java @@ -31,6 +31,7 @@ import org.apache.skywalking.library.elasticsearch.requests.IndexRequest; import org.apache.skywalking.library.elasticsearch.requests.UpdateRequest; import org.apache.skywalking.library.elasticsearch.requests.factory.DocumentFactory; +import org.apache.skywalking.library.elasticsearch.requests.factory.PathSegments; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import static com.google.common.base.Preconditions.checkArgument; @@ -52,7 +53,7 @@ public HttpRequest exist(String index, String type, String id) { return HttpRequest.builder() .head("/{index}/_doc/{id}") .pathParam("index", index) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .build(); } @@ -65,7 +66,7 @@ public HttpRequest get(String index, String type, String id) { return HttpRequest.builder() .get("/{index}/_doc/{id}") .pathParam("index", index) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .build(); } @@ -136,7 +137,7 @@ public HttpRequest index(IndexRequest request, Map params) { builder.put("/{index}/_doc/{id}") .pathParam("index", index) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .content(MediaType.JSON, content); return builder.build(); @@ -165,7 +166,7 @@ public HttpRequest update(UpdateRequest request, Map params) { builder.post("/{index}/_doc/{id}/_update") .pathParam("index", index) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .content(MediaType.JSON, content); return builder.build(); @@ -206,7 +207,7 @@ public HttpRequest deleteById(final String index, final String type, final Strin return builder .delete("/{index}/_doc/{id}") .pathParam("index", index) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .build(); } } diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V7TemplateFactory.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V7TemplateFactory.java index 9c6ef920cdc9..2ea0c0deb346 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V7TemplateFactory.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V7TemplateFactory.java @@ -68,6 +68,7 @@ public HttpRequest createOrUpdate(String name, Map settings, .put("aliases", aliases) .put("settings", settings) .put("mappings", mappings) + .put("order", order) .build(); final byte[] content = version.codec().encode(template); diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V81DocumentFactory.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V81DocumentFactory.java index d7e29898c923..61821816589c 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V81DocumentFactory.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/V81DocumentFactory.java @@ -27,6 +27,7 @@ import org.apache.skywalking.library.elasticsearch.ElasticSearchVersion; import org.apache.skywalking.library.elasticsearch.requests.UpdateRequest; import org.apache.skywalking.library.elasticsearch.requests.factory.DocumentFactory; +import org.apache.skywalking.library.elasticsearch.requests.factory.PathSegments; import com.google.common.collect.ImmutableMap; import com.linecorp.armeria.common.HttpRequest; import com.linecorp.armeria.common.HttpRequestBuilder; @@ -68,7 +69,7 @@ public HttpRequest update(UpdateRequest request, Map params) { builder.post("/{index}/_update/{id}") .pathParam("index", index) - .pathParam("id", id) + .pathParam("id", PathSegments.escape(id)) .content(MediaType.JSON, content); return builder.build(); diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/codec/V78IndexTemplatesDeserializer.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/codec/V78IndexTemplatesDeserializer.java index 40f1d2b1be40..0fb09a84bc62 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/codec/V78IndexTemplatesDeserializer.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/codec/V78IndexTemplatesDeserializer.java @@ -78,6 +78,8 @@ public IndexTemplates deserialize(final JsonParser p, p.getCodec().treeToValue(indexTemplateNode, IndexTemplateWrapper.class); wrapper.getTemplate().setName(name); wrapper.getTemplate().setIndexPatterns(wrapper.getIndexPatterns()); + wrapper.getTemplate().setOrder(wrapper.getPriority()); + wrapper.getTemplate().setComposable(true); templates.add(wrapper.getTemplate()); } @@ -91,6 +93,7 @@ public IndexTemplates deserialize(final JsonParser p, static final class IndexTemplateWrapper { @JsonProperty("index_patterns") private List indexPatterns; + private long priority; private IndexTemplate template; } } diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/codec/V7IndexTemplatesDeserializer.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/codec/V7IndexTemplatesDeserializer.java index 59d04bd69795..6995e35f31aa 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/codec/V7IndexTemplatesDeserializer.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/requests/factory/v7plus/codec/V7IndexTemplatesDeserializer.java @@ -41,6 +41,8 @@ public IndexTemplates deserialize(final JsonParser p, if (templates == null) { return new IndexTemplates(Collections.emptyMap()); } + // A legacy template is keyed by its name, which its body does not repeat. + templates.forEach((name, template) -> template.setName(name)); return new IndexTemplates(templates); } } diff --git a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/response/IndexTemplate.java b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/response/IndexTemplate.java index 97411ffa48b5..3fb95d927111 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/response/IndexTemplate.java +++ b/oap-server/server-library/library-elasticsearch-client/src/main/java/org/apache/skywalking/library/elasticsearch/response/IndexTemplate.java @@ -17,6 +17,7 @@ package org.apache.skywalking.library.elasticsearch.response; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import java.util.List; import java.util.Map; @@ -25,7 +26,15 @@ @Data public final class IndexTemplate { private String name; - private int order; + /** + * A legacy template's order, or a composable template's priority, which cannot be negative. + */ + private long order; + /** + * Whether this is a composable template (Elasticsearch 7.8 and later, and OpenSearch) rather than a legacy one. + */ + @JsonIgnore + private boolean composable; @JsonProperty("index_patterns") private List indexPatterns; private Map settings; diff --git a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchClientITBase.java b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchClientITBase.java new file mode 100644 index 000000000000..269e53be446f --- /dev/null +++ b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchClientITBase.java @@ -0,0 +1,720 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.library.elasticsearch; + +import com.google.common.collect.ImmutableMap; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.apache.skywalking.library.elasticsearch.bulk.BulkProcessor; +import org.apache.skywalking.library.elasticsearch.bulk.BulkProcessorITExt; +import org.apache.skywalking.library.elasticsearch.requests.IndexRequest; +import org.apache.skywalking.library.elasticsearch.requests.UpdateRequest; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.QueryBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchParams; +import org.apache.skywalking.library.elasticsearch.requests.search.Sort; +import org.apache.skywalking.library.elasticsearch.requests.search.aggregation.Aggregation; +import org.apache.skywalking.library.elasticsearch.requests.search.aggregation.BucketOrder; +import org.apache.skywalking.library.elasticsearch.requests.search.aggregation.TermsAggregationBuilder; +import org.apache.skywalking.library.elasticsearch.response.Document; +import org.apache.skywalking.library.elasticsearch.response.Documents; +import org.apache.skywalking.library.elasticsearch.response.Index; +import org.apache.skywalking.library.elasticsearch.response.IndexTemplate; +import org.apache.skywalking.library.elasticsearch.response.Mappings; +import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Sends every request OAP's Elasticsearch storage sends, in the shape it sends it, to one server version, and checks + * what the server stored and what the client parsed back. The client picks its request format by the version it + * detects, so a nested class of {@link ElasticSearchIT} runs these on each supported version. + * + *

The cases follow OAP's callers: the storage installer (index templates with their alias and order, indices, + * appended mappings, and comparing them back), the management and single-document writes (refreshed at once), the + * persistence timer's bulk writes (inserts, updates, custom routing, a rejected item), and the query DAOs (every query + * and aggregation shape they build, paging and sorting, routed reads, searching day indices of which some do not + * exist, and scrolling with its context cleared). + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class ElasticSearchClientITBase { + /** The mapping and document type OAP's storage passes; only Elasticsearch 6 keeps it. */ + private static final String TYPE = "type"; + private static final String OPENSEARCH = "OpenSearch"; + + private final ElasticsearchContainer server; + private final String distribution; + private final String majorMinor; + private ElasticSearch client; + private ElasticSearchVersion version; + + /** + * @param distribution {@code ElasticSearch} or {@code OpenSearch} + * @param majorMinor the server's version, as {@code major.minor} + */ + protected ElasticSearchClientITBase(final ElasticsearchContainer server, final String distribution, + final String majorMinor) { + this.server = server.withEnv("cluster.routing.allocation.disk.threshold_enabled", "false"); + this.distribution = distribution; + this.majorMinor = majorMinor; + } + + @BeforeAll + public void start() throws Exception { + server.start(); + client = ElasticSearch.builder().endpoints(server.getHttpHostAddress()).build(); + version = client.connect().get(1, TimeUnit.MINUTES); + } + + @AfterAll + public void stop() { + if (client != null) { + client.close(); + } + server.stop(); + } + + @Test + public void detectsTheServerVersion() { + assertEquals((distribution + " " + majorMinor).toLowerCase(Locale.ROOT), + version.toString().toLowerCase(Locale.ROOT)); + } + + /** + * As the installer sets up a time-series model: a template for the model's day indices, read back for the same + * comparison the installer makes on the next boot, then a day index created bare that takes the template's + * mappings and alias, a column appended to that index, and the template updated with the column. + */ + @Test + public void keepsAnIndexTemplateThatDayIndicesFollow() { + final String name = "tpl-segment"; + final Map properties = oapProperties(); + assertFalse(client.templates().exists(name)); + assertEquals(Optional.empty(), client.templates().get(name)); + + assertTrue(client.templates().createOrUpdate(name, oapSettings("2"), mappings(properties), 7)); + assertTrue(client.templates().exists(name)); + final IndexTemplate template = client.templates().get(name).orElseThrow(); + assertEquals(name, template.getName()); + assertEquals(Collections.singletonList(name + "-*"), template.getIndexPatterns()); + assertEquals(Collections.singleton(name), template.getAliases().keySet()); + assertEquals(7, template.getOrder(), "the template's order (a composable template's priority)"); + assertEquals(properties, template.getMappings().getProperties()); + assertEquals(Collections.singleton("tags"), template.getMappings().getSource().getExcludes()); + assertEquals(oapSettings("2").get("index"), indexSettingsOf(template.getSettings())); + + final String day = name + "-20261118"; + assertTrue(client.index().create(day, null, null)); + final Index index = client.index().get(day).orElseThrow(); + assertEquals(asAnIndexReturnsThem(properties), index.getMappings().getProperties()); + assertEquals(Collections.singleton(name), index.getAliases().keySet()); + assertEquals(Collections.singleton(day), client.alias().indices(name).keySet()); + + final Map column = ImmutableMap.of("new_column", ImmutableMap.of("type", "keyword")); + assertTrue(client.index().putMapping(day, TYPE, mappings(column))); + final Map appended = new HashMap<>(properties); + appended.putAll(column); + assertEquals(asAnIndexReturnsThem(appended), + client.index().get(day).orElseThrow().getMappings().getProperties()); + + assertTrue(client.templates().createOrUpdate(name, oapSettings("2"), mappings(appended), 7)); + assertEquals(appended, client.templates().get(name).orElseThrow().getMappings().getProperties()); + + assertTrue(client.templates().delete(name)); + assertFalse(client.templates().exists(name)); + assertEquals(Optional.empty(), client.templates().get(name)); + + // An operator may order OAP's templates below others; a composable template cannot take a negative priority. + final String negative = "tpl-negative-order"; + assertTrue(client.templates().createOrUpdate(negative, oapSettings("1"), mappings(properties), -1)); + final IndexTemplate negativeTemplate = client.templates().get(negative).orElseThrow(); + assertEquals(usesLegacyTemplates() ? -1 : 0, negativeTemplate.getOrder()); + assertEquals(!usesLegacyTemplates(), negativeTemplate.isComposable()); + assertTrue(client.templates().delete(negative)); + assertTrue(client.index().delete(day)); + assertFalse(client.index().exists(day)); + assertThrows(Exception.class, () -> client.alias().indices(name), "an alias without indices is missing"); + } + + /** + * As the installer sets up a model stored in one index: created with its mappings and settings, read back for + * the installer's comparison, and a column appended. + */ + @Test + public void keepsAnIndexWithItsMappingsAndSettings() { + final String name = "idx-ui-template"; + assertFalse(client.index().exists(name)); + assertEquals(Optional.empty(), client.index().get(name)); + + final Map properties = oapProperties(); + assertTrue(client.index().create(name, mappings(properties), oapSettings("1"))); + assertTrue(client.index().exists(name)); + final Index index = client.index().get(name).orElseThrow(); + assertEquals(asAnIndexReturnsThem(properties), index.getMappings().getProperties()); + assertEquals(Collections.singleton("tags"), index.getMappings().getSource().getExcludes()); + assertEquals(oapSettings("1").get("index"), indexSettingsOf(index.getSettings())); + + final Map column = ImmutableMap.of("disabled", ImmutableMap.of("type", "integer")); + assertTrue(client.index().putMapping(name, TYPE, mappings(column))); + final Map appended = new HashMap<>(properties); + appended.putAll(column); + assertEquals(asAnIndexReturnsThem(appended), + client.index().get(name).orElseThrow().getMappings().getProperties()); + + assertTrue(client.index().delete(name)); + assertFalse(client.index().exists(name)); + } + + /** + * As the management DAOs and the single-document writes use the documents: each write refreshes, so the next + * search sees it at once. Ids carry what the client must escape in a path: spaces, as UI template ids do, + * slashes, as runtime rule names and Base64 service ids do, and the other characters a path gives a meaning to. + * A Base64 service id can also hold a {@code +}, which only a server reading it as a {@code +} keeps. + */ + @Test + public void writesAndReadsSingleDocumentsSearchableAtOnce() { + final String name = "doc-management"; + assertTrue(client.index().create(name, mappings(oapProperties()), oapSettings("1"))); + final Map refresh = ImmutableMap.of("refresh", "true"); + final List documentIds = new ArrayList<>(Arrays.asList( + "an id", "runtimerule_otel-rules_aws-gateway/gateway-service", "a?b#c%d", "YT8/.1")); + if (readsPlusInPathAsPlus()) { + documentIds.add("b3JkZXI+c3Zj.1"); + } + for (final String id : documentIds) { + final Map doc = ImmutableMap.of("service_id", id, "latency", 10); + client.documents().index(IndexRequest.builder().index(name).type(TYPE).id(id).doc(doc).build(), + refresh); + assertEquals(Collections.singletonList(id), ids(client.search(byIds(id), name)), id); + assertTrue(client.documents().exists(name, TYPE, id), id); + final Document read = client.documents().get(name, TYPE, id).orElseThrow(); + assertEquals(id, read.getId()); + assertEquals(doc, read.getSource(), id); + + final Map update = ImmutableMap.of("latency", 20); + client.documents().update(UpdateRequest.builder().index(name).type(TYPE).id(id).doc(update).build(), + refresh); + assertEquals(ImmutableMap.of("service_id", id, "latency", 20), + client.documents().get(name, TYPE, id).orElseThrow().getSource(), id); + assertEquals(Collections.singletonList(id), ids(client.search( + Search.builder().query(Query.bool().must(Query.term("service_id", id)).must(Query.term("latency", 20))) + .build(), name)), id); + + client.documents().deleteById(name, TYPE, id, refresh); + assertFalse(client.documents().exists(name, TYPE, id), id); + assertEquals(Collections.emptyList(), ids(client.search(byIds(id), name)), id); + } + assertTrue(client.index().delete(name)); + } + + /** + * As a read of rows by id across the day indices they may be in: ids missing from an index are left out. + */ + @Test + public void readsDocumentsOfSeveralIndicesById() { + final String first = "mget-20261117"; + final String second = "mget-20261118"; + final Map refresh = ImmutableMap.of("refresh", "true"); + for (final String index : Arrays.asList(first, second)) { + assertTrue(client.index().create(index, null, null)); + client.documents().index( + IndexRequest.builder().index(index).type(TYPE).id("in-" + index).doc(ImmutableMap.of("day", index)) + .build(), refresh); + } + final Map> ids = new LinkedHashMap<>(); + ids.put(first, Arrays.asList("in-" + first, "in-" + second)); + ids.put(second, Arrays.asList("in-" + second, "missing")); + final Documents documents = client.documents().mget(TYPE, ids).orElseThrow(); + final Map read = new HashMap<>(); + for (final Document document : documents) { + read.put(document.getId(), document.getSource().get("day")); + } + assertEquals(ImmutableMap.of("in-" + first, first, "in-" + second, second), read); + assertTrue(client.index().delete(first)); + assertTrue(client.index().delete(second)); + } + + /** + * As the persistence timer writes: inserts and updates in one bulk, the inserts routed by a column as the + * sharded mode does, read back by a search routed the same way. The routed rows share every searched value and + * differ only in their routing over two shards, so only a search that applies its routing reads a subset. + */ + @Test + public void bulkWritesInsertsUpdatesAndRoutedInserts() throws Exception { + final String name = "bulk-routed"; + // The server's default refresh interval, so the bulk's rows become searchable within a second. + assertTrue(client.index().create(name, mappings(oapProperties()), settings("2", "1s"))); + final Map refresh = ImmutableMap.of("refresh", "true"); + client.documents().index( + IndexRequest.builder().index(name).type(TYPE).id("existing").doc(ImmutableMap.of("latency", 1)).build(), + refresh); + + final List routings = new ArrayList<>(); + final BulkProcessor bulk = bulkProcessor(5 * 1024 * 1024); + try { + final List> written = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + routings.add("route-" + i); + written.add(bulk.add( + IndexRequest.builder().index(name).type(TYPE).id("routed-" + i).routing(Optional.of("route-" + i)) + .doc(ImmutableMap.of("service_id", "svc", "latency", i)).build())); + } + written.add(bulk.add( + UpdateRequest.builder().index(name).type(TYPE).id("existing").doc(ImmutableMap.of("latency", 2)) + .build())); + bulk.flush(); + for (final CompletableFuture future : written) { + future.get(30, TimeUnit.SECONDS); + } + } finally { + BulkProcessorITExt.close(bulk); + } + + final List all = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + all.add("routed-" + i); + } + await().atMost(1, TimeUnit.MINUTES).untilAsserted(() -> assertEquals(all, ids(client.search( + sorted(Query.term("service_id", "svc")), new SearchParams().routing(routings), name)))); + final Set readOneRouteAtATime = new TreeSet<>(); + boolean aRouteReadASubset = false; + for (int i = 0; i < 10; i++) { + final List routed = ids(client.search( + sorted(Query.term("service_id", "svc")), new SearchParams().routing("route-" + i), name)); + assertTrue(routed.contains("routed-" + i), "route-" + i + " reads " + routed); + aRouteReadASubset |= routed.size() < all.size(); + readOneRouteAtATime.addAll(routed); + } + assertEquals(new TreeSet<>(all), readOneRouteAtATime); + assertTrue(aRouteReadASubset, "a routed search read the rows of every shard"); + assertEquals(ImmutableMap.of("latency", 2), client.documents().get(name, TYPE, "existing").orElseThrow() + .getSource()); + assertTrue(client.index().delete(name)); + } + + /** + * A bulk answered with HTTP 200 can still reject single items, e.g. when an index is blocked for writes + * (apache/skywalking#14113): the rejected item's write fails, the other's succeeds, in one bulk request and in + * one request per item ({@code batchOfBytes(1)}). + */ + @Test + public void failsOnlyTheBulkItemsTheServerRejects() throws Exception { + final String normal = "bulk-item-failure-normal"; + final String blocked = "bulk-item-failure-blocked"; + assertTrue(client.index().create(normal, null, null)); + assertTrue(client.index().create(blocked, null, ImmutableMap.of("index.blocks.write", true))); + for (final int batchOfBytes : new int[] {5 * 1024 * 1024, 1}) { + final BulkProcessor bulk = bulkProcessor(batchOfBytes); + try { + final String normalId = "normal-" + batchOfBytes; + final String blockedId = "blocked-" + batchOfBytes; + final CompletableFuture normalWrite = bulk.add( + IndexRequest.builder().index(normal).type(TYPE).id(normalId).doc(ImmutableMap.of("key", "val")) + .build()); + final CompletableFuture blockedWrite = bulk.add( + IndexRequest.builder().index(blocked).type(TYPE).id(blockedId).doc(ImmutableMap.of("key", "val")) + .build()); + bulk.flush(); + // The scheduler's own periodical flush can race the explicit one, so wait on the futures themselves. + normalWrite.get(30, TimeUnit.SECONDS); + assertTrue(client.documents().get(normal, TYPE, normalId).isPresent()); + final String message = assertThrows( + ExecutionException.class, () -> blockedWrite.get(30, TimeUnit.SECONDS)).getCause().getMessage(); + assertTrue(message.contains("type=cluster_block_exception"), message); + assertFalse(message.contains(blockedId), "the rejected document id must not be logged: " + message); + } finally { + BulkProcessorITExt.close(bulk); + } + } + } + + /** + * Every query shape the query DAOs build, each against rows that miss it by one value: term and terms, range + * bounds (inclusive and exclusive), must, must-not and should, match on a keyword copied to an analyzed text + * column and on a number, match-phrase, and ids. + */ + @Test + public void searchesWithEveryQueryShapeOAPBuilds() { + final String name = writeRows("query-shapes"); + assertEquals(Arrays.asList("r1", "r3"), ids(client.search(sorted(Query.term("service_id", "svc-a")), name))); + assertEquals(Arrays.asList("r1", "r2", "r3"), + ids(client.search(sorted(Query.terms("service_id", "svc-a", "svc-b")), name))); + assertEquals(Arrays.asList("r2", "r4"), + ids(client.search(sorted(Query.terms("service_id", Arrays.asList("svc-b", "svc-c"))), name))); + + assertEquals(Arrays.asList("r2", "r3", "r4"), + ids(client.search(sorted(Query.range("latency").gte(20).lte(40)), name))); + assertEquals(Collections.singletonList("r3"), + ids(client.search(sorted(Query.range("latency").gt(20).lt(40)), name))); + + assertEquals(Collections.singletonList("r3"), ids(client.search(sorted( + Query.bool().must(Query.term("service_id", "svc-a")).mustNot(Query.term("latency", 10))), name))); + assertEquals(Arrays.asList("r2", "r4"), ids(client.search(sorted( + Query.bool().must(Query.bool().should(Query.term("service_id", "svc-b")) + .should(Query.term("latency", 40)))), name))); + + assertEquals(Arrays.asList("r1", "r3"), + ids(client.search(sorted(Query.match("statement_match", "select orders")), name))); + assertEquals(Collections.singletonList("r4"), ids(client.search(sorted(Query.match("is_error", 1)), name))); + assertEquals(Collections.singletonList("r1"), + ids(client.search(sorted(Query.matchPhrase("statement_match", "from orders where")), name))); + assertEquals(new TreeSet<>(Arrays.asList("r2", "r4")), + new TreeSet<>(ids(client.search(byIds("r2", "r4", "missing"), name)))); + + final SearchResponse byTag = client.search(sorted(Query.term("tags", "http.method=POST")), name); + assertEquals(Arrays.asList("r2", "r4"), ids(byTag)); + for (final SearchHit hit : byTag.getHits().getHits()) { + assertFalse(hit.getSource().containsKey("tags"), "a column kept out of _source: " + hit.getSource()); + assertEquals("raw-" + hit.getId(), hit.getSource().get("tags_raw")); + } + final Document fetched = client.documents().mget(TYPE, ImmutableMap.of(name, Collections.singletonList("r2"))) + .orElseThrow().iterator().next(); + assertFalse(fetched.getSource().containsKey("tags"), "a column kept out of _source: " + fetched.getSource()); + assertTrue(client.index().delete(name)); + } + + /** + * Pages of a sorted search, as the list queries read: by {@code from} and {@code size}, both orders, and only + * the fields asked for when a query names them. + */ + @Test + public void pagesSortsAndFiltersTheSource() { + final String name = writeRows("paging"); + final Function> page = from -> ids(client.search( + Search.builder().query(Query.range("latency").gte(0)).sort("latency", Sort.Order.DESC).from(from).size(2) + .build(), name)); + assertEquals(Arrays.asList("r4", "r3"), page.apply(0)); + assertEquals(Arrays.asList("r2", "r1"), page.apply(2)); + assertEquals(Collections.emptyList(), page.apply(4)); + assertEquals(Arrays.asList("r1", "r2", "r3", "r4"), ids(client.search( + Search.builder().query(Query.range("latency").gte(0)).sort("latency", Sort.Order.ASC).size(10).build(), + name))); + + final SearchResponse sources = client.search( + Search.builder().query(Query.term("service_id", "svc-b")).source("service_id").source("latency").size(1) + .build(), name); + assertEquals(ImmutableMap.of("service_id", "svc-b", "latency", 20), + sources.getHits().getHits().get(0).getSource()); + assertTrue(client.index().delete(name)); + } + + /** + * Every aggregation the query DAOs build: entities ranked by the mean of a column in both orders with a + * mapped execution hint (TopN), nested terms collected breadth first (topology), and the min and max of a + * column, alone and ordering terms (trace and profiling reads). + */ + @Test + @SuppressWarnings("unchecked") + public void aggregatesAsOAPRanksAndGroups() { + final String name = writeRows("aggregations"); + final Function ranked = asc -> client.search(Search.builder().size(0).aggregation( + Aggregation.terms("service_id").field("service_id").order(BucketOrder.aggregation("latency", asc)) + .size(2).subAggregation(Aggregation.avg("latency").field("latency")) + .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) + .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST)).build(), name); + // Means: svc-a (10, 32) 21, svc-b 20, svc-c 40. + assertEquals(Arrays.asList("svc-b=20.0", "svc-a=21.0"), buckets(ranked.apply(true), "service_id", "latency")); + assertEquals(Arrays.asList("svc-c=40.0", "svc-a=21.0"), buckets(ranked.apply(false), "service_id", "latency")); + + final SearchResponse nested = client.search(Search.builder().size(0).aggregation( + Aggregation.terms("service_id").field("service_id").size(1000) + .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) + .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST) + .subAggregation(Aggregation.terms("is_error").field("is_error") + .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) + .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST))) + .build(), name); + final Map> errorsByService = new LinkedHashMap<>(); + for (final Map bucket : bucketsOf(nested, "service_id")) { + errorsByService.put(String.valueOf(bucket.get("key")), ((List>) ((Map) + bucket.get("is_error")).get("buckets")).stream().map(b -> String.valueOf(b.get("key"))) + .collect(Collectors.toCollection(TreeSet::new))); + } + assertEquals(ImmutableMap.of("svc-a", Collections.singleton("0"), "svc-b", Collections.singleton("0"), + "svc-c", Collections.singleton("1")), errorsByService); + + final SearchResponse range = client.search(Search.builder().query(Query.term("service_id", "svc-a")).size(0) + .aggregation(Aggregation.min("min_latency").field("latency")) + .aggregation(Aggregation.max("max_latency").field("latency")).build(), name); + assertEquals(10.0, valueOf(range, "min_latency")); + assertEquals(32.0, valueOf(range, "max_latency")); + + final SearchResponse latestFirst = client.search(Search.builder().size(0).aggregation( + Aggregation.terms("service_id").field("service_id").size(2) + .order(BucketOrder.aggregation("max_latency", false)) + .subAggregation(Aggregation.max("max_latency").field("latency"))).build(), name); + assertEquals(Arrays.asList("svc-c=40.0", "svc-a=32.0"), buckets(latestFirst, "service_id", "max_latency")); + assertTrue(client.index().delete(name)); + } + + /** + * As OAP searches a time range's day indices, some of which do not exist: the missing ones are skipped. Without + * those parameters the server refuses the search, which is why OAP sends them. + */ + @Test + public void searchesDayIndicesSkippingTheMissingOnes() { + final String day = writeRows("days-20261118"); + final SearchParams skipMissing = new SearchParams().allowNoIndices(true).ignoreUnavailable(true) + .expandWildcards("open"); + assertEquals(Arrays.asList("r1", "r3"), ids(client.search( + sorted(Query.term("service_id", "svc-a")), skipMissing.copy(), "days-20261117", day))); + assertEquals(Collections.emptyList(), ids(client.search( + sorted(Query.term("service_id", "svc-a")), skipMissing.copy(), "days-20261116", "days-20261117"))); + assertThrows(Exception.class, () -> client.search( + sorted(Query.term("service_id", "svc-a")), new SearchParams(), "days-20261117", day)); + assertTrue(client.index().delete(day)); + } + + /** + * As the trace and profiling reads page through more rows than one search returns: a scroll over two day + * indices reads every row once, and clearing the context frees it on the server. + */ + @Test + public void scrollsThroughEveryRowAndClearsTheContext() { + final List expected = new ArrayList<>(); + for (final String day : Arrays.asList("scroll-20261117", "scroll-20261118")) { + assertTrue(client.index().create(day, null, null)); + for (int i = 0; i < 5; i++) { + final String id = day + "-" + i; + client.documents().index(IndexRequest.builder().index(day).type(TYPE).id(id) + .doc(ImmutableMap.of("day", day)).build(), + ImmutableMap.of("refresh", "true")); + expected.add(id); + } + } + final Duration retention = Duration.ofMinutes(1); + SearchResponse response = client.search( + Search.builder().size(3).build(), + new SearchParams().scroll(retention), "scroll-20261117", "scroll-20261118"); + final List read = new ArrayList<>(); + final String scrollId = response.getScrollId(); + while (!response.getHits().getHits().isEmpty()) { + read.addAll(ids(response)); + assertTrue(read.size() <= expected.size(), "read past the rows: " + read); + response = client.scroll(retention, response.getScrollId()); + } + assertEquals(new TreeSet<>(expected), new TreeSet<>(read)); + assertEquals(expected.size(), read.size(), "each row once: " + read); + assertTrue(client.deleteScrollContext(response.getScrollId() == null ? scrollId : response.getScrollId())); + assertThrows(Exception.class, () -> client.scroll(retention, scrollId)); + assertTrue(client.index().delete("scroll-20261117")); + assertTrue(client.index().delete("scroll-20261118")); + } + + /** + * Elasticsearch before 7.8 has only legacy index templates, which keep an order; composable ones keep a priority. + */ + private boolean usesLegacyTemplates() { + final String[] parts = majorMinor.split("\\."); + final int major = Integer.parseInt(parts[0]); + return !OPENSEARCH.equalsIgnoreCase(distribution) + && (major < 7 || major == 7 && Integer.parseInt(parts[1]) < 8); + } + + /** + * Elasticsearch before 7.4 decodes a {@code +} in a path as a space. + */ + private boolean readsPlusInPathAsPlus() { + final String[] parts = majorMinor.split("\\."); + final int major = Integer.parseInt(parts[0]); + return OPENSEARCH.equalsIgnoreCase(distribution) || major > 7 + || major == 7 && Integer.parseInt(parts[1]) >= 4; + } + + /** + * Four rows, each matching a query shape the others miss by one value: + *

+     * id  service_id  latency  is_error  statement                           tags
+     * r1  svc-a       10       0         select * from orders where id = ?   http.method=GET
+     * r2  svc-b       20       0         update users set name = ?           http.method=POST
+     * r3  svc-a       32       0         select count(*) from orders         http.method=GET
+     * r4  svc-c       40       1         delete from carts                   http.method=POST
+     * 
+ */ + private String writeRows(final String name) { + assertTrue(client.index().create(name, mappings(oapProperties()), oapSettings("1"))); + final Object[][] rows = { + {"r1", "svc-a", 10, 0, "select * from orders where id = ?"}, + {"r2", "svc-b", 20, 0, "update users set name = ?"}, + {"r3", "svc-a", 32, 0, "select count(*) from orders"}, + {"r4", "svc-c", 40, 1, "delete from carts"}, + }; + for (final Object[] row : rows) { + final String method = "svc-b".equals(row[1]) || "svc-c".equals(row[1]) ? "POST" : "GET"; + client.documents().index( + IndexRequest.builder().index(name).type(TYPE).id((String) row[0]) + .doc(ImmutableMap.builder() + .put("service_id", row[1]).put("latency", row[2]) + .put("is_error", row[3]).put("statement", row[4]) + .put("tags", "http.method=" + method).put("tags_raw", "raw-" + row[0]) + .build()) + .build(), ImmutableMap.of("refresh", "true")); + } + return name; + } + + /** + * Columns the way OAP's installer maps them: a keyword copied to an analyzed text column for match queries, + * numbers, a binary column, a column searched but kept out of {@code _source} (as segment tags are), and a column + * stored but neither indexed nor kept in doc values. A template reads them back exactly as written, which the + * installer's comparison of a stored template on the next boot relies on. + */ + private static Map oapProperties() { + final Map properties = new HashMap<>(); + properties.put("service_id", ImmutableMap.of("type", "keyword")); + properties.put("latency", ImmutableMap.of("type", "long")); + properties.put("is_error", ImmutableMap.of("type", "integer")); + properties.put("statement", ImmutableMap.of("type", "keyword", "copy_to", "statement_match")); + properties.put("statement_match", ImmutableMap.of("type", "text", "analyzer", "oap_analyzer")); + properties.put("data_binary", ImmutableMap.of("type", "binary")); + properties.put("tags", ImmutableMap.of("type", "keyword")); + properties.put("tags_raw", ImmutableMap.of("type", "keyword", "index", false, "doc_values", false)); + return properties; + } + + /** + * The columns as an index reads them back: the server lists every {@code copy_to}. The installer compares an + * existing day index by its column names only. + */ + @SuppressWarnings("unchecked") + private static Map asAnIndexReturnsThem(final Map properties) { + final Map read = new HashMap<>(); + properties.forEach((column, mapping) -> { + final Map columnMapping = new HashMap<>((Map) mapping); + columnMapping.computeIfPresent("copy_to", (key, to) -> Collections.singletonList(to)); + read.put(column, columnMapping); + }); + return read; + } + + private static Map oapSettings(final String shards) { + return settings(shards, "31s"); + } + + private static Map settings(final String shards, final String refreshInterval) { + return ImmutableMap.of("index", ImmutableMap.of( + "number_of_shards", shards, + "number_of_replicas", "0", + "refresh_interval", refreshInterval, + "analysis", ImmutableMap.of("analyzer", ImmutableMap.of("oap_analyzer", ImmutableMap.of("type", "stop"))) + )); + } + + private static Mappings mappings(final Map properties) { + final Mappings.Source source = new Mappings.Source(); + source.getExcludes().add("tags"); + return Mappings.builder().type(TYPE).properties(properties).source(source).build(); + } + + /** + * The settings OAP wrote, from what the server returns: the server adds its own (creation date, uuid, version, + * provided name), which OAP's comparison ignores. + */ + @SuppressWarnings("unchecked") + private static Map indexSettingsOf(final Map settings) { + final Map index = (Map) settings.get("index"); + final Map written = new HashMap<>(); + for (final String key : Arrays.asList("number_of_shards", "number_of_replicas", "refresh_interval", + "analysis")) { + if (index.containsKey(key)) { + written.put(key, index.get(key)); + } + } + return written; + } + + private BulkProcessor bulkProcessor(final int batchOfBytes) { + return BulkProcessor.builder() + .bulkActions(100) + .batchOfBytes(batchOfBytes) + .flushInterval(Duration.ofSeconds(30)) + .concurrentRequests(2) + .bulkMetrics(new HistogramMetrics() { + @Override + public void observe(final double value) { + } + }) + .build(new AtomicReference<>(client)); + } + + private static Search byIds(final String... ids) { + return Search.builder().query(Query.ids(ids)).size(ids.length).build(); + } + + private static Search sorted(final QueryBuilder query) { + final SearchBuilder search = Search.builder().query(query).size(100); + return search.sort("latency", Sort.Order.ASC).build(); + } + + private static List ids(final SearchResponse response) { + return response.getHits().getHits().stream().map(SearchHit::getId).collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + private static List> bucketsOf(final SearchResponse response, final String aggregation) { + return (List>) ((Map) response.getAggregations().get(aggregation)) + .get("buckets"); + } + + /** + * Each bucket as {@code key=value} of the sub-aggregation named {@code value}. + */ + @SuppressWarnings("unchecked") + private static List buckets(final SearchResponse response, final String aggregation, + final String value) { + return bucketsOf(response, aggregation).stream() + .map(bucket -> bucket.get("key") + "=" + + ((Map) bucket.get(value)).get("value")) + .collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + private static Object valueOf(final SearchResponse response, final String aggregation) { + return ((Map) response.getAggregations().get(aggregation)).get("value"); + } +} diff --git a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchIT.java b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchIT.java index fe0cf0c9e2b5..e4f5743d3d8b 100644 --- a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchIT.java +++ b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/ElasticSearchIT.java @@ -13,537 +13,135 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ package org.apache.skywalking.library.elasticsearch; -import org.apache.skywalking.library.elasticsearch.bulk.BulkProcessor; -import org.apache.skywalking.library.elasticsearch.client.TemplateClient; -import org.apache.skywalking.library.elasticsearch.requests.IndexRequest; -import org.apache.skywalking.library.elasticsearch.requests.UpdateRequest; -import org.apache.skywalking.library.elasticsearch.requests.search.Query; -import org.apache.skywalking.library.elasticsearch.requests.search.Search; -import org.apache.skywalking.library.elasticsearch.requests.search.aggregation.Aggregation; -import org.apache.skywalking.library.elasticsearch.response.Document; -import org.apache.skywalking.library.elasticsearch.response.Documents; -import org.apache.skywalking.library.elasticsearch.response.IndexTemplate; -import org.apache.skywalking.library.elasticsearch.response.Mappings; -import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; -import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; -import org.awaitility.Duration; +import java.time.Duration; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; import org.testcontainers.utility.DockerImageName; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - +/** + * The client's requests on the server versions OAP supports, one nested class per version, each starting its server + * once for all its tests. The client picks its request format by version, and every format runs here: Elasticsearch 6 + * with typed documents (6.8.23, the last 6.x release), 7.0-7.7 with legacy index templates (7.7.1, the last of them), + * 7.8 on with composable index templates (7.8.0, where they start, and 7.17.29, the last 7.x release), 8 and 9 with + * typeless single-document updates, and OpenSearch 1, 2 and 3. + */ @Tag("slow") public class ElasticSearchIT { - public static Collection es() { - // noinspection resource - return Arrays.asList(new Object[][] { - { - "ElasticSearch 7.4.2", - new ElasticsearchContainer( - DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss") - .withTag("7.4.2")) - }, - { - "ElasticSearch 7.8.0", - new ElasticsearchContainer( - DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss") - .withTag("7.8.0")) - }, - { - "ElasticSearch 7.15.0", - new ElasticsearchContainer( - DockerImageName.parse("elastic/elasticsearch") - .withTag("7.15.0") - .asCompatibleSubstituteFor( - "docker.elastic.co/elasticsearch/elasticsearch-oss")) - }, - { - "ElasticSearch 8.19.25", - new ElasticsearchContainer( - DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch") - .withTag("8.19.25")) - .withEnv("xpack.security.enabled", "false") - }, - { - "ElasticSearch 8.9.0", - new ElasticsearchContainer( - DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch") - .withTag("8.9.0")) - .withEnv("xpack.security.enabled", "false") - }, - { - "ElasticSearch 9.3.0", - new ElasticsearchContainer( - DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch") - .withTag("9.3.0")) - .withEnv("xpack.security.enabled", "false") - }, - { - "OpenSearch 1.3.20", - new ElasticsearchContainer( - DockerImageName.parse("opensearchproject/opensearch") - .withTag("1.3.20") - .asCompatibleSubstituteFor( - "docker.elastic.co/elasticsearch/elasticsearch-oss")) - .withEnv("plugins.security.disabled", "true") - // The image starts the Performance Analyzer agent's JVM together with the launcher's. A JVM that - // starts beside another can print an hsperfdata lock warning to stdout, which the launcher reads - // as part of its JVM options, and OpenSearch then fails with "main class Cannot" - // (opensearch-project/opensearch-build#3821). 2.4.0 has no switch for the agent. - .withEnv("DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI", "true") - .withStartupTimeout(java.time.Duration.ofMinutes(5)) - }, - { - "OpenSearch 2.4.0", - new ElasticsearchContainer( - DockerImageName.parse("opensearchproject/opensearch") - .withTag("2.4.0") - .asCompatibleSubstituteFor( - "docker.elastic.co/elasticsearch/elasticsearch-oss")) - .withEnv("plugins.security.disabled", "true") - .withStartupTimeout(java.time.Duration.ofMinutes(5)) - }, - { - "OpenSearch 2.8.0", - new ElasticsearchContainer( - DockerImageName.parse("opensearchproject/opensearch") - .withTag("2.8.0") - .asCompatibleSubstituteFor( - "docker.elastic.co/elasticsearch/elasticsearch-oss")) - .withEnv("plugins.security.disabled", "true") - .withEnv("DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI", "true") - .withStartupTimeout(java.time.Duration.ofMinutes(5)) - } - }); + private static final String ELASTICSEARCH = "ElasticSearch"; + private static final String OPENSEARCH = "OpenSearch"; + + @Nested + @DisplayName("Elasticsearch 6.8.23") + class Elasticsearch6823 extends ElasticSearchClientITBase { + Elasticsearch6823() { + super(new ElasticsearchContainer( + DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss").withTag("6.8.23")), + ELASTICSEARCH, "6.8"); + } } - @ParameterizedTest(name = "version: {0}") - @MethodSource("es") - public void testTemplate(final String ignored, - final ElasticsearchContainer server) { - server.start(); - - final ElasticSearch client = - ElasticSearch.builder() - .endpoints(server.getHttpHostAddress()) - .build(); - client.connect(); - - final String name = "test-template"; - final TemplateClient templateClient = client.templates(); - - final ImmutableMap properties = ImmutableMap.of( - "metric_table", ImmutableMap.of("type", "keyword"), - "service_id", ImmutableMap.of("type", "keyword") - ); - - final Mappings.Source sourceConf = new Mappings.Source(); - sourceConf.getExcludes().add("test"); - - final Mappings mappings = Mappings.builder() - .type("_doc") - .properties(properties) - .source(sourceConf) - .build(); - - assertThat(templateClient.createOrUpdate(name, ImmutableMap.of(), mappings, 0)) - .isTrue(); - - assertThat(templateClient.exists(name)).isTrue(); - - assertThat(templateClient.get(name)) - .isPresent() - .map(IndexTemplate::getMappings) - .map(Mappings::getProperties) - .hasValue(mappings.getProperties()); - assertThat(templateClient.get(name)) - .isPresent() - .map(IndexTemplate::getMappings) - .map(Mappings::getSource) - .map(Mappings.Source::getExcludes) - .hasValue(mappings.getSource().getExcludes()); - - server.close(); + @Nested + @DisplayName("Elasticsearch 7.7.1") + class Elasticsearch771 extends ElasticSearchClientITBase { + Elasticsearch771() { + super(new ElasticsearchContainer( + DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss").withTag("7.7.1")), + ELASTICSEARCH, "7.7"); + } } - @ParameterizedTest(name = "version: {0}") - @MethodSource("es") - public void testIndex(final String ignored, - final ElasticsearchContainer server) { - server.start(); - - final ElasticSearch client = - ElasticSearch.builder() - .endpoints(server.getHttpHostAddress()) - .build(); - client.connect(); - - final String index = "test-index"; - assertFalse(client.index().exists(index)); - assertFalse(client.index().get(index).isPresent()); - assertTrue(client.index().create(index, null, null)); - assertTrue(client.index().exists(index)); - assertNotNull(client.index().get(index)); - assertTrue(client.index().delete(index)); - assertFalse(client.index().get(index).isPresent()); - - server.close(); + @Nested + @DisplayName("Elasticsearch 7.8.0") + class Elasticsearch780 extends ElasticSearchClientITBase { + Elasticsearch780() { + super(new ElasticsearchContainer( + DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch-oss").withTag("7.8.0")), + ELASTICSEARCH, "7.8"); + } } - @ParameterizedTest(name = "version: {0}") - @MethodSource("es") - public void testDoc(final String ignored, - final ElasticsearchContainer server) { - server.start(); - - final ElasticSearch client = - ElasticSearch.builder() - .endpoints(server.getHttpHostAddress()) - .build(); - client.connect(); - - final String index = "test-index"; - assertTrue(client.index().create(index, null, null)); - - final ImmutableMap doc = ImmutableMap.of("key", "val"); - final String idWithSpace = "an id"; // UI management templates' IDs contains spaces - final String type = "type"; - - client.documents().index( - IndexRequest.builder() - .index(index) - .type(type) - .id(idWithSpace) - .doc(doc) - .build(), null); - - assertTrue(client.documents().get(index, type, idWithSpace).isPresent()); - assertEquals(client.documents().get(index, type, idWithSpace).get().getId(), idWithSpace); - assertEquals(client.documents().get(index, type, idWithSpace).get().getSource(), doc); - - server.close(); + @Nested + @DisplayName("Elasticsearch 7.17.29") + class Elasticsearch71729 extends ElasticSearchClientITBase { + Elasticsearch71729() { + super(new ElasticsearchContainer( + DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch").withTag("7.17.29")), + ELASTICSEARCH, "7.17"); + } } - @ParameterizedTest(name = "version: {0}") - @MethodSource("es") - public void testDocUpdate(final String ignored, - final ElasticsearchContainer server) { - server.start(); - - final ElasticSearch client = - ElasticSearch.builder() - .endpoints(server.getHttpHostAddress()) - .build(); - client.connect(); - - final String index = "test-index-update"; - assertTrue(client.index().create(index, null, null)); - - final ImmutableMap doc = ImmutableMap.of("key", "val"); - final String idWithSpace = "an id"; // UI management templates' IDs contains spaces - final String type = "type"; - - client.documents().index( - IndexRequest.builder() - .index(index) - .type(type) - .id(idWithSpace) - .doc(doc) - .build(), null); - - assertTrue(client.documents().get(index, type, idWithSpace).isPresent()); - assertEquals(client.documents().get(index, type, idWithSpace).get().getId(), idWithSpace); - assertEquals(client.documents().get(index, type, idWithSpace).get().getSource(), doc); - - final Map updatedDoc = ImmutableMap.of("key", "new-val"); - client.documents().update( - UpdateRequest - .builder() - .index(index) - .type(type) - .id(idWithSpace) - .doc(updatedDoc) - .build(), - null); - assertEquals(client.documents().get(index, type, idWithSpace).get().getSource(), updatedDoc); - - server.close(); + @Nested + @DisplayName("Elasticsearch 8.19.25") + class Elasticsearch81925 extends ElasticSearchClientITBase { + Elasticsearch81925() { + super(elasticsearch("8.19.25"), ELASTICSEARCH, "8.19"); + } } - @SuppressWarnings("unchecked") - @ParameterizedTest(name = "version: {0}") - @MethodSource("es") - public void testSearch(final String ignored, - final ElasticsearchContainer server) { - server.start(); - - final ElasticSearch client = - ElasticSearch.builder() - .endpoints(server.getHttpHostAddress()) - .build(); - client.connect(); - - final String index = "test-index"; - final Mappings.Source sourceConf = new Mappings.Source(); - sourceConf.getExcludes().add("key3"); - assertTrue( - client.index().create( - index, - Mappings.builder() - .type("type") - .properties(ImmutableMap.of("key1", ImmutableMap.of("type", "keyword"))) - .properties(ImmutableMap.of("key2", ImmutableMap.of("type", "keyword"), - "key3", ImmutableMap.of("type", "keyword") - )) - .source(sourceConf) - .build(), - null - ) - ); - - final String type = "type"; - - for (int i = 0; i < 10; i++) { - client.documents().index( - IndexRequest.builder() - .index(index) - .type(type) - .id("id" + i) - .doc(ImmutableMap.of("key1", "val" + i, "key2", "val" + (i + 1), "key3", "val" + (i + 2) - )) - .build(), null); + @Nested + @DisplayName("Elasticsearch 9.3.0") + class Elasticsearch930 extends ElasticSearchClientITBase { + Elasticsearch930() { + super(elasticsearch("9.3.0"), ELASTICSEARCH, "9.3"); } - - await().atMost(Duration.ONE_MINUTE).untilAsserted(() -> { - SearchResponse response = client.search( - Search.builder().query(Query.bool().must(Query.term("key1", "val1"))).build() - ); - assertEquals(1, response.getHits().getTotal()); - assertEquals("val1", response.getHits().iterator().next().getSource().get("key1")); - }); - //test indexOnly - await().atMost(Duration.ONE_MINUTE).untilAsserted(() -> { - SearchResponse response = client.search( - Search.builder().query(Query.bool().must(Query.term("key3", "val3"))).build() - ); - assertEquals(1, response.getHits().getTotal()); - assertEquals("val1", response.getHits().iterator().next().getSource().get("key1")); - assertNull(response.getHits().iterator().next().getSource().get("key3"), "indexOnly fields should not be stored"); - }); - - await().atMost(Duration.ONE_MINUTE) - .pollInterval(Duration.FIVE_SECONDS) - .untilAsserted(() -> { - SearchResponse response = client.search( - Search.builder() - .query( - Query.bool() - .must( - Query.bool() - .should(Query.bool() - .must(Query.term("key1", "val1")) - .must(Query.term("key2", "val2")) - .build()) - .should(Query.bool() - .must(Query.term("key1", "val3")) - .must(Query.term("key2", "val4")) - .build() - ))) - .aggregation( - Aggregation - .terms("key1").field("key1.keyword") - .subAggregation( - Aggregation.terms("key2").field("key2.keyword")) - .size(1000) - .build()) - .build()); - assertEquals(2, response.getHits().getTotal()); - assertEquals(1, response.getAggregations().size()); - assertEquals( - 2, - ( - (List) - ((Map) response.getAggregations().get("key1")) - .get("buckets") - ).size() - ); - - //test mGet - Map> indexIdsGroup = new HashMap<>(); - indexIdsGroup.put("test-index", Arrays.asList("id1", "id2")); - Optional documents = client.documents().mget(type, indexIdsGroup); - Map> result = new HashMap<>(); - assertThat(documents).isPresent(); - for (final Document document : documents.get()) { - result.put(document.getId(), document.getSource()); - } - assertEquals(2, result.get("id1").size()); - assertEquals(2, result.get("id2").size()); - }); - - server.close(); } - @ParameterizedTest(name = "version: {0}") - @MethodSource("es") - public void testDocDeleteById(final String ignored, - final ElasticsearchContainer server) { - server.start(); - - final ElasticSearch client = - ElasticSearch.builder() - .endpoints(server.getHttpHostAddress()) - .build(); - client.connect(); - - final String index = "test-index-delete"; - assertTrue(client.index().create(index, null, null)); - - final ImmutableMap doc = ImmutableMap.of("key", "val"); - final String idWithSpace = "an id"; // UI management templates' IDs contains spaces - final String type = "type"; - - client.documents().index( - IndexRequest.builder() - .index(index) - .type(type) - .id(idWithSpace) - .doc(doc) - .build(), null); - - assertTrue(client.documents().exists(index, type, idWithSpace)); - client.documents().deleteById(index, type, idWithSpace, ImmutableMap.of("refresh", "true")); - assertFalse(client.documents().exists(index, type, idWithSpace)); - server.close(); + @Nested + @DisplayName("OpenSearch 1.3.20") + class OpenSearch1320 extends ElasticSearchClientITBase { + OpenSearch1320() { + super(openSearch("1.3.20").withEnv("plugins.security.disabled", "true") + // The image starts the Performance Analyzer agent's JVM together with the launcher's. A JVM + // that starts beside another can print an hsperfdata lock warning to stdout, which the + // launcher reads as part of its JVM options, and OpenSearch then fails with "main class + // Cannot" (opensearch-project/opensearch-build#3821). + .withEnv("DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI", "true"), + OPENSEARCH, "1.3"); + } } - @ParameterizedTest(name = "version: {0}") - @MethodSource("es") - public void testClientBuilder(final String ignored, - final ElasticsearchContainer server) { - server.start(); - - // Test basic builder functionality - final ElasticSearch client = - ElasticSearch.builder() - .endpoints(server.getHttpHostAddress()) - .build(); - client.connect(); - - final String index = "test-builder-index"; - assertTrue(client.index().create(index, null, null)); - assertTrue(client.index().exists(index)); - assertTrue(client.index().delete(index)); - - server.close(); + @Nested + @DisplayName("OpenSearch 2.8.0") + class OpenSearch280 extends ElasticSearchClientITBase { + OpenSearch280() { + super(openSearch("2.8.0").withEnv("plugins.security.disabled", "true") + .withEnv("DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI", "true"), + OPENSEARCH, "2.8"); + } } - /** - * https://github.com/apache/skywalking/issues/14113: Elasticsearch answers a bulk write with overall HTTP 200 - * while rejecting individual items, e.g. when an index is blocked for writes. Run against every ES/OpenSearch - * version in the matrix, and with a chunked ({@code batchOfBytes(1)}) and unchunked bulk request, to also cover - * the fix that scoped a chunk's completion to its own requests instead of the whole flush. - */ - @ParameterizedTest(name = "version: {0}") - @MethodSource("es") - public void testBulkProcessorItemLevelFailure(final String ignored, - final ElasticsearchContainer server) throws Exception { - server.start(); - - final ElasticSearch client = - ElasticSearch.builder() - .endpoints(server.getHttpHostAddress()) - .build(); - client.connect(); - - final String normalIndex = "bulk-item-failure-normal"; - final String blockedIndex = "bulk-item-failure-blocked"; - assertTrue(client.index().create(normalIndex, null, null)); - final Map blockedSettings = new HashMap<>(); - blockedSettings.put("index.blocks.write", true); - assertTrue(client.index().create(blockedIndex, null, blockedSettings)); - - // One `_bulk` request carrying both a succeeding and a rejected item. - runBulkAndAssertItemLevelFailure(client, normalIndex, blockedIndex, "one-chunk", 5 * 1024 * 1024); - // `batchOfBytes(1)` forces each item into its own chunk/`_bulk` request. - runBulkAndAssertItemLevelFailure(client, normalIndex, blockedIndex, "many-chunks", 1); - - server.close(); + @Nested + @DisplayName("OpenSearch 3.0.0") + class OpenSearch300 extends ElasticSearchClientITBase { + OpenSearch300() { + super(openSearch("3.0.0") + // From 2.12 on, disabling the security plugin alone is not enough: the image still installs + // the demo security config, which refuses to run without an initial admin password. With the + // plugin removed, its plugins.security.disabled setting is unknown and stops the server. + .withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true") + .withEnv("DISABLE_SECURITY_PLUGIN", "true") + .withEnv("DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI", "true"), + OPENSEARCH, "3.0"); + } } - private static void runBulkAndAssertItemLevelFailure(final ElasticSearch client, - final String normalIndex, - final String blockedIndex, - final String idSuffix, - final int batchOfBytes) throws Exception { - final BulkProcessor bulkProcessor = BulkProcessor.builder() - .bulkActions(2) - .batchOfBytes(batchOfBytes) - .flushInterval(java.time.Duration.ofSeconds(30)) - .concurrentRequests(2) - .bulkMetrics(new HistogramMetrics() { - @Override - public void observe(final double value) { - } - }) - .build(new AtomicReference<>(client)); - - final String type = "type"; - final String normalId = "normal-" + idSuffix; - final String blockedId = "blocked-" + idSuffix; - - final CompletableFuture normalFuture = bulkProcessor.add( - IndexRequest.builder() - .index(normalIndex) - .type(type) - .id(normalId) - .doc(ImmutableMap.of("key", "val")) - .build()); - final CompletableFuture blockedFuture = bulkProcessor.add( - IndexRequest.builder() - .index(blockedIndex) - .type(type) - .id(blockedId) - .doc(ImmutableMap.of("key", "val")) - .build()); - - bulkProcessor.flush(); - - // The scheduler thread's own periodical flush can race this method's explicit flush() and drain either - // or both requests first, so wait on the futures themselves instead of asserting isDone() right after - // flush() returns. - normalFuture.get(30, TimeUnit.SECONDS); - assertTrue(client.documents().get(normalIndex, type, normalId).isPresent()); + private static ElasticsearchContainer elasticsearch(final String version) { + return new ElasticsearchContainer( + DockerImageName.parse("docker.elastic.co/elasticsearch/elasticsearch").withTag(version)) + .withEnv("xpack.security.enabled", "false"); + } - final String message = assertThrows( - ExecutionException.class, () -> blockedFuture.get(30, TimeUnit.SECONDS)).getCause().getMessage(); - assertTrue(message.contains("type=cluster_block_exception"), message); - assertFalse(message.contains(blockedId), "the rejected document id must not be logged: " + message); + private static ElasticsearchContainer openSearch(final String version) { + return new ElasticsearchContainer( + DockerImageName.parse("opensearchproject/opensearch").withTag(version) + .asCompatibleSubstituteFor("docker.elastic.co/elasticsearch/elasticsearch-oss")) + .withStartupTimeout(Duration.ofMinutes(5)); } } diff --git a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessorITExt.java b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessorITExt.java new file mode 100644 index 000000000000..4e2fd98b00db --- /dev/null +++ b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessorITExt.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.library.elasticsearch.bulk; + +import java.lang.reflect.Field; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * Closes a {@link BulkProcessor} for the ITs, which OAP never does: its periodical flush runs as long as OAP. An IT + * closes its processors before it shuts the client down, so no flush thread outlives the test. + */ +public final class BulkProcessorITExt { + private BulkProcessorITExt() { + } + + /** + * Stops the periodical flush, lets a flush already running finish its bulk, and sends what is still queued. + */ + public static void close(final BulkProcessor processor) { + final ScheduledThreadPoolExecutor scheduler = scheduler(processor); + scheduler.shutdown(); + try { + if (!scheduler.awaitTermination(1, TimeUnit.MINUTES)) { + throw new IllegalStateException("The periodical flush did not finish in a minute"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while the periodical flush finished", e); + } + processor.flush(); + } + + private static ScheduledThreadPoolExecutor scheduler(final BulkProcessor processor) { + try { + final Field field = BulkProcessor.class.getDeclaredField("scheduler"); + field.setAccessible(true); + return (ScheduledThreadPoolExecutor) field.get(processor); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("BulkProcessor keeps its periodical flush in a field named scheduler", e); + } + } +} diff --git a/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessorITExtTest.java b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessorITExtTest.java new file mode 100644 index 000000000000..bf75cdc408dd --- /dev/null +++ b/oap-server/server-library/library-elasticsearch-client/src/test/java/org/apache/skywalking/library/elasticsearch/bulk/BulkProcessorITExtTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.library.elasticsearch.bulk; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.skywalking.oap.server.telemetry.api.HistogramMetrics; +import org.junit.jupiter.api.Test; + +import static org.awaitility.Awaitility.await; +import static org.mockito.Mockito.mock; + +class BulkProcessorITExtTest { + /** + * The periodical flush runs on a thread of the processor's own, which an IT has to end, or every processor it + * leaves behind keeps a thread alive. + */ + @Test + void endsTheFlushThreadOnClose() { + final long running = flushThreads(); + final BulkProcessor processor = BulkProcessor.builder() + .bulkActions(10) + .batchOfBytes(1024) + .flushInterval(Duration.ofSeconds(1)) + .concurrentRequests(1) + .bulkMetrics(mock(HistogramMetrics.class)) + .build(new AtomicReference<>()); + await().atMost(10, TimeUnit.SECONDS).until(() -> flushThreads() == running + 1); + + BulkProcessorITExt.close(processor); + + await().atMost(10, TimeUnit.SECONDS).until(() -> flushThreads() == running); + } + + private static long flushThreads() { + return Thread.getAllStackTraces().keySet().stream() + .filter(thread -> thread.isAlive() && "ElasticSearch BulkProcessor".equals(thread.getName())) + .count(); + } +} diff --git a/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/ContinuousProfilingServiceHandler.java b/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/ContinuousProfilingServiceHandler.java index 119975b04aa4..f176f98461a5 100644 --- a/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/ContinuousProfilingServiceHandler.java +++ b/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/ContinuousProfilingServiceHandler.java @@ -187,7 +187,7 @@ public void reportProfilingTask(ContinuousProfilingReport request, StreamObserve task.setFixedTriggerDuration(request.getDuration()); task.setCreateTime(currentTime); task.setLastUpdateTime(currentTime); - task.setTimeBucket(TimeBucket.getRecordTimeBucket(currentTime)); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(currentTime)); final EBPFProfilingTaskContinuousProfiling continuousProfiling = new EBPFProfilingTaskContinuousProfiling(); continuousProfiling.setProcessId(processId); diff --git a/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/ContinuousProfilingServiceHandlerTest.java b/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/ContinuousProfilingServiceHandlerTest.java new file mode 100644 index 000000000000..03406af46061 --- /dev/null +++ b/oap-server/server-receiver-plugin/skywalking-ebpf-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/ebpf/provider/handler/ContinuousProfilingServiceHandlerTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.receiver.ebpf.provider.handler; + +import io.grpc.stub.StreamObserver; +import org.apache.skywalking.apm.network.ebpf.profiling.v3.ContinuousOnCPUProfilingTask; +import org.apache.skywalking.apm.network.ebpf.profiling.v3.ContinuousProfilingReport; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.command.CommandService; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTaskRecord; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.profiling.continuous.IContinuousProfilingPolicyDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.receiver.ebpf.provider.EBPFReceiverModuleConfig; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class ContinuousProfilingServiceHandlerTest { + /** + * The task model has minute downsampling, and Elasticsearch writes a task into the daily index of its time bucket, + * which the TTL deletes with its day. + */ + @Test + @SuppressWarnings("unchecked") + public void storesATriggeredTaskInTheMinuteItIsTriggeredIn() { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleProviderHolder storage = mock(ModuleProviderHolder.class); + final ModuleServiceHolder storageServices = mock(ModuleServiceHolder.class); + when(moduleManager.find(StorageModule.NAME)).thenReturn(storage); + when(storage.provider()).thenReturn(storageServices); + when(storageServices.getService(IContinuousProfilingPolicyDAO.class)) + .thenReturn(mock(IContinuousProfilingPolicyDAO.class)); + final ModuleProviderHolder core = mock(ModuleProviderHolder.class); + final ModuleServiceHolder coreServices = mock(ModuleServiceHolder.class); + when(moduleManager.find(CoreModule.NAME)).thenReturn(core); + when(core.provider()).thenReturn(coreServices); + when(coreServices.getService(CommandService.class)).thenReturn(new CommandService(moduleManager)); + final ContinuousProfilingServiceHandler handler = + new ContinuousProfilingServiceHandler(moduleManager, new EBPFReceiverModuleConfig()); + + final NoneStreamProcessor tasks = mock(NoneStreamProcessor.class); + final long before; + final long after; + try (MockedStatic processor = mockStatic(NoneStreamProcessor.class)) { + processor.when(NoneStreamProcessor::getInstance).thenReturn(tasks); + before = System.currentTimeMillis(); + handler.reportProfilingTask( + ContinuousProfilingReport.newBuilder() + .setServiceName("checkout") + .setInstanceName("checkout-1") + .setProcessName("checkout") + .setDuration(60) + .setOnCPU(ContinuousOnCPUProfilingTask.newBuilder()) + .build(), + mock(StreamObserver.class)); + after = System.currentTimeMillis(); + } + + final ArgumentCaptor written = ArgumentCaptor.forClass(NoneStream.class); + verify(tasks).in(written.capture()); + final EBPFProfilingTaskRecord task = (EBPFProfilingTaskRecord) written.getValue(); + assertTrue(task.getCreateTime() >= before && task.getCreateTime() <= after, + "created at " + task.getCreateTime()); + assertEquals(TimeBucket.getMinuteTimeBucket(task.getCreateTime()), task.getTimeBucket()); + } +} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java index 181929cc798c..552a11a1e626 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/AbstractBanyanDBDAO.java @@ -20,6 +20,7 @@ import com.google.gson.Gson; import java.util.Collections; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.skywalking.banyandb.model.v1.BanyandbModel; @@ -33,6 +34,7 @@ import org.apache.skywalking.library.banyandb.v1.client.TraceQueryResponse; import org.apache.skywalking.library.banyandb.v1.client.Value; import org.apache.skywalking.library.banyandb.v1.client.metadata.Serializable; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.query.input.AttrCondition; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.query.type.KeyValue; @@ -58,6 +60,25 @@ public abstract class AbstractBanyanDBDAO extends AbstractDAO getTaskList(String serviceId, Long startTimeBucke startTS = TimeBucket.getTimestamp(startTimeBucket); } if (endTimeBucket != null) { - endTS = TimeBucket.getTimestamp(endTimeBucket); + endTS = endOfTimeBucket(endTimeBucket); } Conditions where = Conditions.create(); if (StringUtil.isNotEmpty(serviceId)) { diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java index e3df457b2d1b..409a7887dad0 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBPprofTaskQueryDAO.java @@ -66,7 +66,7 @@ public List getTaskList(String serviceId, Long startTimeBucket, Long startTS = TimeBucket.getTimestamp(startTimeBucket); } if (endTimeBucket != null) { - endTS = TimeBucket.getTimestamp(endTimeBucket); + endTS = endOfTimeBucket(endTimeBucket); } Conditions where = Conditions.create(); if (StringUtil.isNotEmpty(serviceId)) { diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileTaskQueryDAO.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileTaskQueryDAO.java index 3178c33db5f8..6bc766825c92 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileTaskQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/stream/BanyanDBProfileTaskQueryDAO.java @@ -63,7 +63,7 @@ public List getTaskList(String serviceId, String endpointName, Long startTS = TimeBucket.getTimestamp(startTimeBucket); } if (endTimeBucket != null) { - endTS = TimeBucket.getTimestamp(endTimeBucket); + endTS = endOfTimeBucket(endTimeBucket); } Conditions where = Conditions.create(); if (StringUtil.isNotEmpty(serviceId)) { diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBProfilingTaskQueryIT.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBProfilingTaskQueryIT.java index f29689d8ea23..5dbc2443d460 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBProfilingTaskQueryIT.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBProfilingTaskQueryIT.java @@ -91,17 +91,17 @@ public static void writeProfiling() throws Exception { profileTask("trace-task-0", SERVICE_ID, "/checkout", 0), profileTask("trace-task-1", SERVICE_ID, "/pay", 1), profileTask("trace-task-2", SERVICE_ID, "/checkout", 2), - profileTask("trace-task-3", OTHER_SERVICE_ID, "/checkout", 3))); + halfAMinuteLater(profileTask("trace-task-3", OTHER_SERVICE_ID, "/checkout", 3)))); SUPPORT.writeNoneStreams(SUPPORT.installNoneStream(AsyncProfilerTaskRecord.class, DefaultScopeDefine.ASYNC_PROFILER_TASK, AsyncProfilerTaskRecord.INDEX_NAME), new AsyncProfilerTaskRecord.Builder(), Arrays.asList( asyncTask("async-task-0", SERVICE_ID, 0), asyncTask("async-task-1", SERVICE_ID, 1), - asyncTask("async-task-2", OTHER_SERVICE_ID, 2))); + halfAMinuteLater(asyncTask("async-task-2", OTHER_SERVICE_ID, 2)))); SUPPORT.writeNoneStreams(SUPPORT.installNoneStream(PprofTaskRecord.class, DefaultScopeDefine.PPROF_TASK, PprofTaskRecord.INDEX_NAME), new PprofTaskRecord.Builder(), Arrays.asList( pprofTask("pprof-task-0", SERVICE_ID, 0), pprofTask("pprof-task-1", SERVICE_ID, 1), - pprofTask("pprof-task-2", OTHER_SERVICE_ID, 2))); + halfAMinuteLater(pprofTask("pprof-task-2", OTHER_SERVICE_ID, 2)))); SUPPORT.writeNoneStreams(SUPPORT.installNoneStream(EBPFProfilingTaskRecord.class, DefaultScopeDefine.EBPF_PROFILING_TASK, EBPFProfilingTaskRecord.INDEX_NAME), new EBPFProfilingTaskRecord.Builder(), Arrays.asList( @@ -193,6 +193,27 @@ public void listsAsyncProfilerAndPprofTasks() throws IOException { assertNull(pprof.getById("pprof-task-missing")); } + /** + * A read bounded by an end time bucket keeps the tasks of the whole bucket, as the JDBC and Elasticsearch task + * DAOs keep the rows whose bucket equals it: the creation checks and the agent task caches bound their reads by + * minute buckets. The other service's tasks start or are created half a minute into their minute. + */ + @Test + public void readsTheTasksOfTheWholeEndMinute() throws IOException { + final BanyanDBProfileTaskQueryDAO trace = new BanyanDBProfileTaskQueryDAO(SUPPORT.getClient(), 200); + assertEquals(Collections.singletonList("trace-task-3"), + ids(trace.getTaskList(OTHER_SERVICE_ID, null, null, minuteBucket(3), 1))); + assertEquals(Collections.emptyList(), ids(trace.getTaskList(OTHER_SERVICE_ID, null, null, minuteBucket(2), 1))); + final BanyanDBAsyncProfilerTaskQueryDAO async = new BanyanDBAsyncProfilerTaskQueryDAO(SUPPORT.getClient(), 200); + assertEquals(Collections.singletonList("async-task-2"), async.getTaskList( + OTHER_SERVICE_ID, null, minuteBucket(2), 1).stream().map(t -> t.getId()).collect(Collectors.toList())); + assertEquals(Collections.emptyList(), async.getTaskList(OTHER_SERVICE_ID, null, minuteBucket(1), 1)); + final BanyanDBPprofTaskQueryDAO pprof = new BanyanDBPprofTaskQueryDAO(SUPPORT.getClient(), 200); + assertEquals(Collections.singletonList("pprof-task-2"), pprof.getTaskList( + OTHER_SERVICE_ID, null, minuteBucket(2), 1).stream().map(t -> t.getId()).collect(Collectors.toList())); + assertEquals(Collections.emptyList(), pprof.getTaskList(OTHER_SERVICE_ID, null, minuteBucket(1), 1)); + } + @Test public void readsTaskLogs() throws IOException { assertEquals(Arrays.asList("trace-task-0|" + INSTANCE_ID, "trace-task-1|" + OTHER_INSTANCE_ID), @@ -262,6 +283,22 @@ private static long minuteBucket(int minute) { return TimeBucket.getMinuteTimeBucket(startTime(minute)); } + private static ProfileTaskRecord halfAMinuteLater(ProfileTaskRecord task) { + task.setStartTime(task.getStartTime() + 30_000L); + task.setCreateTime(task.getCreateTime() + 30_000L); + return task; + } + + private static AsyncProfilerTaskRecord halfAMinuteLater(AsyncProfilerTaskRecord task) { + task.setCreateTime(task.getCreateTime() + 30_000L); + return task; + } + + private static PprofTaskRecord halfAMinuteLater(PprofTaskRecord task) { + task.setCreateTime(task.getCreateTime() + 30_000L); + return task; + } + private static ProfileTaskRecord profileTask(String id, String serviceId, String endpoint, int minute) { final ProfileTaskRecord task = new ProfileTaskRecord(); task.setTaskId(id); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/pom.xml b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/pom.xml index d2c7d8baf2a8..5db1fd4df1ff 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/pom.xml @@ -57,5 +57,12 @@ ${project.version} test + + + org.apache.skywalking + oal-rt + ${project.version} + test + diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java index 5147ed9d067d..056150ae6fe4 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java @@ -56,7 +56,9 @@ public void deleteHistory(Model model, String timeBucketColumnName, int ttl) { } long deadline = Long.parseLong(new DateTime().plusDays(-ttl).toString("yyyyMMdd")); String tableName = IndexController.INSTANCE.getTableName(model); - Long latestSuccessDeadline = this.indexLatestSuccess.get(model.getName()); + // Keyed by the physical table, as the deletion is: once one model cleans the table's indices to a deadline, + // the other models of the table have nothing left to clean for it. + Long latestSuccessDeadline = this.indexLatestSuccess.get(tableName); if (latestSuccessDeadline != null && deadline <= latestSuccessDeadline) { if (log.isDebugEnabled()) { log.debug("Index = {} already deleted, skip, deadline = {}, ttl = {}", tableName, deadline, ttl); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexNames.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexNames.java index 0ff13134754b..f9252ce04294 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexNames.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexNames.java @@ -44,6 +44,7 @@ public final class IndexNames { * Day groups are counted from this day, so a day lands in the same index whenever the OAP starts. */ private static final DateTime DAY_ONE = TIME_BUCKET_FORMATTER.parseDateTime("20000101"); + private static final int HOUR_SUFFIX_LENGTH = 10; private static int DAY_STEP = 1; private static int SUPER_DATASET_DAY_STEP = 1; @@ -60,14 +61,16 @@ public static void configure(final int dayStep, final int superDatasetDayStep) { } /** - * @return the index a document of the model with the given time bucket is written to + * @return the index a document of the model with the given time bucket is written to: the index of the bucket's + * day, read by the bucket's own precision, as the async-profiler and pprof tasks carry second buckets on a + * minute-downsampled model */ public static String writeIndex(final Model model, final long timeBucket) { final IndexFamily family = IndexFamily.of(model); if (family.getRolling() == IndexRolling.NONE) { return family.getPhysicalName(); } - return datedIndex(family, toDay(timeBucket, model.getDownsampling())); + return datedIndex(family, dayOf(timeBucket, model.getDownsampling())); } /** @@ -93,6 +96,19 @@ public static String[] rangeIndices(final IndexFamily family, @Nullable final Du return rangeIndices(family, duration.getStartTimeBucketInSec(), duration.getEndTimeBucketInSec()); } + /** + * @return the indices holding every instant of the duration, from its start through its end instant, + * {@link Duration#getEndTimestamp()}, in day order, or the table's alias when there is no duration. A duration + * ending in a day's last step ends at the next midnight, so its end instant is in the next day's index. + */ + public static String[] rangeIndicesThroughEndInstant(final IndexFamily family, + @Nullable final Duration duration) { + if (duration == null) { + return new String[] {family.getPhysicalName()}; + } + return rangeIndicesBetweenTimestamps(family, duration.getStartTimestamp(), duration.getEndTimestamp()); + } + /** * @return the indices holding the days between the two timestamps. The caller decides what a timestamp * that is not positive means. @@ -112,11 +128,17 @@ public static String latestIndex(final IndexFamily family, final long currentTim } /** + * Earlier versions named the index of an async-profiler, pprof or continuous-profiling task by the hour of the + * task's second bucket, {@code -}, with a day step of 1. Such an index holds the + * hour's day; no other index name has ten digits after its last {@code -}. + * * @param concreteIndexName a dated index name, with or without the namespace * @return the first day the index holds */ public static long indexStartDay(final String concreteIndexName) { - return Long.parseLong(concreteIndexName.substring(concreteIndexName.lastIndexOf(Const.LINE) + 1)); + final String suffix = concreteIndexName.substring(concreteIndexName.lastIndexOf(Const.LINE) + 1); + final long day = Long.parseLong(suffix); + return suffix.length() == HOUR_SUFFIX_LENGTH ? day / 100 : day; } /** @@ -176,6 +198,26 @@ private static long rollDay(final DateTime day, final int dayStep) { return Long.parseLong(day.minusDays(groupOffset).toString(TIME_BUCKET_FORMATTER)); } + /** + * @return the day of the time bucket by the bucket's own precision, or by the downsampling for a bucket of no + * known precision + */ + private static long dayOf(final long timeBucket, final DownSampling downSampling) { + if (TimeBucket.isSecondBucket(timeBucket)) { + return toDay(timeBucket, DownSampling.Second); + } + if (TimeBucket.isMinuteBucket(timeBucket)) { + return toDay(timeBucket, DownSampling.Minute); + } + if (TimeBucket.isHourBucket(timeBucket)) { + return toDay(timeBucket, DownSampling.Hour); + } + if (TimeBucket.isDayBucket(timeBucket)) { + return toDay(timeBucket, DownSampling.Day); + } + return toDay(timeBucket, downSampling); + } + private static long toDay(final long timeBucket, final DownSampling downSampling) { switch (downSampling) { case Second: diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ManagementCRUDEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ManagementCRUDEsDAO.java index 25b3ed468d7e..dc807cf6e9b3 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ManagementCRUDEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/ManagementCRUDEsDAO.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Map; +import org.apache.skywalking.library.elasticsearch.exception.ResponseException; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; @@ -147,11 +148,10 @@ public boolean deleteById(String modelName, String id) throws IOException { final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(modelName); final String docId = IndexController.INSTANCE.generateDocId(modelName, id); - final boolean exist = getClient().existDoc(index, docId); - if (!exist) { - return false; + try { + return getClient().deleteById(index, docId); + } catch (ResponseException e) { + throw new IOException("failed to delete " + docId + " from " + index, e); } - getClient().deleteById(index, docId); - return true; } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java index f0fa766f1261..9cdd9e6228db 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstaller.java @@ -140,13 +140,23 @@ public InstallInfo isExists(Model model, StorageManipulationOpt opt) throws Stor installInfo.setAllFieldsExist(containsMapping); boolean containsSetting = structures.compareIndexSetting(tableName, createSetting(model)); installInfo.setAllIndexSettingsExist(containsSetting); - exist = containsMapping && containsSetting; + exist = containsMapping && containsSetting + && hasConfiguredOrder(template.get(), config.getIndexTemplateOrder()); } } installInfo.setAllExist(exist); return installInfo; } + /** + * A template stored with another order (a composable template's priority) loses to, or beats, the templates the + * operator ordered it against, so it is updated like a template missing a column. A composable template stores a + * negative order as priority 0; a legacy template keeps it. + */ + static boolean hasConfiguredOrder(final IndexTemplate template, final int order) { + return template.getOrder() == (template.isComposable() ? Math.max(0, order) : order); + } + /** * Registers the model's physical table, the local state reads translate a model name with. A node that * applies a model without a schema change, such as a peer reinstalling a bundled rule, never calls @@ -213,7 +223,10 @@ private void createTimeSeriesTable(Model model) throws StorageException { boolean shouldUpdateTemplate = !esClient.isExistsTemplate(tableName); shouldUpdateTemplate = shouldUpdateTemplate || !structures.containsMapping(tableName, mapping) - || !structures.compareIndexSetting(tableName, settings); + || !structures.compareIndexSetting(tableName, settings) + || !esClient.getTemplate(tableName) + .map(template -> hasConfiguredOrder(template, config.getIndexTemplateOrder())) + .orElse(false); if (shouldUpdateTemplate) { structures.putStructure(tableName, mapping, settings); boolean isAcknowledged = esClient.createOrUpdateTemplate( diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AIAgentConversationQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AIAgentConversationQueryEsDAO.java index c7c6b49baa01..b87ae9d02d3c 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AIAgentConversationQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AIAgentConversationQueryEsDAO.java @@ -34,7 +34,6 @@ import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.aiagent.AIAgentSessionDataRecord; import org.apache.skywalking.oap.server.core.analysis.manual.aiagent.AIAgentSessionFlowRecord; -import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.storage.query.IAIAgentConversationQueryDAO; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; @@ -97,12 +96,10 @@ public List queryRounds(final String serviceId, if (StringUtil.isNotEmpty(conversation)) { query.must(Query.term(AIAgentSessionFlowRecord.CONVERSATION, conversation)); } - long startSecondTB = 0; - long endSecondTB = 0; if (duration != null) { - startSecondTB = duration.getStartTimeBucketInSec(); - endSecondTB = duration.getEndTimeBucketInSec(); - query.must(Query.range(Record.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); + query.must(Query.range(AIAgentSessionFlowRecord.TIMESTAMP) + .gte(duration.getStartTimestamp()) + .lte(duration.getEndTimestamp())); } final SearchBuilder search = Search.builder() .query(query) @@ -114,7 +111,7 @@ public List queryRounds(final String serviceId, } } final SearchResponse response = searchDebuggable( - IndexNames.rangeIndices(IndexFamily.of(AIAgentSessionFlowRecord.INDEX_NAME), duration), + IndexNames.rangeIndicesThroughEndInstant(IndexFamily.of(AIAgentSessionFlowRecord.INDEX_NAME), duration), search.build() ); return rounds(response, includeBody); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java index 7ab113e2cd9a..a05a536c0795 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AggregationQueryEsDAO.java @@ -107,6 +107,13 @@ public List sortMetrics(final TopNCondition condition, search.query(boolQuery); } + // A terms aggregation ordered by the mean: each shard returns its own top shard_size buckets (by default + // size * 1.5 + 10) by its own part of the mean, and Elasticsearch documents that ordering terms by a + // sub-aggregation generally produces incorrect ordering across shards (only max descending and min ascending + // are safe). It is exact when one shard holds the range, one daily index of one primary shard. Across indices + // or shards (midnight, a multi-day day step, more primary shards) an entity that ranks high over the range + // but not within each shard's own data can be missing, and a returned entity's mean covers only the shards + // that returned it. Reading every entity's mean instead is not worth its cost. search.aggregation( Aggregation.terms(Metrics.ENTITY_ID) .field(Metrics.ENTITY_ID) diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AlarmQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AlarmQueryEsDAO.java index eadf97b52eeb..3bfe56148981 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AlarmQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AlarmQueryEsDAO.java @@ -21,6 +21,7 @@ import com.google.common.base.Strings; import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.RangeQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Search; import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Sort; @@ -28,6 +29,7 @@ import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.alarm.AlarmRecord; import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.alarm.AlarmRecoveryRecord; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; import org.apache.skywalking.oap.server.core.query.input.AlarmQueryCondition; @@ -63,8 +65,6 @@ public Alarms getAlarm(final Integer scopeId, final String keyword, final int li final Duration duration, final List tags) throws IOException { - long startTB = duration.getStartTimeBucketInSec(); - long endTB = duration.getEndTimeBucketInSec(); final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(AlarmRecord.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); @@ -72,9 +72,7 @@ public Alarms getAlarm(final Integer scopeId, final String keyword, final int li query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, AlarmRecord.INDEX_NAME)); } - if (startTB != 0 && endTB != 0) { - query.must(Query.range(AlarmRecord.TIME_BUCKET).gte(startTB).lte(endTB)); - } + query.must(startTimeWithin(duration)); if (Objects.nonNull(scopeId)) { query.must(Query.term(AlarmRecord.SCOPE, scopeId)); @@ -117,17 +115,13 @@ public Alarms queryAlarms(final AlarmQueryCondition condition, final int limit, return new Alarms(); } final Duration duration = condition.getDuration(); - long startTB = duration.getStartTimeBucketInSec(); - long endTB = duration.getEndTimeBucketInSec(); final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(AlarmRecord.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(AlarmRecord.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, AlarmRecord.INDEX_NAME)); } - if (startTB != 0 && endTB != 0) { - query.must(Query.range(AlarmRecord.TIME_BUCKET).gte(startTB).lte(endTB)); - } + query.must(startTimeWithin(duration)); if (!Strings.isNullOrEmpty(condition.getKeyword())) { String matchCName = MatchCNameBuilder.INSTANCE.build(AlarmRecord.ALARM_MESSAGE); query.must(Query.matchPhrase(matchCName, condition.getKeyword())); @@ -180,6 +174,13 @@ public Alarms queryAlarms(final AlarmQueryCondition condition, final int limit, return alarms; } + /** + * The alarms starting from the duration's start through its end instant, {@link Duration#getEndTimestamp()}. + */ + private static RangeQueryBuilder startTimeWithin(final Duration duration) { + return Query.range(AlarmRecord.START_TIME).gte(duration.getStartTimestamp()).lte(duration.getEndTimestamp()); + } + private void updateAlarmRecoveryTime(Alarms alarms, Duration duration) throws IOException { List alarmMessages = alarms.getMsgs(); Map alarmRecoveryRecordMap = getAlarmRecoveryRecord(alarmMessages, duration); @@ -198,8 +199,10 @@ private Map getAlarmRecoveryRecord(List uuids = msgs.stream().map(AlarmMessage::getUuid).collect(Collectors.toList()); + // A recovery is stored in the time bucket of its alarm's start, and an alarm read starts up to the + // duration's end instant. long startTB = duration.getStartTimeBucketInSec(); - long endTB = duration.getEndTimeBucketInSec(); + long endTB = TimeBucket.getRecordTimeBucket(duration.getEndTimestamp()); final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(AlarmRecoveryRecord.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); @@ -211,7 +214,7 @@ private Map getAlarmRecoveryRecord(List queryTasksByServices(List serviceId @Override public List queryTasksByTargets(String serviceId, String serviceInstanceId, List targetTypes, EBPFProfilingTriggerType triggerType, long taskStartTime, long latestUpdateTime) throws IOException { + // No target type asked for (a continuous-profiling policy saved without targets): no task. + if (CollectionUtils.isEmpty(targetTypes)) { + return new ArrayList<>(); + } final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(EBPFProfilingTaskRecord.INDEX_NAME); final BoolQueryBuilder query = Query.bool(); @@ -96,10 +101,8 @@ public List queryTasksByTargets(String serviceId, Strin if (StringUtil.isNotEmpty(serviceInstanceId)) { query.must(Query.term(EBPFProfilingTaskRecord.INSTANCE_ID, serviceInstanceId)); } - if (CollectionUtils.isNotEmpty(targetTypes)) { - query.must(Query.terms(EBPFProfilingTaskRecord.TARGET_TYPE, targetTypes.stream() - .map(EBPFProfilingTargetType::value).collect(Collectors.toList()))); - } + query.must(Query.terms(EBPFProfilingTaskRecord.TARGET_TYPE, targetTypes.stream() + .map(EBPFProfilingTargetType::value).collect(Collectors.toList()))); if (triggerType != null) { query.must(Query.term(EBPFProfilingTaskRecord.TRIGGER_TYPE, triggerType.value())); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java index 5f255e498c41..d00c19098629 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ESEventQueryDAO.java @@ -38,7 +38,6 @@ import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.MatchCNameBuilder; import java.io.IOException; import java.util.List; @@ -95,10 +94,7 @@ private void buildMustQueryListByCondition(final EventQueryCondition condition, query.must(Query.term(Event.SERVICE_INSTANCE, source.getServiceInstance())); } if (!isNullOrEmpty(source.getEndpoint())) { - query.must(Query.matchPhrase( - MatchCNameBuilder.INSTANCE.build(Event.ENDPOINT), - source.getEndpoint() - )); + query.must(Query.term(Event.ENDPOINT, source.getEndpoint())); } } @@ -113,10 +109,10 @@ private void buildMustQueryListByCondition(final EventQueryCondition condition, final Duration startTime = condition.getTime(); if (startTime != null) { if (startTime.getStartTimestamp() > 0) { - query.must(Query.range(Event.TIMESTAMP).gt(startTime.getStartTimestamp())); + query.must(Query.range(Event.TIMESTAMP).gte(startTime.getStartTimestamp())); } if (startTime.getEndTimestamp() > 0) { - query.must(Query.range(Event.TIMESTAMP).lt(startTime.getEndTimestamp())); + query.must(Query.range(Event.TIMESTAMP).lte(startTime.getEndTimestamp())); } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java index eea0e11c327c..3da3e4524407 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationRecordQueryEsDAO.java @@ -27,7 +27,6 @@ import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; -import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.query.enumeration.GenAIEvaluationRecordSortBy; import org.apache.skywalking.oap.server.core.query.enumeration.GenAIEvaluationValueType; import org.apache.skywalking.oap.server.core.query.enumeration.Order; @@ -62,13 +61,6 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, final int from, final int limit, final Duration duration) throws IOException { - long startSecondTB = 0; - long endSecondTB = 0; - if (nonNull(duration)) { - startSecondTB = duration.getStartTimeBucketInSec(); - endSecondTB = duration.getEndTimeBucketInSec(); - } - final BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(GenAIEvaluationRecord.INDEX_NAME)) { query.must(Query.term( @@ -76,8 +68,10 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, GenAIEvaluationRecord.INDEX_NAME )); } - if (startSecondTB != 0 && endSecondTB != 0) { - query.must(Query.range(Record.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); + if (nonNull(duration)) { + query.must(Query.range(GenAIEvaluationRecord.EVALUATION_TIME) + .gte(duration.getStartTimestamp()) + .lte(duration.getEndTimestamp())); } if (isNotEmpty(serviceId)) { query.must(Query.term(GenAIEvaluationRecord.SERVICE_ID, serviceId)); @@ -144,7 +138,8 @@ public GenAIEvaluationRecords queryGenAIEvaluationRecord(final String serviceId, .from(from); final SearchResponse response = searchDebuggable( - IndexNames.rangeIndices(IndexFamily.of(GenAIEvaluationRecord.INDEX_NAME), duration), search.build()); + IndexNames.rangeIndicesThroughEndInstant(IndexFamily.of(GenAIEvaluationRecord.INDEX_NAME), duration), + search.build()); final GenAIEvaluationRecords records = new GenAIEvaluationRecords(); for (SearchHit searchHit : response.getHits().getHits()) { diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/HierarchyQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/HierarchyQueryEsDAO.java index 1119a9379e0d..14db1a63787f 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/HierarchyQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/HierarchyQueryEsDAO.java @@ -18,7 +18,6 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; -import java.util.ArrayList; import java.util.List; import java.util.function.Function; import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; @@ -26,7 +25,6 @@ import org.apache.skywalking.library.elasticsearch.requests.search.Search; import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; -import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.hierarchy.instance.InstanceHierarchyRelationTraffic; import org.apache.skywalking.oap.server.core.hierarchy.service.ServiceHierarchyRelationTraffic; @@ -118,16 +116,16 @@ public List readInstanceHierarchyRelations(fin InstanceHierarchyRelationTraffic.INDEX_NAME )); } - final SearchBuilder search = Search.builder().query(query); - final SearchResponse response = getClient().search(index, search.build()); - return buildInstanceHierarchyRelations(response); - } - - private List buildInstanceHierarchyRelations(SearchResponse response) { - List relations = new ArrayList<>(); - for (SearchHit searchHit : response.getHits()) { - relations.add(instanceRelationsFunction.apply(searchHit)); - } - return relations; + final int batchSize = Math.min(queryMaxSize, scrollingBatchSize); + final SearchBuilder search = Search.builder().query(query).size(batchSize); + final var scroller = ElasticSearchScroller + .builder() + .client(getClient()) + .search(search.build()) + .index(index) + .queryMaxSize(queryMaxSize) + .resultConverter(instanceRelationsFunction) + .build(); + return scroller.scroll(); } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/JFRDataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/JFRDataQueryEsDAO.java index c44ad939f6b4..ac457f3482ea 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/JFRDataQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/JFRDataQueryEsDAO.java @@ -18,16 +18,15 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; -import com.google.common.collect.Lists; import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; -import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.JFRProfilingDataRecord; import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IJFRDataQueryDAO; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchScroller; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; @@ -39,11 +38,20 @@ import java.util.Map; public class JFRDataQueryEsDAO extends EsDAO implements IJFRDataQueryDAO { + /** + * An upload is a frame tree of up to megabytes, which a hit's source holds as Base64, and the client refuses a + * response larger than its limit, Armeria's default of 10 MiB. A page of one upload is as small as a read of that + * upload can be. + */ + private static final int UPLOADS_PER_PAGE = 1; public JFRDataQueryEsDAO(ElasticSearchClient client) { super(client); } + /** + * Reads every upload of the task, as the analysis merges all of them, scrolling one upload per page. + */ @Override public List getByTaskIdAndInstancesAndEvent(String taskId, List instanceIds, String eventType) { if (StringUtil.isBlank(taskId) || StringUtil.isBlank(eventType)) { @@ -60,14 +68,16 @@ public List getByTaskIdAndInstancesAndEvent(String taskI query.must(Query.terms(JFRProfilingDataRecord.INSTANCE_ID, instanceIds)); } final SearchBuilder search = Search.builder() - .query(query); + .query(query) + .size(UPLOADS_PER_PAGE); - final SearchResponse response = getClient().search(index, search.build()); - List dataRecords = Lists.newArrayList(); - for (SearchHit searchHit : response.getHits().getHits()) { - dataRecords.add(parseData(searchHit)); - } - return dataRecords; + return ElasticSearchScroller.builder() + .client(getClient()) + .search(search.build()) + .index(index) + .resultConverter(this::parseData) + .build() + .scroll(); } private JFRProfilingDataRecord parseData(SearchHit data) { diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java index 8acf4b56d3a9..c310827ae7c9 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryEsDAO.java @@ -32,7 +32,6 @@ import org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord; import org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; -import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.query.enumeration.Order; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.query.input.TraceScopeCondition; @@ -73,18 +72,14 @@ public Logs queryLogs(final String serviceId, final List tags, final List keywordsOfContent, final List excludingKeywordsOfContent) throws IOException { - long startSecondTB = 0; - long endSecondTB = 0; - if (nonNull(duration)) { - startSecondTB = duration.getStartTimeBucketInSec(); - endSecondTB = duration.getEndTimeBucketInSec(); - } final BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(LogRecord.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, LogRecord.INDEX_NAME)); } - if (startSecondTB != 0 && endSecondTB != 0) { - query.must(Query.range(Record.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); + if (nonNull(duration)) { + query.must(Query.range(AbstractLogRecord.TIMESTAMP) + .gte(duration.getStartTimestamp()) + .lte(duration.getEndTimestamp())); } if (isNotEmpty(serviceId)) { query.must(Query.term(AbstractLogRecord.SERVICE_ID, serviceId)); @@ -150,7 +145,7 @@ public Logs queryLogs(final String serviceId, .from(from); SearchResponse response = searchDebuggable( - IndexNames.rangeIndices(IndexFamily.of(LogRecord.INDEX_NAME), duration), search.build()); + IndexNames.rangeIndicesThroughEndInstant(IndexFamily.of(LogRecord.INDEX_NAME), duration), search.build()); Logs logs = new Logs(); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java index 5633350f0e4e..9e8e3b2e2ac2 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryEsDAO.java @@ -37,7 +37,6 @@ import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.analysis.IDManager; -import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessDetectType; @@ -154,8 +153,8 @@ public List listInstances(@Nullable Duration duration, Query.bool() .must(Query.term(InstanceTraffic.SERVICE_ID, serviceId)); if (duration != null) { - final long startMinuteTimeBucket = TimeBucket.getMinuteTimeBucket(duration.getStartTimestamp()); - final long endMinuteTimeBucket = TimeBucket.getMinuteTimeBucket(duration.getEndTimestamp()); + final long startMinuteTimeBucket = duration.getStartTimeBucketInMin(); + final long endMinuteTimeBucket = duration.getEndTimeBucketInMin(); query.must(Query.range(InstanceTraffic.LAST_PING_TIME_BUCKET).gte(startMinuteTimeBucket)) .must(Query.range(InstanceTraffic.TIME_BUCKET).lte(endMinuteTimeBucket)); } @@ -231,8 +230,8 @@ public List findEndpoint(String keyword, String serviceId, int limit, } if (duration != null) { - final long startMinuteTimeBucket = TimeBucket.getMinuteTimeBucket(duration.getStartTimestamp()); - final long endMinuteTimeBucket = TimeBucket.getMinuteTimeBucket(duration.getEndTimestamp()); + final long startMinuteTimeBucket = duration.getStartTimeBucketInMin(); + final long endMinuteTimeBucket = duration.getEndTimeBucketInMin(); query.must(Query.range(EndpointTraffic.LAST_PING_TIME_BUCKET).gte(startMinuteTimeBucket)) .must(Query.range(EndpointTraffic.TIME_BUCKET).lte(endMinuteTimeBucket)); } @@ -289,8 +288,9 @@ public List listProcesses(String serviceId, ProfilingSupportStatus supp @Override public List listProcesses(String serviceInstanceId, Duration duration, boolean includeVirtual) { - long lastPingStartTimeBucket = duration.getStartTimeBucket(); - long lastPingEndTimeBucket = duration.getEndTimeBucket(); + // Traffic is stored by minute, whatever the step of the duration. + long lastPingStartTimeBucket = duration.getStartTimeBucketInMin(); + long lastPingEndTimeBucket = duration.getEndTimeBucketInMin(); final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME); @@ -400,8 +400,12 @@ private void appendProcessWhereQuery(BoolQueryBuilder query, String serviceId, S public Process getProcess(String processId) { final String index = IndexController.LogicIndicesRegister.getPhysicalTableName(ProcessTraffic.INDEX_NAME); + String id = processId; + if (IndexController.LogicIndicesRegister.isMergedTable(ProcessTraffic.INDEX_NAME)) { + id = IndexController.INSTANCE.generateDocId(ProcessTraffic.INDEX_NAME, processId); + } final BoolQueryBuilder query = Query.bool() - .must(Query.term("_id", processId)); + .must(Query.term("_id", id)); if (IndexController.LogicIndicesRegister.isMergedTable(ProcessTraffic.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, ProcessTraffic.INDEX_NAME)); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java index 77ea0c55bb7d..0c74f015834e 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/PprofDataQueryEsDAO.java @@ -19,7 +19,6 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; -import com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -28,10 +27,10 @@ import org.apache.skywalking.library.elasticsearch.requests.search.Search; import org.apache.skywalking.library.elasticsearch.requests.search.SearchBuilder; import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; -import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofDataQueryDAO; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchScroller; import org.apache.skywalking.oap.server.library.util.CollectionUtils; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; @@ -39,13 +38,23 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; public class PprofDataQueryEsDAO extends EsDAO implements IPprofDataQueryDAO { + /** + * An upload is a frame tree of up to megabytes, which a hit's source holds as Base64, and the client refuses a + * response larger than its limit, Armeria's default of 10 MiB. A page of one upload is as small as a read of that + * upload can be. + */ + private static final int UPLOADS_PER_PAGE = 1; + public PprofDataQueryEsDAO(ElasticSearchClient client) { super(client); } + /** + * Reads every upload of the task, as the analysis merges all of them, scrolling one upload per page. + */ @Override public List getByTaskIdAndInstances(String taskId, List instanceIds) { - if (StringUtil.isBlank(taskId) || CollectionUtils.isEmpty(instanceIds)) { + if (StringUtil.isBlank(taskId)) { return new ArrayList<>(); } final String index = IndexController.LogicIndicesRegister.getPhysicalTableName( @@ -58,14 +67,18 @@ public List getByTaskIdAndInstances(String taskId, Lis )); } query.must(Query.term(PprofProfilingDataRecord.TASK_ID, taskId)); - query.must(Query.terms(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); - final SearchBuilder search = Search.builder().query(query); - final SearchResponse response = getClient().search(index, search.build()); - List dataRecords = Lists.newArrayList(); - for (SearchHit searchHit : response.getHits().getHits()) { - dataRecords.add(parseData(searchHit)); + if (CollectionUtils.isNotEmpty(instanceIds)) { + query.must(Query.terms(PprofProfilingDataRecord.INSTANCE_ID, instanceIds)); } - return dataRecords; + final SearchBuilder search = Search.builder().query(query).size(UPLOADS_PER_PAGE); + + return ElasticSearchScroller.builder() + .client(getClient()) + .search(search.build()) + .index(index) + .resultConverter(this::parseData) + .build() + .scroll(); } private PprofProfilingDataRecord parseData(SearchHit data) { diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/RecordsQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/RecordsQueryEsDAO.java index ce6663f927ea..56256560fb52 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/RecordsQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/RecordsQueryEsDAO.java @@ -52,9 +52,9 @@ public List readRecords(final RecordCondition condition, final Duration duration) throws IOException { final BoolQueryBuilder query = Query.bool() - .must(Query.range(TopN.TIME_BUCKET) - .gte(duration.getStartTimeBucketInSec()) - .lte(duration.getEndTimeBucketInSec())); + .must(Query.range(TopN.TIMESTAMP) + .gte(duration.getStartTimestamp()) + .lte(duration.getEndTimestamp())); if (IndexController.LogicIndicesRegister.isMergedTable(condition.getName())) { query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, condition.getName())); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/RuntimeRuleManagementEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/RuntimeRuleManagementEsDAO.java index d476e7d178da..fe3b645fe4ab 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/RuntimeRuleManagementEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/RuntimeRuleManagementEsDAO.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.library.elasticsearch.exception.ResponseException; import org.apache.skywalking.library.elasticsearch.requests.search.BoolQueryBuilder; import org.apache.skywalking.library.elasticsearch.requests.search.Query; import org.apache.skywalking.library.elasticsearch.requests.search.Search; @@ -97,7 +98,12 @@ public void delete(final String catalog, final String name) throws IOException { // composite — see RuntimeRule.id() which appends (catalog, name). IndexController // passes the id through to the _id field on insert, so delete by that same id. final String docId = RuntimeRule.INDEX_NAME + "_" + catalog + "_" + name; - getClient().deleteById(index, docId); + // A delete of no rule returns silently: the client reports a missing document instead of throwing. + try { + getClient().deleteById(index, docId); + } catch (ResponseException e) { + throw new IOException("failed to delete runtime rule " + catalog + "/" + name, e); + } } private static String asString(final Object v) { diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java index f99e7a4f84ec..88da56157dd6 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutoCompleteQueryDAO.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; import java.io.IOException; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -31,11 +32,13 @@ import org.apache.skywalking.library.elasticsearch.requests.search.aggregation.Aggregation; import org.apache.skywalking.library.elasticsearch.response.search.SearchHit; import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.storage.query.ITagAutoCompleteQueryDAO; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchScroller; import org.apache.skywalking.oap.server.library.util.StringUtil; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.EsDAO; @@ -52,14 +55,11 @@ public TagAutoCompleteQueryDAO(ElasticSearchClient client) { public Set queryTagAutocompleteKeys(final TagType tagType, final int limit, final Duration duration) throws IOException { - // Tags combine records by day and es rolling index by day, need search the whole day - // just use the time to locate the physical indexes and ignore time conditions. - BoolQueryBuilder query = Query.bool(); - query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name())); - if (IndexController.LogicIndicesRegister.isMergedTable(TagAutocompleteData.INDEX_NAME)) { - query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME)); + // A terms aggregation refuses a size of 0. + if (limit <= 0) { + return Collections.emptySet(); } - final SearchBuilder search = Search.builder().query(query); + final SearchBuilder search = Search.builder().query(tagsOfTheDays(tagType, duration)); search.aggregation(Aggregation.terms(TagAutocompleteData.TAG_KEY) .field(TagAutocompleteData.TAG_KEY) .size(limit)); @@ -85,27 +85,58 @@ public Set queryTagAutocompleteKeys(final TagType tagType, return tagKeys; } + /** + * A value is stored once for each day it is seen, and the value column has no doc values to aggregate on, so one + * value can take several of the documents read. The documents are read page by page until {@code limit} + * distinct values are found or none is left. + */ @Override public Set queryTagAutocompleteValues(final TagType tagType, final String tagKey, final int limit, final Duration duration) throws IOException { - BoolQueryBuilder query = Query.bool().must(Query.term(TagAutocompleteData.TAG_KEY, tagKey)); - query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name())); - if (IndexController.LogicIndicesRegister.isMergedTable(TagAutocompleteData.INDEX_NAME)) { - query.must(Query.term(IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME)); + // A scroll refuses a page size of 0. + if (limit <= 0) { + return Collections.emptySet(); } + final BoolQueryBuilder query = tagsOfTheDays(tagType, duration) + .must(Query.term(TagAutocompleteData.TAG_KEY, tagKey)); final SearchBuilder search = Search.builder().query(query).size(limit); - final SearchResponse response = getClient().search( - IndexNames.rangeIndices(IndexFamily.of(TagAutocompleteData.INDEX_NAME), duration), - search.build() - ); - Set tagValues = new HashSet<>(); - for (SearchHit searchHit : response.getHits().getHits()) { - TagAutocompleteData tag = new TagAutocompleteData.Builder().storage2Entity( - new ElasticSearchConverter.ToEntity(TagAutocompleteData.INDEX_NAME, searchHit.getSource())); - tagValues.add(tag.getTagValue()); + final var scroller = ElasticSearchScroller + .builder() + .client(getClient()) + .search(search.build()) + .indices(IndexNames.rangeIndices(IndexFamily.of(TagAutocompleteData.INDEX_NAME), duration)) + .queryMaxSize(limit) + .distinct(true) + .resultConverter(TagAutoCompleteQueryDAO::tagValue) + .build(); + return new HashSet<>(scroller.scroll()); + } + + /** + * The tags of the type seen on the days from the duration's start through its last step. A tag is stored under + * the first minute of the day it is seen, so the read takes whole days, and an index rolled by more than one day + * holds the tags of other days. The end instant, {@link Duration#getEndTimestamp()}, opens the next day, and + * honoring it would read all of that day's tags. + */ + private static BoolQueryBuilder tagsOfTheDays(final TagType tagType, final Duration duration) { + final BoolQueryBuilder query = Query.bool(); + query.must(Query.term(TagAutocompleteData.TAG_TYPE, tagType.name())); + if (IndexController.LogicIndicesRegister.isMergedTable(TagAutocompleteData.INDEX_NAME)) { + query.must(Query.term( + IndexController.LogicIndicesRegister.METRIC_TABLE_NAME, TagAutocompleteData.INDEX_NAME)); } - return tagValues; + if (duration != null) { + query.must(Query.range(TagAutocompleteData.TIME_BUCKET) + .gte(TimeBucket.retainToDay4MinuteBucket(duration.getStartTimeBucketInMin())) + .lte(TimeBucket.retainToDayLastMin4MinuteBucket(duration.getEndTimeBucketInMin()))); + } + return query; + } + + private static String tagValue(final SearchHit searchHit) { + return new TagAutocompleteData.Builder().storage2Entity( + new ElasticSearchConverter.ToEntity(TagAutocompleteData.INDEX_NAME, searchHit.getSource())).getTagValue(); } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java index a226c265bbf4..f879b82a7e6a 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryEsDAO.java @@ -49,6 +49,11 @@ import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; public class TopologyQueryEsDAO extends EsDAO implements ITopologyQueryDAO { + /** + * How many component values of one relation a read keeps, more than the component library defines. A relation + * stores its components per minute, so a range may hold several, and a terms aggregation keeps 10 by default. + */ + private static final int COMPONENTS_PER_RELATION = 1000; public TopologyQueryEsDAO(ElasticSearchClient client) { super(client); @@ -248,7 +253,8 @@ private List buildProcessRelation(String serviceInstanceId, Aggregation.terms(ProcessRelationServerSideMetrics.COMPONENT_ID) .field(ProcessRelationServerSideMetrics.COMPONENT_ID) .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) - .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST)) + .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST) + .size(COMPONENTS_PER_RELATION)) .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST) .size(1000)); @@ -274,11 +280,13 @@ private List buildProcessRelation(String serviceInstanceId, ProcessRelationServerSideMetrics.COMPONENT_ID); final List> subAgg = (List>) componentTerms.get("buckets"); - final int componentId = ((Number) subAgg.iterator().next().get("key")).intValue(); - - Call.CallDetail call = new Call.CallDetail(); - call.buildProcessRelation(entityId, componentId, detectPoint); - calls.add(call); + // The component is stored per minute, so a relation may name another one in each minute of the range. + for (final Map componentBucket : subAgg) { + final int componentId = ((Number) componentBucket.get("key")).intValue(); + Call.CallDetail call = new Call.CallDetail(); + call.buildProcessRelation(entityId, componentId, detectPoint); + calls.add(call); + } } return calls; } @@ -293,7 +301,8 @@ private List buildServiceRelation(SearchBuilder sourceBuilder, Aggregation.terms(ServiceRelationServerSideMetrics.COMPONENT_IDS) .field(ServiceRelationServerSideMetrics.COMPONENT_IDS) .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) - .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST)) + .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST) + .size(COMPONENTS_PER_RELATION)) .executionHint(TermsAggregationBuilder.ExecutionHint.MAP) .collectMode(TermsAggregationBuilder.CollectMode.BREADTH_FIRST) .size(1000)); @@ -314,12 +323,14 @@ private List buildServiceRelation(SearchBuilder sourceBuilder, ServiceRelationServerSideMetrics.COMPONENT_IDS); final List> subAgg = (List>) componentTerms.get("buckets"); - final IntList componentIds = new IntList((String) subAgg.iterator().next().get("key")); - - for (int i = 0; i < componentIds.size(); i++) { - Call.CallDetail call = new Call.CallDetail(); - call.buildFromServiceRelation(entityId, componentIds.get(i), detectPoint); - calls.add(call); + // The components are stored per minute, so a relation may name others in each minute of the range. + for (final Map componentBucket : subAgg) { + final IntList componentIds = new IntList((String) componentBucket.get("key")); + for (int i = 0; i < componentIds.size(); i++) { + Call.CallDetail call = new Call.CallDetail(); + call.buildFromServiceRelation(entityId, componentIds.get(i), detectPoint); + calls.add(call); + } } } return calls; diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java index d0ddbde2eb1e..31007ad80002 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAO.java @@ -78,19 +78,15 @@ public TraceBrief queryBasicTraces(Duration duration, TraceState traceState, QueryOrder queryOrder, final List tags) throws IOException { - long startSecondTB = 0; - long endSecondTB = 0; - if (nonNull(duration)) { - startSecondTB = duration.getStartTimeBucketInSec(); - endSecondTB = duration.getEndTimeBucketInSec(); - } final BoolQueryBuilder query = Query.bool(); if (IndexController.LogicIndicesRegister.isMergedTable(SegmentRecord.INDEX_NAME)) { query.must(Query.term(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME, SegmentRecord.INDEX_NAME)); } - if (startSecondTB != 0 && endSecondTB != 0) { - query.must(Query.range(SegmentRecord.TIME_BUCKET).gte(startSecondTB).lte(endSecondTB)); + if (nonNull(duration)) { + query.must(Query.range(SegmentRecord.START_TIME) + .gte(duration.getStartTimestamp()) + .lte(duration.getEndTimestamp())); } if (minDuration != 0 || maxDuration != 0) { @@ -150,7 +146,8 @@ public TraceBrief queryBasicTraces(Duration duration, search.size(limit).from(from); final SearchResponse response = searchDebuggable( - IndexNames.rangeIndices(IndexFamily.of(SegmentRecord.INDEX_NAME), duration), search.build()); + IndexNames.rangeIndicesThroughEndInstant(IndexFamily.of(SegmentRecord.INDEX_NAME), duration), + search.build()); final TraceBrief traceBrief = new TraceBrief(); for (SearchHit searchHit : response.getHits().getHits()) { diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/otlp/OTLPTraceQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/otlp/OTLPTraceQueryEsDAO.java index 9cf2b897eb0e..fcbe81953fd5 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/otlp/OTLPTraceQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/otlp/OTLPTraceQueryEsDAO.java @@ -207,7 +207,12 @@ public List> queryTraces(final OTLPTraceQueryCondition conditi } } - // One bucket per trace, ordered by the trace's earliest start or longest span, sized to the page. + // One bucket per trace, ordered by the trace's earliest start or longest span, sized to the page. The earliest + // start, descending, is an ordering Elasticsearch documents as inaccurate across shards: each shard ranks a + // trace only by the spans it holds and returns its top shard_size (by default size * 1.5 + 10). With the + // default routing, by document id, a trace's spans spread over a daily index's shards, so near the cut-off an + // older trace can take the place of a later one; with custom routing a trace's spans share a shard within a + // day, so only a trace crossing midnight can. Accepted for its cost. The longest span, descending, is exact. final boolean byDuration = condition.getQueryOrder() == QueryOrder.BY_DURATION; final String orderAggregation = byDuration ? OTLPSpanRecord.DURATION : OTLPSpanRecord.START_TIME; final TermsAggregationBuilder traceIdAggregation = diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java index bdcb36d5a0f3..c3b72f9fd27a 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAO.java @@ -191,6 +191,12 @@ public List> getTraces(final QueryRequest request, Duration duration) if (request.maxDuration() != null) { query.must(Query.range(ZipkinSpanRecord.DURATION).lte(request.maxDuration())); } + // The traces rank by a terms aggregation on trace_id ordered by the minimum span timestamp, descending, an + // ordering Elasticsearch documents as inaccurate across shards: each shard ranks a trace only by the spans it + // holds and returns its top shard_size (by default size * 1.5 + 10). With the default routing, by document id, + // a trace's spans spread over a daily index's shards, so near the cut-off an older trace can take the place + // of a later one; with custom routing a trace's spans share a shard within a day, so only a trace crossing + // midnight can. Accepted for its cost. final TermsAggregationBuilder traceIdAggregation = Aggregation.terms(ZipkinSpanRecord.TRACE_ID) .field(ZipkinSpanRecord.TRACE_ID) diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ElasticSearchQueryITSupport.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ElasticSearchQueryITSupport.java index 556dc0d249d0..0afaf148b224 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ElasticSearchQueryITSupport.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ElasticSearchQueryITSupport.java @@ -18,35 +18,44 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import java.io.Closeable; import java.io.IOException; import java.net.URI; +import java.net.URLEncoder; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.Supplier; import lombok.Getter; import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.MetricsExtension; import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.management.ManagementData; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; import org.apache.skywalking.oap.server.core.storage.IRecordDAO; import org.apache.skywalking.oap.server.core.storage.StorageException; import org.apache.skywalking.oap.server.core.storage.annotation.Storage; import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller; import org.apache.skywalking.oap.server.core.storage.model.StorageManipulationOpt; import org.apache.skywalking.oap.server.core.storage.model.StorageModels; import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; @@ -54,8 +63,11 @@ import org.apache.skywalking.oap.server.library.client.request.PrepareRequest; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITExt; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.ElasticSearchConverter; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexNames; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.RoutingUtils; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsDAO; import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsInstaller; import org.apache.skywalking.oap.server.telemetry.TelemetryModule; @@ -77,6 +89,10 @@ * {@link StorageModuleElasticsearchProvider} sets it up — the client, the index installer and the bulk writer — for * the ITs that read what OAP's own write path stored back through OAP's query DAOs. * + *

The ITs of every area share one server per version and mode, so each IT writes its indices under a namespace of + * its own, named after its class. A write is refreshed before it returns, as the ITs read right after they write and + * OAP's refresh interval is 30 seconds. + * *

{@link IndexController} and {@link IndexNames} are JVM-wide, so {@link #connect} sets them for the mode of the IT * class connecting, and IT classes must run one after another. */ @@ -95,6 +111,7 @@ public enum Mode { } private final String httpHostAddress; + private final String namespace; @Getter private final StorageModuleElasticsearchConfig config; @Getter @@ -102,16 +119,22 @@ public enum Mode { private final StorageModels models; private final StorageEsDAO storageDAO; private final BatchProcessEsDAO batchDAO; + private final Supplier installers; + /** One client for every request the ITs send the server outside OAP's client: refreshes, stats, shard probes. */ + private final HttpClient http = HttpClient.newHttpClient(); private ElasticSearchQueryITSupport(final String httpHostAddress, final StorageModuleElasticsearchConfig config, final ElasticSearchClient client, final StorageModels models, - final BatchProcessEsDAO batchDAO) { + final BatchProcessEsDAO batchDAO, + final Supplier installers) { this.httpHostAddress = httpHostAddress; + this.namespace = config.getNamespace(); this.config = config; this.client = client; this.models = models; this.storageDAO = new StorageEsDAO(client); this.batchDAO = batchDAO; + this.installers = installers; } public static ElasticsearchContainer elasticsearch(final String version) { @@ -137,15 +160,20 @@ public static ElasticsearchContainer openSearch(final String version) { .withStartupTimeout(Duration.ofMinutes(5)); } - public static ElasticSearchQueryITSupport connect(final ElasticsearchContainer server, - final Mode mode) throws StorageException { - return connect(server, mode, config -> { }); + /** + * @param it the IT class connecting, which names the namespace its indices are written under + */ + public static ElasticSearchQueryITSupport connect(final ElasticsearchContainer server, final Mode mode, + final Class it) throws StorageException { + return connect(server, mode, it, config -> { }); } /** + * @param it the IT class connecting, which names the namespace its indices are written under * @param settings sets what an operator would set in the storage's {@code application.yml} section */ public static ElasticSearchQueryITSupport connect(final ElasticsearchContainer server, final Mode mode, + final Class it, final Consumer settings) throws StorageException { final StorageModuleElasticsearchConfig config = new StorageModuleElasticsearchConfig(); @@ -158,7 +186,7 @@ public static ElasticSearchQueryITSupport connect(final ElasticsearchContainer s settings.accept(config); // What StorageModuleElasticsearchProvider#prepare() and #start() do with the config. - config.setNamespace("sw"); + config.setNamespace(it.getSimpleName().toLowerCase(Locale.ROOT)); IndexNames.configure(config.getDayStep(), config.getSuperDatasetDayStep()); final ModuleManager moduleManager = moduleManager(); final ElasticSearchClient client = new ElasticSearchClient( @@ -172,19 +200,23 @@ public static ElasticSearchQueryITSupport connect(final ElasticsearchContainer s config.getSocketTimeout(), config.getResponseTimeout(), config.getNumHttpClientThread() ); - final StorageEsInstaller installer = new StorageEsInstaller(client, moduleManager, config); + final Supplier installers = () -> { + final StorageEsInstaller installer = new StorageEsInstaller(client, moduleManager, config); + installer.setIndexRefreshInterval(new CoreModuleConfig().getPersistentPeriod() + 5); + installer.start(); + return installer; + }; final BatchProcessEsDAO batchDAO = new BatchProcessEsDAO( client, config.getBulkActions(), config.getFlushInterval(), config.getConcurrentRequests(), config.getBatchOfBytes()); IndexController.INSTANCE.setLogicSharding(config.isLogicSharding()); IndexController.INSTANCE.setEnableCustomRouting(config.isEnableCustomRouting()); client.connect(); - installer.setIndexRefreshInterval(new CoreModuleConfig().getPersistentPeriod() + 5); - installer.start(); final StorageModels models = new StorageModels(); - models.addModelListener(installer); - return new ElasticSearchQueryITSupport(server.getHttpHostAddress(), config, client, models, batchDAO); + models.addModelListener(installers.get()); + return new ElasticSearchQueryITSupport( + server.getHttpHostAddress(), config, client, models, batchDAO, installers); } /** @@ -205,11 +237,60 @@ public Model installMetrics(final Class metricsClass) throws return install(metricsClass, stream.scopeId(), new Storage(stream.name(), timeRelativeID, DownSampling.Minute)); } + /** + * Installs the index template of a metrics class at one of its downsamplings, the way OAP does at boot. + */ + public Model installMetrics(final Class metricsClass, + final DownSampling downSampling) throws StorageException { + final Stream stream = metricsClass.getAnnotation(Stream.class); + final MetricsExtension extension = metricsClass.getAnnotation(MetricsExtension.class); + final boolean timeRelativeID = extension == null || extension.timeRelativeID(); + return install(metricsClass, stream.scopeId(), new Storage(stream.name(), timeRelativeID, downSampling)); + } + + /** + * Installs the minute index template of a MAL meter the way OAP does at boot. The meter's class is its function's + * class, which names neither the metric nor its scope. + */ + public Model installMeter(final Class meterClass, final int scopeId, + final String name) throws StorageException { + return install(meterClass, scopeId, new Storage(name, true, DownSampling.Minute)); + } + + /** + * Installs the index template of a none-stream class (a task) the way {@link NoneStreamProcessor} does at boot. + */ + public Model installNoneStream(final Class noneStreamClass) throws StorageException { + final Stream stream = noneStreamClass.getAnnotation(Stream.class); + return install(noneStreamClass, stream.scopeId(), new Storage(stream.name(), false, DownSampling.Minute)); + } + + /** + * Installs a management model the way OAP does at boot, into the one index all management data shares. + */ + public Model installManagement(final Class managementClass) throws StorageException { + final Stream stream = managementClass.getAnnotation(Stream.class); + return install(managementClass, stream.scopeId(), new Storage(stream.name(), false, DownSampling.None)); + } + + /** + * Installs the model, then checks that a new installer, with nothing cached, finds it installed, as the next + * boot's does: a stored template or index that reads back differently from what the installer writes is + * rewritten on every boot. + */ private Model install(final Class storageClass, final int scopeId, final Storage storage) throws StorageException { + final Model model; try (MockedStatic ignored = mockStatic(DefaultScopeDefine.class)) { - return models.add(storageClass, scopeId, storage, StorageManipulationOpt.schemaCreateIfAbsent()); + model = models.add(storageClass, scopeId, storage, StorageManipulationOpt.schemaCreateIfAbsent()); + } + final ModelInstaller.InstallInfo installed = + installers.get().isExists(model, StorageManipulationOpt.schemaCreateIfAbsent()); + if (!installed.isAllExist()) { + throw new IllegalStateException( + "the installer does not find what it installed: " + installed.buildInstallInfoMsg()); } + return model; } /** @@ -238,6 +319,30 @@ public void writeMetrics(final Model model, final StorageBuilder builder, + final List metrics) throws Exception { + final IMetricsDAO metricsDAO = storageDAO.newMetricsDao(builder); + final List requests = new ArrayList<>(metrics.size()); + for (final Metrics metric : metrics) { + requests.add(metricsDAO.prepareBatchUpdate(model, metric, null)); + } + flush(requests); + } + + /** + * Writes a none-stream row (a task) the way {@link NoneStreamProcessor} does: at once, through OAP's own + * none-stream DAO, outside the bulk, into the index of the row's time bucket. + */ + public void writeNoneStream(final Model model, final StorageBuilder builder, + final NoneStream row) throws IOException, InterruptedException { + storageDAO.newNoneStreamDao(builder).insert(model, row); + refresh(); + } + /** * Sends the requests the way the persistence timer does: queued into the bulk, which {@code endOfFlush} sends. */ @@ -245,6 +350,21 @@ private void flush(final List requests) throws Exception { final CompletableFuture written = batchDAO.flush(requests); batchDAO.endOfFlush(); written.get(30, TimeUnit.SECONDS); + refresh(); + } + + /** + * Makes every write to this IT's indices searchable now, rather than at their next refresh. + */ + public void refresh() throws IOException, InterruptedException { + final HttpResponse response = http.send( + HttpRequest.newBuilder(URI.create("http://" + httpHostAddress + "/" + namespace + "_*/_refresh")) + .POST(HttpRequest.BodyPublishers.noBody()) + .build(), + HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + throw new IOException("refresh " + namespace + "_*: " + response.statusCode() + " " + response.body()); + } } /** @@ -252,7 +372,7 @@ private void flush(final List requests) throws Exception { * keep-alive ends, and a node refuses every new scroll past {@code search.max_open_scroll_context}. */ public long openScrollContexts() throws IOException, InterruptedException { - final HttpResponse response = HttpClient.newHttpClient().send( + final HttpResponse response = http.send( HttpRequest.newBuilder(URI.create("http://" + httpHostAddress + "/_nodes/stats/indices/search")).build(), HttpResponse.BodyHandlers.ofString()); long open = 0; @@ -266,8 +386,69 @@ public long openScrollContexts() throws IOException, InterruptedException { return open; } + /** + * The primary shard the server routes a record to, asked of the server ({@code _search_shards}) with the routing + * OAP's record DAO writes the record with: the model's routing column under custom routing, the document id + * otherwise. The record's index must exist. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public int shardOf(final Model model, final StorageBuilder builder, + final Record record) throws IOException, InterruptedException { + final ElasticSearchConverter.ToStorage columns = new ElasticSearchConverter.ToStorage(model.getName()); + ((StorageBuilder) (StorageBuilder) builder).entity2Storage(record, columns); + final String routing = RoutingUtils.getRoutingValue(model, columns) + .orElse(IndexController.INSTANCE.generateDocId(model, record.id().build())); + final String index = namespace + "_" + IndexNames.writeIndex(model, record.getTimeBucket()); + final HttpResponse response = http.send( + HttpRequest.newBuilder(URI.create("http://" + httpHostAddress + "/" + index + "/_search_shards?routing=" + + URLEncoder.encode(routing, StandardCharsets.UTF_8))).build(), + HttpResponse.BodyHandlers.ofString()); + if (response.statusCode() != 200) { + throw new IOException("search shards of " + index + ": " + response.statusCode() + " " + response.body()); + } + for (final JsonElement copy : JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonArray("shards").get(0).getAsJsonArray()) { + if (copy.getAsJsonObject().get("primary").getAsBoolean()) { + return copy.getAsJsonObject().get("shard").getAsInt(); + } + } + throw new IOException("no primary shard of " + index + " for " + routing + ": " + response.body()); + } + + /** + * The shard of its index that holds a written record, as a search of its document id reports it. + */ + public int shardHolding(final Model model, final Record record) throws IOException, InterruptedException { + final String index = namespace + "_" + IndexNames.writeIndex(model, record.getTimeBucket()); + final JsonObject body = new JsonObject(); + body.addProperty("explain", true); + body.addProperty("_source", false); + final JsonObject ids = new JsonObject(); + final JsonArray values = new JsonArray(); + values.add(IndexController.INSTANCE.generateDocId(model, record.id().build())); + ids.add("values", values); + final JsonObject query = new JsonObject(); + query.add("ids", ids); + body.add("query", query); + final HttpResponse response = http.send( + HttpRequest.newBuilder(URI.create("http://" + httpHostAddress + "/" + index + "/_search")) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(body.toString())) + .build(), + HttpResponse.BodyHandlers.ofString()); + final JsonArray hits = JsonParser.parseString(response.body()).getAsJsonObject() + .getAsJsonObject("hits").getAsJsonArray("hits"); + if (response.statusCode() != 200 || hits.size() != 1) { + throw new IOException("search " + index + " by id: " + response.statusCode() + " " + response.body()); + } + // "[index][shard]" + final String shard = hits.get(0).getAsJsonObject().get("_shard").getAsString(); + return Integer.parseInt(shard.substring(shard.lastIndexOf('[') + 1, shard.length() - 1)); + } + @Override public void close() { + BatchProcessEsDAOITExt.close(batchDAO); client.shutdown(); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/BatchProcessEsDAOITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/BatchProcessEsDAOITBase.java new file mode 100644 index 000000000000..a84caa916663 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/BatchProcessEsDAOITBase.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.management.ui.template.UITemplate; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingDataRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTargetType; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTriggerType; +import org.apache.skywalking.oap.server.core.query.input.DashboardSetting; +import org.apache.skywalking.oap.server.core.query.type.DashboardConfiguration; +import org.apache.skywalking.oap.server.core.storage.IRecordDAO; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.EBPFProfilingDataEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.EBPFProfilingTaskEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.UITemplateManagementEsDAO; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +/** + * Writes through OAP's batch DAO and the write DAOs the storage DAO hands out, built as the storage provider builds + * them, the way OAP's workers use them, and reads each write back through its query DAO: records one by one through + * {@code insert}, as a record worker hands them over, which the bulk sends on its own flush interval; a task row at + * once through the none-stream DAO, as a task worker writes it, readable as soon as the write returns; and a + * management row through the management DAO, which inserts a row only when its id is new. The records lie on both + * sides of midnight, so in two daily indices. A nested class of a server's IT names the server and the storage mode. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class BatchProcessEsDAOITBase { + /** A fixed local midnight, so the daily indices the records are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final int ROWS = 10; + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private BatchProcessEsDAO batchDAO; + private StorageDAO storageDAO; + private Model dataModel; + private Model taskModel; + private Model templateModel; + + protected BatchProcessEsDAOITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void install() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + dataModel = support.installRecord(EBPFProfilingDataRecord.class); + taskModel = support.installNoneStream(EBPFProfilingTaskRecord.class); + templateModel = support.installManagement(UITemplate.class); + final StorageModuleElasticsearchConfig config = support.getConfig(); + batchDAO = new BatchProcessEsDAO( + support.getClient(), config.getBulkActions(), config.getFlushInterval(), config.getConcurrentRequests(), + config.getBatchOfBytes()); + storageDAO = new StorageEsDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (batchDAO != null) { + BatchProcessEsDAOITExt.close(batchDAO); + } + if (support != null) { + support.close(); + } + } + + @Test + public void insertsRecordsThatTheBulkSendsOnItsFlushInterval() throws Exception { + final List rows = new ArrayList<>(); + for (int i = 0; i < ROWS; i++) { + final long uploadTime = MIDNIGHT + (i - ROWS / 2) * 1_000L; + final EBPFProfilingDataRecord row = new EBPFProfilingDataRecord(); + row.setScheduleId("insert-schedule"); + row.setTaskId("batch-it-task"); + row.setStackIdList("stack-" + i); + row.setTargetType(EBPFProfilingTargetType.ON_CPU.value()); + row.setDataBinary(("stack " + i).getBytes()); + row.setUploadTime(uploadTime); + row.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime)); + rows.add(row); + } + final IRecordDAO recordDAO = storageDAO.newRecordDao(new EBPFProfilingDataRecord.Builder()); + for (final EBPFProfilingDataRecord row : rows) { + batchDAO.insert(recordDAO.prepareBatchInsert(dataModel, row)); + } + + final EBPFProfilingDataEsDAO dao = new EBPFProfilingDataEsDAO(support.getClient(), support.getConfig()); + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { + support.refresh(); + assertEquals(describe(rows), describe(dao.queryData( + Collections.singletonList("insert-schedule"), MIDNIGHT - 10_000L, MIDNIGHT + 10_000L))); + }); + } + + /** + * A task is created at once, so the profiling page lists it right after. + */ + @Test + public void writesATaskRowAtOnceThroughTheNoneStreamDAO() throws Exception { + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(IDManager.ServiceID.buildId("batch-it", true)); + task.setProcessLabelsJson(Const.EMPTY_STRING); + task.setStartTime(MIDNIGHT); + task.setTriggerType(EBPFProfilingTriggerType.FIXED_TIME.value()); + task.setFixedTriggerDuration(60); + task.setTargetType(EBPFProfilingTargetType.OFF_CPU.value()); + task.setCreateTime(MIDNIGHT); + task.setLastUpdateTime(MIDNIGHT); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(MIDNIGHT)); + task.generateLogicalId(); + task.setExtensionConfigJson(Const.EMPTY_STRING); + + storageDAO.newNoneStreamDao(new EBPFProfilingTaskRecord.Builder()).insert(taskModel, task); + + assertEquals( + Collections.singletonList(describe(task)), + new EBPFProfilingTaskEsDAO(support.getClient(), support.getConfig()) + .getTaskRecord(task.getLogicalId()).stream() + .map(BatchProcessEsDAOITBase::describe) + .collect(Collectors.toList())); + } + + @Test + public void insertsAManagementRowOnlyWhenItsIdIsNew() throws Exception { + final UITemplate first = template("batch-it-template", "{\"version\":1}"); + final UITemplate second = template("batch-it-template", "{\"version\":2}"); + final UITemplate other = template("batch-it-other-template", "{\"version\":1}"); + for (final UITemplate template : new UITemplate[] {first, second, other}) { + storageDAO.newManagementDao(new UITemplate.Builder()).insert(templateModel, template); + } + + final UITemplateManagementEsDAO dao = new UITemplateManagementEsDAO(support.getClient(), new UITemplate.Builder()); + assertEquals("batch-it-template|{\"version\":1}|false", describe(dao.getTemplate("batch-it-template"))); + assertEquals("batch-it-other-template|{\"version\":1}|false", + describe(dao.getTemplate("batch-it-other-template"))); + } + + private static List describe(final List rows) { + return rows.stream() + .map(row -> String.join("|", row.getScheduleId(), row.getTaskId(), row.getStackIdList(), + String.valueOf(row.getTargetType()), + Base64.getEncoder().encodeToString(row.getDataBinary()), + String.valueOf(row.getUploadTime()))) + .sorted() + .collect(Collectors.toList()); + } + + private static String describe(final EBPFProfilingTaskRecord task) { + final HashMapConverter.ToStorage columns = new HashMapConverter.ToStorage(); + new EBPFProfilingTaskRecord.Builder().entity2Storage(task, columns); + return new TreeMap<>(columns.obtain()).toString(); + } + + private static String describe(final DashboardConfiguration template) { + return template.getId() + "|" + template.getConfiguration() + "|" + template.isDisabled(); + } + + private static UITemplate template(final String id, final String configuration) { + final DashboardSetting setting = new DashboardSetting(); + setting.setId(id); + setting.setConfiguration(configuration); + return setting.toEntity(); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/BatchProcessEsDAOITExt.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/BatchProcessEsDAOITExt.java new file mode 100644 index 000000000000..efa18ae54555 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/BatchProcessEsDAOITExt.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base; + +import java.lang.reflect.Field; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.library.elasticsearch.bulk.BulkProcessor; + +/** + * Closes the bulk processor of a {@link BatchProcessEsDAO} for the ITs, which OAP never does: its periodical flush runs + * as long as OAP. An IT closes it before it shuts the client down, so no flush thread outlives the test. The client + * module's {@code BulkProcessorITExt} does the same for a processor of its own; test classes are not shared between + * modules. + */ +public final class BatchProcessEsDAOITExt { + private BatchProcessEsDAOITExt() { + } + + /** + * Stops the periodical flush, lets a flush already running finish its bulk, and sends what is still queued. + */ + public static void close(final BatchProcessEsDAO dao) { + final BulkProcessor processor = (BulkProcessor) field(BatchProcessEsDAO.class, "bulkProcessor", dao); + if (processor == null) { + return; + } + final ScheduledThreadPoolExecutor scheduler = + (ScheduledThreadPoolExecutor) field(BulkProcessor.class, "scheduler", processor); + scheduler.shutdown(); + try { + if (!scheduler.awaitTermination(1, TimeUnit.MINUTES)) { + throw new IllegalStateException("The periodical flush did not finish in a minute"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while the periodical flush finished", e); + } + processor.flush(); + } + + private static Object field(final Class type, final String name, final Object target) { + try { + final Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(type.getSimpleName() + " has no field named " + name, e); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAOITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAOITBase.java new file mode 100644 index 000000000000..b877619ef6df --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAOITBase.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.IntList; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingDataRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTargetType; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTriggerType; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.LocalDate; +import org.joda.time.LocalTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Deletes the history of a model the way the TTL timer does, through {@link HistoryDeleteEsDAO}: the daily indices of + * the model's physical table whose day is past the TTL go, the others stay. A nested class of a server's IT names the + * server and the storage mode. + * + *

Two records, eBPF profiling tasks and two metrics are written on the last day an index is deleted on, the + * first day one is kept on, a day well past the TTL, and today, before each test, so every test starts from every + * index. The tasks are written the way the task workers write them, with the minute they are created in. In OAP's + * default mode the records and the tasks share one physical table and the two metrics another, so deleting the + * history of one model deletes that of the others; with logic sharding each model has a table of its own, which the + * history of another model must not touch. Metrics of every downsampling share their indices, and only the minute + * model deletes them. A table cleaned to a deadline is not cleaned again for that deadline, by any of its models. + * An index that earlier versions named by an hour goes with the indices of its day. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class HistoryDeleteEsDAOITBase { + private static final int RECORD_TTL_DAYS = 3; + private static final int METRICS_TTL_DAYS = 7; + private static final List RECORD_DAYS_AGO = Arrays.asList(5, RECORD_TTL_DAYS, RECORD_TTL_DAYS - 1, 0); + private static final List METRICS_DAYS_AGO = Arrays.asList(9, METRICS_TTL_DAYS, METRICS_TTL_DAYS - 1, 0); + private static final String SOURCE_ID = IDManager.ServiceID.buildId("checkout", true); + private static final String DEST_ID = IDManager.ServiceID.buildId("payment", true); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private Model ebpfData; + private Model pprofData; + private Model ebpfTasks; + private Model serverRelation; + private Model clientRelation; + private Model serverRelationHour; + private Model asyncProfilerTasks; + private LocalDate today; + + protected HistoryDeleteEsDAOITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void install() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + ebpfData = support.installRecord(EBPFProfilingDataRecord.class); + pprofData = support.installRecord(PprofProfilingDataRecord.class); + ebpfTasks = support.installNoneStream(EBPFProfilingTaskRecord.class); + serverRelation = support.installMetrics(ServiceRelationServerSideMetrics.class); + serverRelationHour = support.installMetrics(ServiceRelationServerSideMetrics.class, DownSampling.Hour); + clientRelation = support.installMetrics(ServiceRelationClientSideMetrics.class); + asyncProfilerTasks = support.installNoneStream(AsyncProfilerTaskRecord.class); + } + + @BeforeEach + public void writeEveryDay() throws Exception { + today = LocalDate.now(); + final List ebpfRows = new ArrayList<>(); + final List pprofRows = new ArrayList<>(); + for (final int daysAgo : RECORD_DAYS_AGO) { + final long time = daysAgo(daysAgo); + final EBPFProfilingDataRecord ebpfRow = new EBPFProfilingDataRecord(); + ebpfRow.setScheduleId("schedule"); + ebpfRow.setTaskId("task"); + ebpfRow.setStackIdList("stack-" + daysAgo); + ebpfRow.setDataBinary(new byte[] {1}); + ebpfRow.setUploadTime(time); + ebpfRow.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + ebpfRows.add(ebpfRow); + final PprofProfilingDataRecord pprofRow = new PprofProfilingDataRecord(); + pprofRow.setTaskId("task-" + daysAgo); + pprofRow.setInstanceId("instance"); + pprofRow.setDataBinary(new byte[] {1}); + pprofRow.setUploadTime(time); + pprofRow.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + pprofRows.add(pprofRow); + } + support.writeRecords(ebpfData, new EBPFProfilingDataRecord.Builder(), ebpfRows); + support.writeRecords(pprofData, new PprofProfilingDataRecord.Builder(), pprofRows); + for (final int daysAgo : RECORD_DAYS_AGO) { + final long time = daysAgo(daysAgo); + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(SOURCE_ID); + task.setProcessLabelsJson(Const.EMPTY_STRING); + task.setStartTime(time); + task.setTriggerType(EBPFProfilingTriggerType.FIXED_TIME.value()); + task.setFixedTriggerDuration(60); + task.setTargetType(EBPFProfilingTargetType.ON_CPU.value()); + task.setCreateTime(time); + task.setLastUpdateTime(time); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(time)); + task.generateLogicalId(); + task.setExtensionConfigJson(Const.EMPTY_STRING); + support.writeNoneStream(ebpfTasks, new EBPFProfilingTaskRecord.Builder(), task); + } + + final List serverRows = new ArrayList<>(); + final List clientRows = new ArrayList<>(); + for (final int daysAgo : METRICS_DAYS_AGO) { + final long minute = TimeBucket.getMinuteTimeBucket(daysAgo(daysAgo)); + final ServiceRelationServerSideMetrics serverRow = new ServiceRelationServerSideMetrics(); + serverRow.setSourceServiceId(SOURCE_ID); + serverRow.setDestServiceId(DEST_ID); + serverRow.setEntityId(relationId()); + serverRow.setComponentIds(new IntList(1)); + serverRow.setTimeBucket(minute); + serverRows.add(serverRow); + final ServiceRelationClientSideMetrics clientRow = new ServiceRelationClientSideMetrics(); + clientRow.setSourceServiceId(SOURCE_ID); + clientRow.setDestServiceId(DEST_ID); + clientRow.setEntityId(relationId()); + clientRow.setComponentIds(new IntList(1)); + clientRow.setTimeBucket(minute); + clientRows.add(clientRow); + } + support.writeMetrics(serverRelation, new ServiceRelationServerSideMetrics.Builder(), serverRows); + support.writeMetrics(clientRelation, new ServiceRelationClientSideMetrics.Builder(), clientRows); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void deletesTheRecordIndicesPastTheTTL() throws Exception { + new HistoryDeleteEsDAO(support.getClient()).deleteHistory(ebpfData, Metrics.TIME_BUCKET, RECORD_TTL_DAYS); + + assertEquals(expected(ebpfData, RECORD_TTL_DAYS), indices()); + } + + @Test + public void deletesTheTaskIndicesPastTheTTL() throws Exception { + new HistoryDeleteEsDAO(support.getClient()).deleteHistory(ebpfTasks, Metrics.TIME_BUCKET, RECORD_TTL_DAYS); + + assertEquals(expected(ebpfTasks, RECORD_TTL_DAYS), indices()); + } + + /** + * The TTL timer deletes the history of every model in turn, every few minutes. Once a table is cleaned to a + * deadline, the indices written past the deadline since then wait for the next deadline, whichever model of the + * table comes next; a table of its own is cleaned for its own model. + */ + @Test + public void cleansATableOncePerDeadline() throws Exception { + final HistoryDeleteEsDAO dao = new HistoryDeleteEsDAO(support.getClient()); + dao.deleteHistory(ebpfData, Metrics.TIME_BUCKET, RECORD_TTL_DAYS); + writeEveryDay(); + + dao.deleteHistory(ebpfData, Metrics.TIME_BUCKET, RECORD_TTL_DAYS); + dao.deleteHistory(pprofData, Metrics.TIME_BUCKET, RECORD_TTL_DAYS); + + final boolean sharedTable = + IndexController.INSTANCE.getTableName(pprofData).equals(IndexController.INSTANCE.getTableName(ebpfData)); + assertEquals(expected(sharedTable ? null : pprofData, RECORD_TTL_DAYS), indices()); + } + + @Test + public void deletesTheMetricsIndicesPastTheTTL() throws Exception { + new HistoryDeleteEsDAO(support.getClient()).deleteHistory(serverRelation, Metrics.TIME_BUCKET, METRICS_TTL_DAYS); + + assertEquals(expected(serverRelation, METRICS_TTL_DAYS), indices()); + } + + @Test + public void leavesTheMetricsIndicesToTheMinuteModel() throws Exception { + new HistoryDeleteEsDAO(support.getClient()) + .deleteHistory(serverRelationHour, Metrics.TIME_BUCKET, METRICS_TTL_DAYS); + + assertEquals(expected(null, 0), indices()); + } + + /** + * Earlier versions wrote an async-profiler or pprof task, a second bucket on a minute-downsampled model, into an + * index named by its hour, {@code -}. Such an index goes with the daily index of its day once + * that day is past the TTL, and stays with it while the day is not. The tasks of today's version, written on the + * same days, lie in the daily indices. + */ + @Test + public void deletesTheHourIndicesOfEarlierVersionsWithTheirDay() throws Exception { + final String table = IndexController.INSTANCE.getTableName(asyncProfilerTasks); + final ElasticSearchClient client = support.getClient(); + try { + for (final int daysAgo : Arrays.asList(RECORD_TTL_DAYS, RECORD_TTL_DAYS - 1)) { + final long time = daysAgo(daysAgo); + final AsyncProfilerTaskRecord task = new AsyncProfilerTaskRecord(); + task.setTaskId(time + Const.ID_CONNECTOR + SOURCE_ID); + task.setServiceId(SOURCE_ID); + task.setServiceInstanceIds("[]"); + task.setEvents("[]"); + task.setExecArgs(Const.EMPTY_STRING); + task.setCreateTime(time); + task.setDuration(60); + task.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + support.writeNoneStream(asyncProfilerTasks, new AsyncProfilerTaskRecord.Builder(), task); + client.createIndex(table + Const.LINE + day(daysAgo) + "09"); + } + + new HistoryDeleteEsDAO(client).deleteHistory(asyncProfilerTasks, Metrics.TIME_BUCKET, RECORD_TTL_DAYS); + + final TreeSet indices = new TreeSet<>(client.retrievalIndexByAliases(table)); + assertEquals(Arrays.asList(false, false, true, true), Arrays.asList( + indices.contains(client.formatIndexName(table + Const.LINE + day(RECORD_TTL_DAYS) + "09")), + indices.contains(client.formatIndexName(table + Const.LINE + day(RECORD_TTL_DAYS))), + indices.contains(client.formatIndexName(table + Const.LINE + day(RECORD_TTL_DAYS - 1) + "09")), + indices.contains(client.formatIndexName(table + Const.LINE + day(RECORD_TTL_DAYS - 1)))), + indices.toString()); + } finally { + // The other tests share the table in the default mode and expect daily indices only. + for (final String index : client.retrievalIndexByAliases(table)) { + if (index.substring(index.lastIndexOf(Const.LINE) + 1).length() == 10) { + client.deleteByIndexName(index); + } + } + } + } + + private String day(final int daysAgo) { + return String.valueOf(TimeBucket.getTimeBucket(daysAgo(daysAgo), DownSampling.Day)); + } + + /** + * Noon of the local calendar day this many days before today. The DAO counts the TTL in calendar days, which are + * not all 24 hours long, and noon is clear of any daylight-saving change. + */ + private long daysAgo(final int days) { + return today.minusDays(days).toDateTime(new LocalTime(12, 0)).getMillis(); + } + + private static String relationId() { + return IDManager.ServiceID.buildRelationId(new IDManager.ServiceID.ServiceRelationDefine(SOURCE_ID, DEST_ID)); + } + + /** + * The indices of each physical table, by its alias. + */ + private Map> indices() { + final Map> indices = new TreeMap<>(); + for (final Model model : Arrays.asList(ebpfData, pprofData, ebpfTasks, serverRelation, clientRelation)) { + final String table = IndexController.INSTANCE.getTableName(model); + indices.put(table, new TreeSet<>(support.getClient().retrievalIndexByAliases(table))); + } + return indices; + } + + /** + * The indices of each physical table: a daily index of every day written, today's included, less the indices + * past the TTL of the cleaned model's table. + */ + private Map> expected(final Model cleaned, final int ttl) { + final Map> indices = new TreeMap<>(); + for (final Model model : Arrays.asList(ebpfData, pprofData, ebpfTasks, serverRelation, clientRelation)) { + final String table = IndexController.INSTANCE.getTableName(model); + final TreeSet tableIndices = indices.computeIfAbsent(table, t -> new TreeSet<>()); + final boolean isCleaned = cleaned != null && table.equals(IndexController.INSTANCE.getTableName(cleaned)); + for (final int daysAgo : model.isRecord() ? RECORD_DAYS_AGO : METRICS_DAYS_AGO) { + if (isCleaned && daysAgo >= ttl) { + continue; + } + tableIndices.add(support.getClient().formatIndexName( + IndexNames.latestIndex(IndexFamily.of(model), daysAgo(daysAgo)))); + } + } + return indices; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexNamesTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexNamesTest.java index 6be8cf7d0ccc..54a02922d194 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexNamesTest.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/IndexNamesTest.java @@ -99,6 +99,21 @@ void writeIndex_rollsMetricsAndRecordsByDayStepAndSuperDatasetsBySuperDatasetDay assertEquals("metrics-all-20200810", IndexNames.writeIndex(metrics, 2020_0810_1010L)); } + /** + * The async-profiler and pprof tasks carry second buckets on a minute-downsampled model. Read as a minute bucket, + * such a bucket named an hourly index, or failed to name one with a day step above one. + */ + @Test + void writeIndex_takesTheDayOfTheBucketAtItsOwnPrecision() { + final Model task = model("async_profiler_task", DownSampling.Minute, false, Record.class); + + assertEquals("records-all-20200809", IndexNames.writeIndex(task, 2020_0809_2359_59L)); + assertEquals("records-all-20200809", IndexNames.writeIndex(task, 2020_0809_2359L)); + assertEquals("records-all-20200810", IndexNames.writeIndex(task, 2020_0810_0000_00L)); + IndexNames.configure(3, 1); + assertEquals("records-all-20200807", IndexNames.writeIndex(task, 2020_0809_2359_59L)); + } + @Test void writeIndex_ofManagementData_isTheTableWithoutADate() { final Model management = model("ui_template", DownSampling.None, false, Record.class); @@ -145,6 +160,23 @@ SEGMENT, new DateTime(2020, 8, 8, 12, 0).getMillis(), new DateTime(2020, 8, 10, ); } + /** + * A duration ending in a day's last minute ends at the next midnight, so the next day's index holds its end + * instant. + */ + @Test + void rangeIndicesThroughEndInstant_namesTheDayOfTheEndInstant() { + assertArrayEquals( + new String[] {"segment-20200604", "segment-20200605"}, + IndexNames.rangeIndicesThroughEndInstant(SEGMENT, duration("2020-06-04 1400", "2020-06-04 2359")) + ); + assertArrayEquals( + new String[] {"segment-20200604"}, + IndexNames.rangeIndicesThroughEndInstant(SEGMENT, duration("2020-06-04 1400", "2020-06-04 2358")) + ); + assertArrayEquals(new String[] {"segment"}, IndexNames.rangeIndicesThroughEndInstant(SEGMENT, null)); + } + @Test void latestIndex_isTodaysIndex() { IndexNames.configure(3, 1); @@ -161,6 +193,16 @@ void indexStartDay_isTheDateSuffix() { assertEquals(20190602L, IndexNames.indexStartDay("namespace_metrics-all-20190602")); } + /** + * Earlier versions named the index of a task with a second bucket on a minute-downsampled model by its hour. + */ + @Test + void indexStartDay_ofAnIndexNamedByAnHour_isTheHoursDay() { + assertEquals(20190602L, IndexNames.indexStartDay("namespace_records-all-2019060223")); + assertEquals(20190602L, IndexNames.indexStartDay("async_profiler_task-2019060200")); + assertEquals(20190602L, IndexNames.indexStartDay("namespace_records-all-20190602")); + } + private static Model model(final String name, final DownSampling downSampling, final boolean superDataset, diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstallerTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstallerTest.java index 422f6e4ff457..b43e2c4a65d7 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstallerTest.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/StorageEsInstallerTest.java @@ -19,6 +19,7 @@ package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base; import java.util.Collections; +import org.apache.skywalking.library.elasticsearch.response.IndexTemplate; import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.storage.model.BanyanDBModelExtension; @@ -35,6 +36,8 @@ import org.mockito.junit.jupiter.MockitoExtension; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.verifyNoInteractions; @ExtendWith(MockitoExtension.class) @@ -63,4 +66,28 @@ void whenCreating_withoutSchemaChange_registersThePhysicalTableWithoutCallingEla ); verifyNoInteractions(client); } + + /** + * A legacy template (Elasticsearch before 7.8) keeps a negative order, so one stored with 0 is updated to the + * configured -1. A composable template keeps a negative order as priority 0, which is then up to date. + */ + @Test + void comparesALegacyTemplateOrderAsConfiguredAndAComposablePriorityAtLeastZero() { + assertTrue(StorageEsInstaller.hasConfiguredOrder(template(-1, false), -1)); + assertFalse(StorageEsInstaller.hasConfiguredOrder(template(0, false), -1)); + assertFalse(StorageEsInstaller.hasConfiguredOrder(template(0, false), 3)); + assertTrue(StorageEsInstaller.hasConfiguredOrder(template(3, false), 3)); + + assertTrue(StorageEsInstaller.hasConfiguredOrder(template(0, true), -1)); + assertTrue(StorageEsInstaller.hasConfiguredOrder(template(0, true), 0)); + assertFalse(StorageEsInstaller.hasConfiguredOrder(template(5, true), 0)); + assertTrue(StorageEsInstaller.hasConfiguredOrder(template(5, true), 5)); + } + + private static IndexTemplate template(final long order, final boolean composable) { + final IndexTemplate template = new IndexTemplate(); + template.setOrder(order); + template.setComposable(composable); + return template; + } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AIAgentConversationQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AIAgentConversationQueryITBase.java new file mode 100644 index 000000000000..992eba721526 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AIAgentConversationQueryITBase.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.aiagent.AIAgentSessionDataRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.aiagent.AIAgentSessionFlowRecord; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.query.IAIAgentConversationQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads AI agent conversations the way the conversation query service does on Elasticsearch and OpenSearch, through + * {@link AIAgentConversationQueryEsDAO}: the latest rounds of a service, a sender or a conversation, with or without + * their bodies; the highest round of a conversation; the rounds of a round-number window, every copy of a round two + * senders pushed included; and the files of a session in a seq window and a timestamp range. A nested class of a + * server's IT names the server and the storage mode. + * + *

The rounds and the files are written on both sides of midnight, so a read spans two daily indices, and the + * highest round of the conversation is its oldest row. Each read has a row of another service, sender, conversation + * or session that matches it otherwise, and rows just outside the window. A round pushed by more senders than a window + * makes room for shows the window's limit. Rounds of their own lie on and just outside the bounds of a duration: on + * its first millisecond and the one before, on its last millisecond, on the instant it ends at + * ({@link Duration#getEndTimestamp()}, which a read includes) and the millisecond after. + * One more lies on the end instant of a duration ending in a day's last minute: the next midnight, in the next + * day's index. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class AIAgentConversationQueryITBase { + /** A fixed local midnight, so the daily indices the rows are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final long MINUTE = 60_000L; + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + private static final Duration ENDING = minutes(MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE); + /** The midnight a day before {@link #MIDNIGHT}. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = minutes(PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE); + + private static final String SERVICE = "ai-agent"; + private static final String OTHER_SERVICE = "other-agent"; + private static final String CONVERSATION = "conversation-read"; + private static final String SESSION = "session-read"; + /** More senders than the window of one round makes room for. */ + private static final int COPIES = IAIAgentConversationQueryDAO.MAX_COPIES + 1; + + /** Round 5 was pushed first, so the highest round is not the newest row. */ + private static final List ROUNDS = Arrays.asList( + new Round(SERVICE, "instance-0", CONVERSATION, 5, MIDNIGHT - 150_000L), + new Round(SERVICE, "instance-0", CONVERSATION, 1, MIDNIGHT - 110_000L), + new Round(SERVICE, "instance-0", CONVERSATION, 2, MIDNIGHT - 70_000L), + new Round(SERVICE, "instance-0", CONVERSATION, 3, MIDNIGHT - 30_000L), + new Round(SERVICE, "instance-1", CONVERSATION, 3, MIDNIGHT + 30_000L), + new Round(SERVICE, "instance-0", CONVERSATION, 4, MIDNIGHT + 70_000L).counted(), + new Round(SERVICE, "instance-0", "conversation-other", 9, MIDNIGHT + 110_000L), + new Round(OTHER_SERVICE, "instance-0", CONVERSATION, 7, MIDNIGHT + 150_000L), + new Round(SERVICE, "instance-0", "conversation-bounds", 1, BOUNDED.getStartTimestamp() - 1), + new Round(SERVICE, "instance-0", "conversation-bounds", 2, BOUNDED.getStartTimestamp()), + new Round(SERVICE, "instance-0", "conversation-bounds", 3, BOUNDED.getEndTimestamp() - 1), + new Round(SERVICE, "instance-0", "conversation-bounds", 4, BOUNDED.getEndTimestamp() + 1), + new Round(SERVICE, "instance-0", "conversation-bounds", 5, ENDING.getEndTimestamp()), + new Round(SERVICE, "instance-0", "conversation-bounds", 6, ENDING_AT_MIDNIGHT.getEndTimestamp()) + ); + /** Seq 3 is pushed by a second sender as well, a second later. */ + private static final List FILES = Arrays.asList( + new SessionFile(SERVICE, "instance-0", SESSION, 1), + new SessionFile(SERVICE, "instance-0", SESSION, 2), + new SessionFile(SERVICE, "instance-0", SESSION, 3), + new SessionFile(SERVICE, "instance-1", SESSION, 3), + new SessionFile(SERVICE, "instance-0", SESSION, 4), + new SessionFile(SERVICE, "instance-0", SESSION, 5), + new SessionFile(SERVICE, "instance-0", SESSION, 6), + new SessionFile(SERVICE, "instance-0", "session-other", 3), + new SessionFile(OTHER_SERVICE, "instance-0", SESSION, 3) + ); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private AIAgentConversationQueryEsDAO dao; + + AIAgentConversationQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeConversations() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final List rounds = ROUNDS.stream().map(Round::record).collect(Collectors.toList()); + final List files = FILES.stream().map(SessionFile::record) + .collect(Collectors.toList()); + for (int i = 0; i < COPIES; i++) { + rounds.add(new Round(SERVICE, "copy-" + i, "conversation-copies", 1, MIDNIGHT + 40 * MINUTE).record()); + files.add(new SessionFile(SERVICE, "copy-" + i, "session-copies", 1).record()); + } + final Model roundModel = support.installRecord(AIAgentSessionFlowRecord.class); + final Model fileModel = support.installRecord(AIAgentSessionDataRecord.class); + support.writeRecords(roundModel, new AIAgentSessionFlowRecord.Builder(), rounds); + support.writeRecords(fileModel, new AIAgentSessionDataRecord.Builder(), files); + dao = new AIAgentConversationQueryEsDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheLatestRoundsOfAConversationNewestFirst() throws IOException { + assertEquals(Arrays.asList("4@instance-0", "3@instance-1", "3@instance-0", "2@instance-0", "1@instance-0", + "5@instance-0"), + rounds(dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 10, false))); + assertEquals(Arrays.asList("4@instance-0", "3@instance-1", "3@instance-0"), + rounds(dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 3, false))); + } + + @Test + public void readsTheRoundsOfASenderOfEveryConversationAndOfAService() throws IOException { + assertEquals(Collections.singletonList("3@instance-1"), rounds(dao.queryRounds( + serviceId(SERVICE), instanceId(SERVICE, "instance-1"), CONVERSATION, AROUND_MIDNIGHT, 10, false))); + assertEquals(Arrays.asList("9@instance-0", "4@instance-0", "3@instance-1", "3@instance-0", "2@instance-0", + "1@instance-0", "5@instance-0"), + rounds(dao.queryRounds(serviceId(SERVICE), null, null, AROUND_MIDNIGHT, 10, false))); + assertEquals(Collections.singletonList("7@instance-0"), rounds(dao.queryRounds( + serviceId(OTHER_SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 10, false))); + } + + @Test + public void readsTheRoundsOfEveryDayWithoutADuration() throws IOException { + assertEquals(Arrays.asList("4@instance-0", "3@instance-1", "3@instance-0", "2@instance-0", "1@instance-0", + "5@instance-0"), + rounds(dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, null, 10, false))); + } + + @Test + public void readsEachRoundWholeWithOrWithoutItsBody() throws IOException { + final List withBody = new ArrayList<>(); + final List withoutBody = new ArrayList<>(); + for (int i = 5; i >= 0; i--) { + withBody.add(render(ROUNDS.get(i).record())); + final AIAgentSessionFlowRecord noBody = ROUNDS.get(i).record(); + noBody.setBody(null); + withoutBody.add(render(noBody)); + } + assertEquals(withBody, dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 10, true) + .stream().map(AIAgentConversationQueryITBase::render).collect(Collectors.toList())); + assertEquals(withoutBody, dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 10, false) + .stream().map(AIAgentConversationQueryITBase::render) + .collect(Collectors.toList())); + } + + @Test + public void keepsTheRoundsWithinTheDuration() throws IOException { + assertEquals(Arrays.asList("3@instance-0", "2@instance-0"), rounds(dao.queryRounds( + serviceId(SERVICE), null, "conversation-bounds", BOUNDED, 10, false))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's index. + */ + @Test + public void readsARoundOnTheInstantTheDurationEndsAt() throws IOException { + assertEquals(Collections.singletonList("5@instance-0"), rounds(dao.queryRounds( + serviceId(SERVICE), null, "conversation-bounds", ENDING, 10, false))); + assertEquals(Collections.singletonList("6@instance-0"), rounds(dao.queryRounds( + serviceId(SERVICE), null, "conversation-bounds", ENDING_AT_MIDNIGHT, 10, false))); + } + + @Test + public void readsTheHighestRoundOfAConversation() throws IOException { + assertEquals(5, dao.queryHeadRound(serviceId(SERVICE), null, CONVERSATION, false)); + assertEquals(3, dao.queryHeadRound(serviceId(SERVICE), instanceId(SERVICE, "instance-1"), CONVERSATION, false)); + assertEquals(9, dao.queryHeadRound(serviceId(SERVICE), null, "conversation-other", false)); + assertEquals(7, dao.queryHeadRound(serviceId(OTHER_SERVICE), null, CONVERSATION, false)); + assertEquals(0, dao.queryHeadRound(serviceId(SERVICE), null, "conversation-missing", false)); + } + + /** + * Copies of one round come in no order. + */ + @Test + public void readsEveryCopyOfTheRoundsOfAWindowInRoundOrder() throws IOException { + final List window = dao.queryRoundsByNumber( + serviceId(SERVICE), null, CONVERSATION, 2, 4, Integer.MAX_VALUE, false); + assertEquals(Arrays.asList(2L, 3L, 3L, 4L), window.stream().map(AIAgentSessionFlowRecord::getRound) + .collect(Collectors.toList())); + assertEquals(Arrays.asList(render(ROUNDS.get(2).record()), render(ROUNDS.get(3).record()), + render(ROUNDS.get(4).record()), render(ROUNDS.get(5).record())), + window.stream().sorted(Comparator.comparing(AIAgentSessionFlowRecord::getRound) + .thenComparing(AIAgentSessionFlowRecord::getServiceInstanceId)) + .map(AIAgentConversationQueryITBase::render).collect(Collectors.toList())); + assertEquals(Arrays.asList("2@instance-0", "3@instance-0", "4@instance-0"), rounds(dao.queryRoundsByNumber( + serviceId(SERVICE), instanceId(SERVICE, "instance-0"), CONVERSATION, 2, 4, Integer.MAX_VALUE, false))); + assertEquals(Collections.singletonList("5@instance-0"), rounds(dao.queryRoundsByNumber( + serviceId(SERVICE), null, CONVERSATION, 5, 5, Integer.MAX_VALUE, false))); + assertEquals(Collections.emptyList(), rounds(dao.queryRoundsByNumber( + serviceId(SERVICE), null, CONVERSATION, 6, 8, Integer.MAX_VALUE, false))); + } + + @Test + public void readsAtMostTheRowsTheWindowMakesRoomFor() throws IOException { + final List rounds = dao.queryRoundsByNumber( + serviceId(SERVICE), null, "conversation-copies", 1, 1, Integer.MAX_VALUE, false); + assertEquals(Collections.nCopies(IAIAgentConversationQueryDAO.MAX_COPIES, 1L), rounds.stream() + .map(AIAgentSessionFlowRecord::getRound).collect(Collectors.toList())); + final List files = dao.queryFiles( + serviceId(SERVICE), null, "session-copies", time(1) - 1, time(1) + 1, 1, 1, Integer.MAX_VALUE, false); + assertEquals(Collections.nCopies(IAIAgentConversationQueryDAO.MAX_COPIES, 1L), files.stream() + .map(AIAgentSessionDataRecord::getSeq).collect(Collectors.toList())); + } + + /** + * Copies of one seq come in no order. + */ + @Test + public void readsTheFilesOfASeqWindowBySeq() throws IOException { + final List files = dao.queryFiles( + serviceId(SERVICE), null, SESSION, time(1), time(6) + 1000L, 2, 4, Integer.MAX_VALUE, false); + assertEquals(Arrays.asList(2L, 3L, 3L, 4L), files.stream().map(AIAgentSessionDataRecord::getSeq) + .collect(Collectors.toList())); + assertEquals(Arrays.asList(render(FILES.get(1).record()), render(FILES.get(2).record()), + render(FILES.get(3).record()), render(FILES.get(4).record())), + files.stream().sorted(Comparator.comparing(AIAgentSessionDataRecord::getSeq) + .thenComparing(AIAgentSessionDataRecord::getServiceInstanceId)) + .map(AIAgentConversationQueryITBase::render).collect(Collectors.toList())); + assertEquals(Collections.singletonList("3@instance-1"), files(dao.queryFiles( + serviceId(SERVICE), instanceId(SERVICE, "instance-1"), SESSION, time(1), time(6) + 1000L, 1, 6, + Integer.MAX_VALUE, false))); + } + + /** + * Both ends of the timestamp range are inclusive. + */ + @Test + public void readsTheFilesOnAndWithinTheTimestampRange() throws IOException { + assertEquals(Arrays.asList("2@instance-0", "3@instance-0", "3@instance-1", "4@instance-0", "5@instance-0"), + files(dao.queryFiles(serviceId(SERVICE), null, SESSION, time(2), time(5), 1, 6, + Integer.MAX_VALUE, false))); + assertEquals(Arrays.asList("3@instance-0", "3@instance-1", "4@instance-0"), + files(dao.queryFiles(serviceId(SERVICE), null, SESSION, time(2) + 1, time(5) - 1, 1, 6, + Integer.MAX_VALUE, false))); + } + + @Test + public void readsTheFilesOfEveryDaySinceTheEpoch() throws IOException { + assertEquals(Arrays.asList("1@instance-0", "2@instance-0", "3@instance-0", "3@instance-1", "4@instance-0", + "5@instance-0", "6@instance-0"), + files(dao.queryFiles(serviceId(SERVICE), null, SESSION, 0, time(6) + 1000L, 1, 6, + Integer.MAX_VALUE, false))); + } + + private static List rounds(final List rounds) { + return rounds.stream() + .map(round -> round.getRound() + "@" + instanceName(round.getServiceInstanceId())) + .collect(Collectors.toList()); + } + + /** Copies of one seq sorted by their sender. */ + private static List files(final List files) { + return files.stream() + .sorted(Comparator.comparing(AIAgentSessionDataRecord::getSeq) + .thenComparing(AIAgentSessionDataRecord::getServiceInstanceId)) + .map(file -> file.getSeq() + "@" + instanceName(file.getServiceInstanceId())) + .collect(Collectors.toList()); + } + + private static String render(final AIAgentSessionFlowRecord round) { + return String.join("|", round.getServiceId(), round.getServiceInstanceId(), round.getConversation(), + String.valueOf(round.getRound()), String.valueOf(round.getSessionFromTime()), + round.getTitle(), String.valueOf(round.getTalks()), String.valueOf(round.getSteps()), + String.valueOf(round.getStreams()), String.valueOf(round.getSegments()), + String.valueOf(round.getUnresolved()), String.valueOf(round.getChanges()), + String.valueOf(round.getLinesAdded()), String.valueOf(round.getLinesRemoved()), + String.valueOf(round.getLlmCalls()), String.valueOf(round.getSubagents()), + String.valueOf(round.getBashRuns()), round.getDigest(), + String.valueOf(round.getTimestamp()), String.valueOf(round.getTimeBucket()), + body(round.getBody())); + } + + private static String render(final AIAgentSessionDataRecord file) { + return String.join("|", file.getServiceId(), file.getServiceInstanceId(), file.getSession(), + String.valueOf(file.getSeq()), file.getDigest(), String.valueOf(file.getTimestamp()), + String.valueOf(file.getTimeBucket()), body(file.getBody())); + } + + private static String body(final byte[] body) { + return body == null ? "no body" : new String(body, StandardCharsets.UTF_8); + } + + private static String instanceName(final String instanceId) { + return IDManager.ServiceInstanceID.analysisId(instanceId).getName(); + } + + /** The time of a session's file of the seq: seq 3 at midnight, a minute apart. */ + private static long time(final long seq) { + return MIDNIGHT + (seq - 3) * MINUTE; + } + + private static Duration minutes(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.MINUTE); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmm")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmm")); + return duration; + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + private static String instanceId(final String service, final String instance) { + return IDManager.ServiceInstanceID.buildId(serviceId(service), instance); + } + + /** + * A conversation round as the Sessionizer pushes it; only round 4 carries the optional counts. + */ + private static final class Round { + private final String service; + private final String instance; + private final String conversation; + private final long round; + private final long time; + private boolean counted; + + Round(final String service, final String instance, final String conversation, final long round, + final long time) { + this.service = service; + this.instance = instance; + this.conversation = conversation; + this.round = round; + this.time = time; + } + + Round counted() { + counted = true; + return this; + } + + AIAgentSessionFlowRecord record() { + final AIAgentSessionFlowRecord record = new AIAgentSessionFlowRecord(); + record.setServiceId(serviceId(service)); + record.setServiceInstanceId(instanceId(service, instance)); + record.setConversation(conversation); + record.setRound(round); + record.setSessionFromTime(MIDNIGHT - 10 * MINUTE); + record.setTitle("title of " + conversation); + record.setTalks(round); + record.setSteps(round * 2); + record.setStreams(round * 3); + record.setSegments(round * 4); + record.setUnresolved(round % 2); + if (counted) { + record.setChanges(2L); + record.setLinesAdded(10L); + record.setLinesRemoved(3L); + record.setLlmCalls(7L); + record.setSubagents(1L); + record.setBashRuns(4L); + } + record.setDigest("digest-" + conversation + "-" + round); + record.setTimestamp(time); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + record.setBody(("round " + round + " of " + conversation + " by " + instance) + .getBytes(StandardCharsets.UTF_8)); + return record; + } + } + + /** + * A session file as the Sessionizer pushes it, at the time of its seq; a second sender pushes a second later. + */ + private static final class SessionFile { + private final String service; + private final String instance; + private final String session; + private final long seq; + + SessionFile(final String service, final String instance, final String session, final long seq) { + this.service = service; + this.instance = instance; + this.session = session; + this.seq = seq; + } + + AIAgentSessionDataRecord record() { + final long timestamp = "instance-1".equals(instance) ? time(seq) + 1000L : time(seq); + final AIAgentSessionDataRecord record = new AIAgentSessionDataRecord(); + record.setServiceId(serviceId(service)); + record.setServiceInstanceId(instanceId(service, instance)); + record.setSession(session); + record.setSeq(seq); + record.setDigest("file-" + session + "-" + seq); + record.setTimestamp(timestamp); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + record.setBody(("file " + seq + " of " + session + " by " + instance).getBytes(StandardCharsets.UTF_8)); + return record; + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AlarmQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AlarmQueryITBase.java new file mode 100644 index 000000000000..e3ab6a1b77a3 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/AlarmQueryITBase.java @@ -0,0 +1,511 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.alarm.AlarmRecord; +import org.apache.skywalking.oap.server.core.alarm.AlarmRecoveryRecord; +import org.apache.skywalking.oap.server.core.alarm.AlarmSnapshotRecord; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.query.enumeration.Scope; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.AlarmQueryCondition; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.Entity; +import org.apache.skywalking.oap.server.core.query.mqe.MQEValue; +import org.apache.skywalking.oap.server.core.query.mqe.MQEValues; +import org.apache.skywalking.oap.server.core.query.type.AlarmMessage; +import org.apache.skywalking.oap.server.core.query.type.Alarms; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads alarms the way both alarm queries do on Elasticsearch and OpenSearch, through {@link AlarmQueryEsDAO}: the + * legacy {@code getAlarm} by scope, keyword and tags, and {@code queryAlarms} by entity, layer, rule name, keyword + * and tags, both newest first and page by page, each alarm with the recovery time {@code alarm_recovery_record} + * holds for it. A nested class of a server's IT names the server and the storage mode. + * + *

The alarms are written as the alarm persistence writes them, on both sides of midnight, so a read spans two + * daily indices: the searchable tags indexed and every tag in the raw tags, the recovery in the time bucket of the + * alarm's start. An entity is the primary of some alarms and the destination of a relation alarm, the relation also + * fires the other way round, and for each other filter an alarm matches every condition but that one. A page holds + * more recovered alarms than a search returns by default. Alarms of their own lie on and just outside the bounds of + * a duration: on its first millisecond and the one before, on its last millisecond, on the instant it ends at + * ({@link Duration#getEndTimestamp()}, which a read includes) and the millisecond after. + * One more lies on the end instant of a duration ending in a day's last minute: the next midnight, in the next + * day's index. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class AlarmQueryITBase { + /** A fixed local midnight, so the daily indices the alarms are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final long MINUTE = 60_000L; + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + private static final Duration ENDING = minutes(MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE); + /** The midnight a day before {@link #MIDNIGHT}. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = minutes(PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE); + private static final Duration RECOVERING = minutes(MIDNIGHT + 30 * MINUTE, MIDNIGHT + 30 * MINUTE); + /** More than the 10 hits a search returns when it does not ask for a size. */ + private static final int RECOVERED_IN_ONE_PAGE = 12; + private static final Gson GSON = new Gson(); + + private static final String CHECKOUT = "checkout"; + private static final String PAYMENT = "payment"; + private static final String BOUNDS = "bounds"; + private static final String SLA = "Successful rate of service %s is lower than 80%%"; + private static final String RESP = "Response time of service %s is more than 1000ms"; + + private static final List ALARMS = Arrays.asList( + AlarmRow.service("a1", MIDNIGHT - 150_000L, CHECKOUT, String.format(RESP, CHECKOUT), + "service_resp_time_rule", Layer.GENERAL).tags("env=prod", "owner=alice").recovered() + .snapshot("avg(service_resp_time) > 1000", "service_resp_time", "1200"), + AlarmRow.service("a2", MIDNIGHT - 110_000L, PAYMENT, String.format(SLA, PAYMENT), "service_sla_rule", + Layer.MESH).tags("env=test"), + AlarmRow.endpoint("a3", MIDNIGHT - 70_000L, CHECKOUT, "POST /orders", + "Response time of endpoint POST /orders is more than 1000ms", "endpoint_resp_time_rule", + Layer.GENERAL).tags("env=prod", "team=a").recovered(), + AlarmRow.relation("a4", MIDNIGHT - 30_000L, CHECKOUT, PAYMENT, Layer.GENERAL), + AlarmRow.service("a5", MIDNIGHT + 30_000L, CHECKOUT, String.format(SLA, CHECKOUT), "service_sla_rule", + Layer.GENERAL).tags("env=prod").recovered(), + AlarmRow.instance("a6", MIDNIGHT + 70_000L, CHECKOUT, "checkout-1", + "Response time of service instance checkout-1 is more than 1000ms", + "instance_resp_time_rule", Layer.GENERAL).tags("env=prod", "team=b"), + AlarmRow.service("a7", MIDNIGHT + 110_000L, PAYMENT, "Success count of service payment is lower than 10", + "service_success_rule", Layer.MESH), + AlarmRow.relation("a8", MIDNIGHT + 150_000L, PAYMENT, CHECKOUT, Layer.MESH) + ); + private static final List BOUND_ALARMS = Arrays.asList( + AlarmRow.service("before-start", BOUNDED.getStartTimestamp() - 1, BOUNDS, "bounds", "bounds_rule", + Layer.GENERAL), + AlarmRow.service("on-start", BOUNDED.getStartTimestamp(), BOUNDS, "bounds", "bounds_rule", Layer.GENERAL), + AlarmRow.service("last-millisecond", BOUNDED.getEndTimestamp() - 1, BOUNDS, "bounds", "bounds_rule", + Layer.GENERAL), + AlarmRow.service("after-end", BOUNDED.getEndTimestamp() + 1, BOUNDS, "bounds", "bounds_rule", + Layer.GENERAL), + AlarmRow.service("on-end", ENDING.getEndTimestamp(), BOUNDS, "bounds", "bounds_rule", Layer.GENERAL) + .recovered(), + AlarmRow.service("on-midnight", ENDING_AT_MIDNIGHT.getEndTimestamp(), BOUNDS, "bounds", "bounds_rule", + Layer.GENERAL).recovered() + ); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private AlarmQueryEsDAO dao; + + AlarmQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeAlarms() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final List alarms = new ArrayList<>(ALARMS); + alarms.addAll(BOUND_ALARMS); + for (int i = 0; i < RECOVERED_IN_ONE_PAGE; i++) { + alarms.add(AlarmRow.service(String.format("recovered-%02d", i), RECOVERING.getStartTimestamp() + i * 1000L, + "recovering", "recovering", "recovering_rule", Layer.GENERAL).recovered()); + } + final Model alarmModel = support.installRecord(AlarmRecord.class); + final Model recoveryModel = support.installRecord(AlarmRecoveryRecord.class); + support.writeRecords(alarmModel, new AlarmRecord.Builder(), alarms.stream() + .map(AlarmRow::record).collect(Collectors.toList())); + support.writeRecords(recoveryModel, new AlarmRecoveryRecord.Builder(), alarms.stream() + .filter(alarm -> alarm.recoveryTime != null).map(AlarmRow::recovery).collect(Collectors.toList())); + dao = new AlarmQueryEsDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void pagesThroughEveryAlarmNewestFirst() throws IOException { + assertEquals(Arrays.asList("a8", "a7", "a6"), uuids(dao.queryAlarms(condition(), 3, 0))); + assertEquals(Arrays.asList("a5", "a4", "a3"), uuids(dao.queryAlarms(condition(), 3, 3))); + assertEquals(Arrays.asList("a2", "a1"), uuids(dao.queryAlarms(condition(), 3, 6))); + assertEquals(Collections.emptyList(), uuids(dao.queryAlarms(condition(), 3, 9))); + } + + /** + * A service is the primary of its own alarms and the destination of a relation alarm. + */ + @Test + public void readsTheAlarmsOfAServiceAsThePrimaryOrTheRelationsDestination() throws IOException { + assertEquals(Arrays.asList("a8", "a5", "a4", "a1"), uuids(dao.queryAlarms(entities(service(CHECKOUT)), 10, 0))); + assertEquals(Arrays.asList("a8", "a7", "a4", "a2"), uuids(dao.queryAlarms(entities(service(PAYMENT)), 10, 0))); + } + + @Test + public void readsTheAlarmsOfARelationInItsDirectionOnly() throws IOException { + assertEquals(Collections.singletonList("a4"), uuids(dao.queryAlarms( + entities(relation(CHECKOUT, PAYMENT)), 10, 0))); + } + + @Test + public void readsTheAlarmsOfAnInstanceAndOfAnEndpoint() throws IOException { + final Entity instance = new Entity(); + instance.setScope(Scope.ServiceInstance); + instance.setServiceName(CHECKOUT); + instance.setServiceInstanceName("checkout-1"); + assertEquals(Collections.singletonList("a6"), uuids(dao.queryAlarms(entities(instance), 10, 0))); + final Entity endpoint = new Entity(); + endpoint.setScope(Scope.Endpoint); + endpoint.setServiceName(CHECKOUT); + endpoint.setEndpointName("POST /orders"); + assertEquals(Collections.singletonList("a3"), uuids(dao.queryAlarms(entities(endpoint), 10, 0))); + } + + @Test + public void readsAnAlarmOnceWhenTwoEntitiesMatchIt() throws IOException { + assertEquals(Arrays.asList("a8", "a5", "a4", "a1"), uuids(dao.queryAlarms( + entities(service(CHECKOUT), service(CHECKOUT)), 10, 0))); + assertEquals(Arrays.asList("a8", "a5", "a4", "a1"), uuids(dao.queryAlarms( + entities(service(CHECKOUT), relation(CHECKOUT, PAYMENT)), 10, 0))); + } + + @Test + public void pagesThroughTheAlarmsOfAService() throws IOException { + final AlarmQueryCondition checkout = entities(service(CHECKOUT)); + assertEquals(Arrays.asList("a8", "a5"), uuids(dao.queryAlarms(checkout, 2, 0))); + assertEquals(Arrays.asList("a4", "a1"), uuids(dao.queryAlarms(checkout, 2, 2))); + assertEquals(Collections.emptyList(), uuids(dao.queryAlarms(checkout, 2, 4))); + } + + /** + * {@code Success count} is not the keyword {@code Successful}, and {@code owner} is not a searchable tag. + */ + @Test + public void filtersByLayerRuleNameKeywordAndTags() throws IOException { + final AlarmQueryCondition mesh = condition(); + mesh.setLayer(Layer.MESH.name()); + assertEquals(Arrays.asList("a8", "a7", "a2"), uuids(dao.queryAlarms(mesh, 10, 0))); + final AlarmQueryCondition sla = condition(); + sla.setRuleNames(Collections.singletonList("service_sla_rule")); + assertEquals(Arrays.asList("a5", "a2"), uuids(dao.queryAlarms(sla, 10, 0))); + final AlarmQueryCondition responseTime = condition(); + responseTime.setRuleNames(Arrays.asList("service_resp_time_rule", "endpoint_resp_time_rule")); + assertEquals(Arrays.asList("a3", "a1"), uuids(dao.queryAlarms(responseTime, 10, 0))); + final AlarmQueryCondition keyword = condition(); + keyword.setKeyword("Successful"); + assertEquals(Arrays.asList("a5", "a2"), uuids(dao.queryAlarms(keyword, 10, 0))); + final AlarmQueryCondition prod = condition(); + prod.setTags(tags("env=prod")); + assertEquals(Arrays.asList("a6", "a5", "a3", "a1"), uuids(dao.queryAlarms(prod, 10, 0))); + final AlarmQueryCondition teamA = condition(); + teamA.setTags(tags("env=prod", "team=a")); + assertEquals(Collections.singletonList("a3"), uuids(dao.queryAlarms(teamA, 10, 0))); + final AlarmQueryCondition owner = condition(); + owner.setTags(tags("owner=alice")); + assertEquals(Collections.emptyList(), uuids(dao.queryAlarms(owner, 10, 0))); + } + + @Test + public void readsTheLegacyAlarmListByScopeKeywordAndTags() throws IOException { + assertEquals(Arrays.asList("a7", "a5", "a2", "a1"), uuids(legacy(DefaultScopeDefine.SERVICE, null, null))); + assertEquals(Collections.singletonList("a6"), uuids(legacy(DefaultScopeDefine.SERVICE_INSTANCE, null, null))); + assertEquals(Collections.singletonList("a3"), uuids(legacy(DefaultScopeDefine.ENDPOINT, null, null))); + assertEquals(Arrays.asList("a8", "a4"), uuids(legacy(DefaultScopeDefine.SERVICE_RELATION, null, null))); + assertEquals(Arrays.asList("a5", "a2"), uuids(legacy(null, "Successful", null))); + assertEquals(Arrays.asList("a6", "a5", "a3", "a1"), uuids(legacy(null, null, tags("env=prod")))); + assertEquals(Collections.singletonList("a3"), uuids(legacy(null, null, tags("env=prod", "team=a")))); + assertEquals(Collections.emptyList(), uuids(legacy(null, null, tags("owner=alice")))); + } + + @Test + public void pagesThroughTheLegacyAlarmList() throws IOException { + assertEquals(Arrays.asList("a8", "a7", "a6"), uuids(dao.getAlarm(null, null, 3, 0, AROUND_MIDNIGHT, null))); + assertEquals(Arrays.asList("a5", "a4", "a3"), uuids(dao.getAlarm(null, null, 3, 3, AROUND_MIDNIGHT, null))); + assertEquals(Arrays.asList("a2", "a1"), uuids(dao.getAlarm(null, null, 3, 6, AROUND_MIDNIGHT, null))); + assertEquals(Collections.emptyList(), uuids(dao.getAlarm(null, null, 3, 9, AROUND_MIDNIGHT, null))); + } + + @Test + public void readsEachAlarmWhole() throws IOException { + final List expected = new ArrayList<>(); + for (int i = ALARMS.size() - 1; i >= 0; i--) { + expected.add(ALARMS.get(i).render()); + } + assertEquals(expected, render(dao.queryAlarms(condition(), 10, 0))); + assertEquals(expected, render(dao.getAlarm(null, null, 10, 0, AROUND_MIDNIGHT, null))); + } + + @Test + public void readsTheRecoveryTimeOfEveryAlarmOfAPage() throws IOException { + final AlarmQueryCondition recovering = condition(); + recovering.setDuration(RECOVERING); + final List expected = new ArrayList<>(); + for (int i = RECOVERED_IN_ONE_PAGE - 1; i >= 0; i--) { + final long startTime = RECOVERING.getStartTimestamp() + i * 1000L; + expected.add(String.format("recovered-%02d", i) + "|" + (startTime + MINUTE)); + } + assertEquals(expected, recoveries(dao.queryAlarms(recovering, 20, 0))); + assertEquals(expected, recoveries(dao.getAlarm(null, null, 20, 0, RECOVERING, null))); + } + + @Test + public void keepsTheAlarmsWithinTheDuration() throws IOException { + final AlarmQueryCondition bounded = entities(service(BOUNDS)); + bounded.setDuration(BOUNDED); + assertEquals(Arrays.asList("last-millisecond", "on-start"), uuids(dao.queryAlarms(bounded, 10, 0))); + assertEquals(Arrays.asList("last-millisecond", "on-start"), uuids(dao.getAlarm( + DefaultScopeDefine.SERVICE, null, 10, 0, BOUNDED, null))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's index; an + * alarm's recovery is stored in the time bucket of its start. + */ + @Test + public void readsAnAlarmOnTheInstantTheDurationEndsAt() throws IOException { + for (final Duration duration : Arrays.asList(ENDING, ENDING_AT_MIDNIGHT)) { + final long startTime = duration.getEndTimestamp(); + final List expected = Collections.singletonList( + (duration == ENDING ? "on-end" : "on-midnight") + "|" + (startTime + MINUTE)); + final AlarmQueryCondition ending = entities(service(BOUNDS)); + ending.setDuration(duration); + assertEquals(expected, recoveries(dao.queryAlarms(ending, 10, 0))); + assertEquals(expected, recoveries(dao.getAlarm(DefaultScopeDefine.SERVICE, null, 10, 0, duration, null))); + } + } + + private Alarms legacy(final Integer scopeId, final String keyword, final List tags) throws IOException { + return dao.getAlarm(scopeId, keyword, 10, 0, AROUND_MIDNIGHT, tags); + } + + private static AlarmQueryCondition condition() { + final AlarmQueryCondition condition = new AlarmQueryCondition(); + condition.setDuration(AROUND_MIDNIGHT); + return condition; + } + + private static AlarmQueryCondition entities(final Entity... entities) { + final AlarmQueryCondition condition = condition(); + condition.setEntities(Arrays.asList(entities)); + return condition; + } + + private static Entity service(final String name) { + final Entity service = new Entity(); + service.setScope(Scope.Service); + service.setServiceName(name); + service.setNormal(true); + return service; + } + + private static Entity relation(final String source, final String dest) { + final Entity relation = new Entity(); + relation.setScope(Scope.ServiceRelation); + relation.setServiceName(source); + relation.setDestServiceName(dest); + return relation; + } + + private static List tags(final String... tags) { + return Arrays.stream(tags).map(tag -> tag.split("=")).map(kv -> new Tag(kv[0], kv[1])) + .collect(Collectors.toList()); + } + + private static List uuids(final Alarms alarms) { + return alarms.getMsgs().stream().map(AlarmMessage::getUuid).collect(Collectors.toList()); + } + + private static List recoveries(final Alarms alarms) { + return alarms.getMsgs().stream().map(alarm -> alarm.getUuid() + "|" + alarm.getRecoveryTime()) + .collect(Collectors.toList()); + } + + private static List render(final Alarms alarms) { + return alarms.getMsgs().stream().map(alarm -> String.join( + "|", alarm.getUuid(), String.valueOf(alarm.getScopeId()), alarm.getId(), alarm.getId1(), alarm.getName(), + alarm.getMessage(), String.valueOf(alarm.getStartTime()), String.valueOf(alarm.getRecoveryTime()), + alarm.getTags().stream().map(tag -> tag.getKey() + "=" + tag.getValue()).collect(Collectors.joining(",")), + alarm.getSnapshot().getExpression() + ":" + alarm.getSnapshot().getMetrics().stream() + .map(metric -> metric.getName() + "=" + metric.getResults().stream() + .flatMap(values -> values.getValues().stream()).map(MQEValue::getValue) + .collect(Collectors.joining(","))) + .collect(Collectors.joining(";")) + )).collect(Collectors.toList()); + } + + private static Duration minutes(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.MINUTE); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmm")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmm")); + return duration; + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + /** + * An alarm as the alarm persistence hands it to the record stream; a recovered one recovers a minute after it + * starts. + */ + private static final class AlarmRow { + /** The tag keys the ITs configure as {@code searchableAlarmTags}. */ + private static final List SEARCHABLE_TAG_KEYS = Arrays.asList("env", "team"); + + private final String uuid; + private final long startTime; + private final int scope; + private final String id0; + private final String id1; + private final String name; + private final String message; + private final String ruleName; + private final Layer layer; + private List tags = Collections.emptyList(); + private Long recoveryTime; + private AlarmSnapshotRecord snapshot = new AlarmSnapshotRecord(); + + private AlarmRow(final String uuid, final long startTime, final int scope, final String id0, final String id1, + final String name, final String message, final String ruleName, final Layer layer) { + this.uuid = uuid; + this.startTime = startTime; + this.scope = scope; + this.id0 = id0; + this.id1 = id1; + this.name = name; + this.message = message; + this.ruleName = ruleName; + this.layer = layer; + } + + static AlarmRow service(final String uuid, final long startTime, final String service, final String message, + final String ruleName, final Layer layer) { + return new AlarmRow(uuid, startTime, DefaultScopeDefine.SERVICE, serviceId(service), "", service, + message, ruleName, layer); + } + + static AlarmRow instance(final String uuid, final long startTime, final String service, final String instance, + final String message, final String ruleName, final Layer layer) { + return new AlarmRow(uuid, startTime, DefaultScopeDefine.SERVICE_INSTANCE, + IDManager.ServiceInstanceID.buildId(serviceId(service), instance), "", + instance + " of " + service, message, ruleName, layer); + } + + static AlarmRow endpoint(final String uuid, final long startTime, final String service, final String endpoint, + final String message, final String ruleName, final Layer layer) { + return new AlarmRow(uuid, startTime, DefaultScopeDefine.ENDPOINT, + IDManager.EndpointID.buildId(serviceId(service), endpoint), "", + endpoint + " in " + service, message, ruleName, layer); + } + + static AlarmRow relation(final String uuid, final long startTime, final String source, final String dest, + final Layer layer) { + return new AlarmRow(uuid, startTime, DefaultScopeDefine.SERVICE_RELATION, serviceId(source), + serviceId(dest), source + " to " + dest, + "Response time of service " + source + " to service " + dest + " is more than 1000ms", + "service_relation_resp_time_rule", layer); + } + + AlarmRow tags(final String... tags) { + this.tags = Arrays.asList(tags); + return this; + } + + AlarmRow recovered() { + this.recoveryTime = startTime + MINUTE; + return this; + } + + AlarmRow snapshot(final String expression, final String metric, final String value) { + final MQEValue point = new MQEValue(); + point.setId(String.valueOf(TimeBucket.getMinuteTimeBucket(startTime))); + point.setValue(value); + point.setDoubleValue(Double.parseDouble(value)); + final MQEValues values = new MQEValues(); + values.getValues().add(point); + final JsonObject metrics = new JsonObject(); + metrics.addProperty(metric, GSON.toJson(Collections.singletonList(values))); + snapshot.setExpression(expression); + snapshot.setMetrics(metrics); + return this; + } + + AlarmRecord record() { + final AlarmRecord record = new AlarmRecord(); + record.setUuid(uuid); + record.setScope(scope); + record.setId0(id0); + record.setId1(id1); + record.setName(name); + record.setAlarmMessage(message); + record.setStartTime(startTime); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(startTime)); + record.setRuleName(ruleName); + record.setLayer(layer); + record.setTagsRawData(GSON.toJson(tags.stream().map(tag -> tag.split("=")) + .map(kv -> new Tag(kv[0], kv[1])).collect(Collectors.toList())) + .getBytes(StandardCharsets.UTF_8)); + record.setTagsInString(tags.stream().filter(tag -> SEARCHABLE_TAG_KEYS.contains(tag.split("=")[0])) + .collect(Collectors.toList())); + record.setSnapshot(GSON.toJson(snapshot)); + return record; + } + + AlarmRecoveryRecord recovery() { + final AlarmRecoveryRecord recovery = new AlarmRecoveryRecord(); + recovery.setUuid(uuid); + recovery.setRecoveryTime(recoveryTime); + recovery.setTimeBucket(TimeBucket.getRecordTimeBucket(startTime)); + return recovery; + } + + String render() { + final String metrics = snapshot.getMetrics() == null ? "" : snapshot.getMetrics().keySet().stream() + .map(metric -> metric + "=" + GSON.fromJson(snapshot.getMetrics().get(metric).getAsString(), + MQEValues[].class)[0].getValues().get(0).getValue()) + .collect(Collectors.joining(";")); + return String.join("|", uuid, String.valueOf(scope), id0, id1, name, message, String.valueOf(startTime), + String.valueOf(recoveryTime), String.join(",", tags), + snapshot.getExpression() + ":" + metrics); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/BrowserLogQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/BrowserLogQueryITBase.java new file mode 100644 index 000000000000..ec742fbe1d70 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/BrowserLogQueryITBase.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog; +import org.apache.skywalking.apm.network.language.agent.v3.ErrorCategory; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.browser.manual.errorlog.BrowserErrorLogRecord; +import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.BrowserErrorLogs; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads browser error logs the way the browser log query does on Elasticsearch and OpenSearch, through + * {@link BrowserLogQueryEsDAO}: by service, version, page and error category, newest first, page by page, with or + * without a duration. A nested class of a server's IT names the server and the storage mode. + * + *

The errors are written as the browser receiver writes them, on both sides of midnight, so a read spans two + * daily indices. For each filter an error matches every other condition but that one. Errors of their own lie on and + * just outside the bounds of a duration: on its first millisecond and the one before, on its last millisecond, on the + * instant it ends at ({@link Duration#getEndTimestamp()}, which a read includes) and the millisecond after. + * One more lies on the end instant of a duration ending in a day's last minute: the next midnight, in the next + * day's index. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class BrowserLogQueryITBase { + /** A fixed local midnight, so the daily indices the errors are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final long MINUTE = 60_000L; + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + private static final Duration ENDING = minutes(MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE); + /** The midnight a day before {@link #MIDNIGHT}. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = minutes(PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE); + + private static final String SHOP = "shop-web"; + private static final String ADMIN = "admin-web"; + private static final String BOUNDS = "bounds-web"; + + private static final List ERRORS = Arrays.asList( + new ErrorLog("e1", MIDNIGHT - 150_000L, SHOP, "v1", "/cart", ErrorCategory.js), + new ErrorLog("e2", MIDNIGHT - 90_000L, SHOP, "v1", "/cart", ErrorCategory.ajax), + new ErrorLog("e3", MIDNIGHT - 30_000L, SHOP, "v2", "/cart", ErrorCategory.js), + new ErrorLog("e4", MIDNIGHT + 30_000L, SHOP, "v1", "/checkout", ErrorCategory.js), + new ErrorLog("e5", MIDNIGHT + 90_000L, SHOP, "v1", "/cart", ErrorCategory.js), + new ErrorLog("e6", MIDNIGHT + 150_000L, ADMIN, "v1", "/cart", ErrorCategory.js), + new ErrorLog("before-start", BOUNDED.getStartTimestamp() - 1, BOUNDS, "v1", "/", ErrorCategory.resource), + new ErrorLog("on-start", BOUNDED.getStartTimestamp(), BOUNDS, "v1", "/", ErrorCategory.resource), + new ErrorLog("last-millisecond", BOUNDED.getEndTimestamp() - 1, BOUNDS, "v1", "/", ErrorCategory.resource), + new ErrorLog("after-end", BOUNDED.getEndTimestamp() + 1, BOUNDS, "v1", "/", ErrorCategory.resource), + new ErrorLog("on-end", ENDING.getEndTimestamp(), BOUNDS, "v1", "/", ErrorCategory.resource), + new ErrorLog("on-midnight", ENDING_AT_MIDNIGHT.getEndTimestamp(), BOUNDS, "v1", "/", ErrorCategory.resource) + ); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private BrowserLogQueryEsDAO dao; + + BrowserLogQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeErrors() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final Model model = support.installRecord(BrowserErrorLogRecord.class); + support.writeRecords(model, new BrowserErrorLogRecord.Builder(), ERRORS.stream() + .map(ErrorLog::record).collect(Collectors.toList())); + dao = new BrowserLogQueryEsDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheErrorsOfAServiceNewestFirst() throws IOException { + assertEquals(Arrays.asList("e5", "e4", "e3", "e2", "e1"), ids(serviceId(SHOP), null, null, null)); + assertEquals(Collections.singletonList("e6"), ids(serviceId(ADMIN), null, null, null)); + } + + @Test + public void readsTheErrorsOfAVersion() throws IOException { + assertEquals(Arrays.asList("e5", "e4", "e2", "e1"), ids(null, versionId(SHOP, "v1"), null, null)); + } + + @Test + public void readsTheErrorsOfAPage() throws IOException { + assertEquals(Arrays.asList("e5", "e3", "e2", "e1"), ids(null, null, pageId(SHOP, "/cart"), null)); + } + + @Test + public void readsTheErrorsOfACategory() throws IOException { + assertEquals(Arrays.asList("e6", "e5", "e4", "e3", "e1"), ids(null, null, null, BrowserErrorCategory.JS)); + } + + @Test + public void readsTheErrorsMatchingEveryFilter() throws IOException { + assertEquals(Arrays.asList("e5", "e1"), ids( + serviceId(SHOP), versionId(SHOP, "v1"), pageId(SHOP, "/cart"), BrowserErrorCategory.JS)); + } + + @Test + public void pagesThroughTheErrorsOfAService() throws IOException { + assertEquals(Arrays.asList("e5", "e4"), page(0, 2)); + assertEquals(Arrays.asList("e3", "e2"), page(2, 2)); + assertEquals(Collections.singletonList("e1"), page(4, 2)); + assertEquals(Collections.emptyList(), page(6, 2)); + } + + @Test + public void readsEachErrorWhole() throws IOException { + final List expected = new ArrayList<>(); + for (int i = 5; i >= 0; i--) { + expected.add(ERRORS.get(i).render()); + } + assertEquals(expected, dao.queryBrowserErrorLogs(null, null, null, null, AROUND_MIDNIGHT, 10, 0) + .getLogs().stream().map(BrowserLogQueryITBase::render) + .collect(Collectors.toList())); + } + + @Test + public void readsTheErrorsOfEveryDayWithoutADuration() throws IOException { + assertEquals(Arrays.asList("e5", "e4", "e2", "e1"), messages( + dao.queryBrowserErrorLogs(null, versionId(SHOP, "v1"), null, null, null, 10, 0))); + final List everyBound = Arrays.asList( + "on-end", "after-end", "last-millisecond", "on-start", "before-start", "on-midnight"); + assertEquals(everyBound, messages(dao.queryBrowserErrorLogs(serviceId(BOUNDS), null, null, null, null, 10, 0))); + } + + @Test + public void keepsTheErrorsWithinTheDuration() throws IOException { + assertEquals(Arrays.asList("last-millisecond", "on-start"), messages( + dao.queryBrowserErrorLogs(serviceId(BOUNDS), null, null, null, BOUNDED, 10, 0))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's index. + */ + @Test + public void readsAnErrorOnTheInstantTheDurationEndsAt() throws IOException { + assertEquals(Collections.singletonList("on-end"), messages( + dao.queryBrowserErrorLogs(serviceId(BOUNDS), null, null, null, ENDING, 10, 0))); + assertEquals(Collections.singletonList("on-midnight"), messages( + dao.queryBrowserErrorLogs(serviceId(BOUNDS), null, null, null, ENDING_AT_MIDNIGHT, 10, 0))); + } + + private List ids(final String serviceId, final String versionId, final String pageId, + final BrowserErrorCategory category) throws IOException { + return messages(dao.queryBrowserErrorLogs(serviceId, versionId, pageId, category, AROUND_MIDNIGHT, 10, 0)); + } + + private List page(final int from, final int limit) throws IOException { + return messages(dao.queryBrowserErrorLogs(serviceId(SHOP), null, null, null, AROUND_MIDNIGHT, limit, from)); + } + + /** Each error's message is its id. */ + private static List messages(final BrowserErrorLogs logs) { + return logs.getLogs().stream().map(log -> log.getMessage()).collect(Collectors.toList()); + } + + private static String render(final org.apache.skywalking.oap.server.core.query.type.BrowserErrorLog log) { + return String.join("|", log.getService(), log.getServiceVersion(), String.valueOf(log.getTime()), + log.getPagePath(), log.getCategory().name(), log.getGrade(), log.getMessage(), + String.valueOf(log.getLine()), String.valueOf(log.getCol()), log.getStack(), + log.getErrorUrl(), String.valueOf(log.isFirstReportedError())); + } + + private static Duration minutes(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.MINUTE); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmm")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmm")); + return duration; + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + private static String versionId(final String service, final String version) { + return IDManager.ServiceInstanceID.buildId(serviceId(service), version); + } + + private static String pageId(final String service, final String page) { + return IDManager.EndpointID.buildId(serviceId(service), page); + } + + /** + * A browser error log as the browser error log receiver hands it to the record stream. + */ + private static final class ErrorLog { + private final String id; + private final long time; + private final String service; + private final String version; + private final String page; + private final ErrorCategory category; + + ErrorLog(final String id, final long time, final String service, final String version, final String page, + final ErrorCategory category) { + this.id = id; + this.time = time; + this.service = service; + this.version = version; + this.page = page; + this.category = category; + } + + BrowserErrorLogRecord record() { + final BrowserErrorLogRecord record = new BrowserErrorLogRecord(); + record.setUniqueId(id); + record.setServiceId(serviceId(service)); + record.setServiceVersionId(versionId(service, version)); + record.setPagePathId(pageId(service, page)); + record.setErrorCategory(BrowserErrorCategory.valueOf(category.name().toUpperCase()).getValue()); + record.setTimestamp(time); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + record.setDataBinary(BrowserErrorLog.newBuilder() + .setUniqueId(id) + .setService(service) + .setServiceVersion(version) + .setTime(time) + .setPagePath(page) + .setCategory(category) + .setGrade("error") + .setMessage(id) + .setLine(12) + .setCol(34) + .setStack("at " + page) + .setErrorUrl("https://shop.example.com" + page + ".js") + .setFirstReportedError("e1".equals(id)) + .build().toByteArray()); + return record; + } + + String render() { + return String.join("|", service, version, String.valueOf(time), page, category.name().toUpperCase(), + "error", id, "12", "34", "at " + page, "https://shop.example.com" + page + ".js", + String.valueOf("e1".equals(id))); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingQueryITBase.java new file mode 100644 index 000000000000..653849a61e5a --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EBPFProfilingQueryITBase.java @@ -0,0 +1,627 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import com.google.gson.Gson; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.skywalking.library.elasticsearch.requests.search.Query; +import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.worker.NoneStreamProcessor; +import org.apache.skywalking.oap.server.core.profiling.ebpf.EBPFProfilingMutationService; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingDataRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingScheduleRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTargetType; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTriggerType; +import org.apache.skywalking.oap.server.core.query.type.EBPFNetworkKeepProfilingResult; +import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingSchedule; +import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingTaskContinuousProfiling; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.ModelRegistry; +import org.apache.skywalking.oap.server.core.storage.profiling.ebpf.IEBPFProfilingTaskDAO; +import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexNames; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsDAO; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Reads eBPF profiling tasks, schedules and data the way the eBPF receiver, the profiling query and mutation + * services and the profiling analysis do, through {@link EBPFProfilingTaskEsDAO}, {@link EBPFProfilingScheduleEsDAO} + * and {@link EBPFProfilingDataEsDAO}, from what OAP's own write paths stored in Elasticsearch or OpenSearch. A nested + * class of a server's IT names the server and the storage mode. + * + *

The tasks are created on both sides of midnight, so in two daily indices: fixed-time tasks of two services, a + * network task of an instance with the copy that extends it after midnight (a second row with the same logical id), + * and continuous-profiling tasks of two instances. Each filter has a near miss, and the start and update times lie + * on and just outside the bounds asked for. A task search returns the tasks newest created first, cut to + * {@code profileTaskQueryMaxSize}, set here below the tasks of a service. The schedules of a task start on both sides + * of midnight beside a schedule of another task. The eBPF data of a schedule is uploaded every half second through a + * window that spans midnight, with uploads just before the window and on its end, beside the uploads of other + * schedules, and a schedule has more stacks in the window than a scroll page ({@code profileDataQueryBatchSize}). + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class EBPFProfilingQueryITBase { + /** A fixed local midnight, so the daily indices the rows are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + /** Fewer than the tasks of the service, which are six. */ + private static final int TASKS_PER_QUERY = 5; + /** Fewer than one schedule's stacks in the window. */ + private static final int STACKS_PER_PAGE = 7; + private static final Gson GSON = new Gson(); + private static final String SERVICE_ID = IDManager.ServiceID.buildId("checkout", true); + private static final String OTHER_SERVICE_ID = IDManager.ServiceID.buildId("payment", true); + private static final String INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-1"); + private static final String OTHER_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-2"); + + private static final EBPFProfilingTaskRecord ON_CPU = fixedTimeTask( + SERVICE_ID, Collections.emptyList(), EBPFProfilingTargetType.ON_CPU, minutes(-30), minutes(-30)); + /** Created five minutes after midnight to start five minutes later, for processes with a label. */ + private static final EBPFProfilingTaskRecord OFF_CPU = fixedTimeTask( + SERVICE_ID, Collections.singletonList("env:prod"), EBPFProfilingTargetType.OFF_CPU, minutes(5), minutes(10)); + private static final EBPFProfilingTaskRecord OTHER_SERVICE_ON_CPU = fixedTimeTask( + OTHER_SERVICE_ID, Collections.emptyList(), EBPFProfilingTargetType.ON_CPU, minutes(-10), minutes(-10)); + private static final EBPFProfilingTaskRecord NETWORK = networkTask(INSTANCE_ID, minutes(-20)); + private static final EBPFProfilingTaskRecord NETWORK_EXTENDED = extended(NETWORK, minutes(1)); + private static final EBPFProfilingTaskRecord CONTINUOUS_ON_CPU = continuousProfilingTask( + INSTANCE_ID, EBPFProfilingTargetType.ON_CPU, minutes(-5)); + private static final EBPFProfilingTaskRecord CONTINUOUS_OFF_CPU = continuousProfilingTask( + OTHER_INSTANCE_ID, EBPFProfilingTargetType.OFF_CPU, minutes(3)); + /** Neither in creation order nor against it. */ + private static final List TASKS = Arrays.asList( + NETWORK_EXTENDED, ON_CPU, CONTINUOUS_OFF_CPU, OTHER_SERVICE_ON_CPU, OFF_CPU, NETWORK, CONTINUOUS_ON_CPU); + private static final List ALL_TARGETS = Arrays.asList(EBPFProfilingTargetType.values()); + private static final List CPU = + Arrays.asList(EBPFProfilingTargetType.ON_CPU, EBPFProfilingTargetType.OFF_CPU); + + private static final EBPFProfilingScheduleRecord FIRST_SCHEDULE = + schedule(NETWORK.getLogicalId(), INSTANCE_ID, "checkout", minutes(-15), minutes(-14)); + /** Started before midnight and still running after it. */ + private static final EBPFProfilingScheduleRecord ACROSS_MIDNIGHT_SCHEDULE = + schedule(NETWORK.getLogicalId(), INSTANCE_ID, "checkout-worker", minutes(-3), minutes(2)); + private static final EBPFProfilingScheduleRecord LAST_SCHEDULE = + schedule(NETWORK.getLogicalId(), INSTANCE_ID, "checkout", minutes(5), minutes(6)); + private static final EBPFProfilingScheduleRecord OTHER_TASK_SCHEDULE = + schedule(ON_CPU.getLogicalId(), INSTANCE_ID, "checkout", minutes(-1), minutes(0)); + + private static final String SCHEDULE = "schedule-read"; + private static final String OTHER_SCHEDULE = "schedule-also-read"; + private static final String UNASKED_SCHEDULE = "schedule-not-read"; + private static final long WINDOW_BEGIN = MIDNIGHT - 5_000L; + private static final long WINDOW_END = MIDNIGHT + 5_000L; + private static final int UPLOADS = 20; + private static final int STACKS_PER_UPLOAD = 3; + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private Model tasks; + private EBPFProfilingTaskEsDAO taskDAO; + private EBPFProfilingScheduleEsDAO scheduleDAO; + private EBPFProfilingDataEsDAO dataDAO; + private final List windowOfSchedule = new ArrayList<>(); + private final List windowOfOtherSchedule = new ArrayList<>(); + private final List beforeWindow = new ArrayList<>(); + private final List onWindowEnd = new ArrayList<>(); + + EBPFProfilingQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeProfiling() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass(), config -> { + config.setProfileTaskQueryMaxSize(TASKS_PER_QUERY); + config.setProfileDataQueryBatchSize(STACKS_PER_PAGE); + }); + + // Every model is installed before the first write, as at boot: in OAP's default mode the tasks and the data + // share one physical table, whose dated indices take their mappings when the first write creates them. + tasks = support.installNoneStream(EBPFProfilingTaskRecord.class); + final Model schedules = support.installMetrics(EBPFProfilingScheduleRecord.class); + final Model dataModel = support.installRecord(EBPFProfilingDataRecord.class); + + for (final EBPFProfilingTaskRecord task : TASKS) { + support.writeNoneStream(tasks, new EBPFProfilingTaskRecord.Builder(), task); + } + support.writeMetrics(schedules, new EBPFProfilingScheduleRecord.Builder(), Arrays.asList( + ACROSS_MIDNIGHT_SCHEDULE, OTHER_TASK_SCHEDULE, FIRST_SCHEDULE, LAST_SCHEDULE)); + + for (int upload = 0; upload < UPLOADS; upload++) { + for (int stack = 0; stack < STACKS_PER_UPLOAD; stack++) { + windowOfSchedule.add(data(SCHEDULE, upload + "-" + stack, WINDOW_BEGIN + upload * 500L)); + } + } + for (int stack = 0; stack < STACKS_PER_UPLOAD; stack++) { + beforeWindow.add(data(SCHEDULE, "before-" + stack, WINDOW_BEGIN - 1)); + onWindowEnd.add(data(SCHEDULE, "end-" + stack, WINDOW_END)); + windowOfOtherSchedule.add(data(OTHER_SCHEDULE, "other-" + stack, MIDNIGHT - 1_000L)); + windowOfOtherSchedule.add(data(OTHER_SCHEDULE, "other-later-" + stack, MIDNIGHT + 1_000L)); + } + final List data = new ArrayList<>(windowOfSchedule); + data.addAll(windowOfOtherSchedule); + data.addAll(beforeWindow); + data.addAll(onWindowEnd); + data.add(data(UNASKED_SCHEDULE, "unasked", MIDNIGHT)); + support.writeRecords(dataModel, new EBPFProfilingDataRecord.Builder(), data); + + taskDAO = new EBPFProfilingTaskEsDAO(support.getClient(), support.getConfig()); + scheduleDAO = new EBPFProfilingScheduleEsDAO(support.getClient(), support.getConfig()); + dataDAO = new EBPFProfilingDataEsDAO(support.getClient(), support.getConfig()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + /** + * The rover asks for the fixed-time tasks of the services of its processes: neither the tasks of another + * service nor the continuous-profiling tasks, which the rover triggers itself. + */ + @Test + public void readsTheFixedTimeTasksOfTheRoversServicesNewestFirst() throws Exception { + assertEquals(describe(OFF_CPU, NETWORK_EXTENDED, NETWORK, ON_CPU), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, 0, 0))); + assertEquals(describe(OTHER_SERVICE_ON_CPU), describe(taskDAO.queryTasksByServices( + Collections.singletonList(OTHER_SERVICE_ID), null, 0, 0))); + } + + /** + * The rover asks for the tasks updated after its last poll: a task updated at that time is not read again, and + * the copy extending the network task after midnight is. + */ + @Test + public void readsTheTasksUpdatedAfterTheRoversLastPoll() throws Exception { + assertEquals(describe(OFF_CPU, NETWORK_EXTENDED), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, 0, + NETWORK.getLastUpdateTime()))); + assertEquals(describe(OFF_CPU, NETWORK_EXTENDED), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, 0, + NETWORK_EXTENDED.getLastUpdateTime() - 1))); + assertEquals(describe(OFF_CPU), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, 0, + NETWORK_EXTENDED.getLastUpdateTime()))); + } + + @Test + public void readsTheTasksStartingFromATime() throws Exception { + assertEquals(describe(OFF_CPU, NETWORK_EXTENDED, NETWORK), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, NETWORK.getStartTime(), 0))); + assertEquals(describe(OFF_CPU), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, NETWORK.getStartTime() + 1, + 0))); + } + + /** + * The task list of a service, as the profiling page asks for it, by target type. + */ + @Test + public void readsTheTasksOfTheTargetTypesAsked() throws Exception { + assertEquals(describe(CONTINUOUS_ON_CPU, ON_CPU), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, Collections.singletonList(EBPFProfilingTargetType.ON_CPU), null, 0, 0))); + assertEquals(describe(OFF_CPU, CONTINUOUS_OFF_CPU, CONTINUOUS_ON_CPU, ON_CPU), + describe(taskDAO.queryTasksByTargets(SERVICE_ID, null, CPU, null, 0, 0))); + assertEquals(describe(CONTINUOUS_ON_CPU, OTHER_SERVICE_ON_CPU, ON_CPU), describe(taskDAO.queryTasksByTargets( + null, null, Collections.singletonList(EBPFProfilingTargetType.ON_CPU), null, 0, 0))); + } + + @Test + public void readsTheTasksOfAnInstance() throws Exception { + assertEquals(describe(NETWORK_EXTENDED, CONTINUOUS_ON_CPU, NETWORK), + describe(taskDAO.queryTasksByTargets(SERVICE_ID, INSTANCE_ID, ALL_TARGETS, null, 0, 0))); + assertEquals(describe(CONTINUOUS_OFF_CPU), + describe(taskDAO.queryTasksByTargets(null, OTHER_INSTANCE_ID, ALL_TARGETS, null, 0, 0))); + } + + /** + * The continuous-profiling page counts the tasks a policy triggered since a time, creating a fixed-time task + * looks for the tasks of its target starting from its own start, and the task list of a duration asks for the + * tasks updated after its end. + */ + @Test + public void readsTheTasksOfATriggerTypeByTheirStartAndUpdateTimes() throws Exception { + assertEquals(describe(CONTINUOUS_OFF_CPU, CONTINUOUS_ON_CPU), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, CPU, EBPFProfilingTriggerType.CONTINUOUS_PROFILING, CONTINUOUS_ON_CPU.getStartTime(), 0))); + assertEquals(describe(CONTINUOUS_OFF_CPU), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, CPU, EBPFProfilingTriggerType.CONTINUOUS_PROFILING, + CONTINUOUS_ON_CPU.getStartTime() + 1, 0))); + assertEquals(describe(NETWORK_EXTENDED, NETWORK), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, Collections.singletonList(EBPFProfilingTargetType.NETWORK), + EBPFProfilingTriggerType.FIXED_TIME, NETWORK.getStartTime(), 0))); + assertEquals(describe(OFF_CPU), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, CPU, EBPFProfilingTriggerType.FIXED_TIME, 0, NETWORK_EXTENDED.getLastUpdateTime()))); + } + + /** + * A continuous-profiling policy saved without targets asks for the tasks of no target type, as the + * continuous-profiling page counts the tasks each policy triggered. + */ + @Test + public void readsNoTaskForNoTargetType() throws Exception { + assertEquals(Collections.emptyList(), taskDAO.queryTasksByTargets( + SERVICE_ID, null, Collections.emptyList(), EBPFProfilingTriggerType.CONTINUOUS_PROFILING, + minutes(-60), 0)); + } + + @Test + public void keepsTheNewestTasksOfALimit() throws Exception { + assertEquals(describe(OFF_CPU, CONTINUOUS_OFF_CPU, NETWORK_EXTENDED, CONTINUOUS_ON_CPU, NETWORK), + describe(taskDAO.queryTasksByTargets(SERVICE_ID, null, ALL_TARGETS, null, 0, 0))); + assertEquals(describe(OFF_CPU, CONTINUOUS_OFF_CPU, NETWORK_EXTENDED, CONTINUOUS_ON_CPU, OTHER_SERVICE_ON_CPU), + describe(taskDAO.queryTasksByServices(Arrays.asList(SERVICE_ID, OTHER_SERVICE_ID), null, 0, 0))); + } + + /** + * Keeping a network task alive reads every row of the task, the one the task was created with and the copies + * that extend it, whichever daily index each is in. + */ + @Test + public void readsEveryRowOfATaskById() throws Exception { + assertEquals(sorted(describe(NETWORK, NETWORK_EXTENDED)), + sorted(describe(taskDAO.getTaskRecord(NETWORK.getLogicalId())))); + assertEquals(describe(OFF_CPU), describe(taskDAO.getTaskRecord(OFF_CPU.getLogicalId()))); + assertEquals(Collections.emptyList(), taskDAO.getTaskRecord("missing-task")); + } + + @Test + public void readsTheSchedulesOfOneTaskLatestStartFirst() throws Exception { + assertEquals(describeSchedules(LAST_SCHEDULE, ACROSS_MIDNIGHT_SCHEDULE, FIRST_SCHEDULE), + describeSchedules(scheduleDAO.querySchedules(NETWORK.getLogicalId()))); + assertEquals(describeSchedules(OTHER_TASK_SCHEDULE), + describeSchedules(scheduleDAO.querySchedules(ON_CPU.getLogicalId()))); + assertEquals(Collections.emptyList(), scheduleDAO.querySchedules("missing-task")); + } + + /** + * The analysis reads every stack a schedule uploaded in a window, from both daily indices and over several + * scroll pages, and none uploaded just before the window or on its end. + */ + @Test + public void readsEveryStackOfAScheduleInTheWindow() throws Exception { + assertEquals(describeData(windowOfSchedule), + describeData(dataDAO.queryData(Collections.singletonList(SCHEDULE), WINDOW_BEGIN, WINDOW_END))); + } + + @Test + public void readsTheStacksOfTheSchedulesAskedOnly() throws Exception { + final List both = new ArrayList<>(windowOfSchedule); + both.addAll(windowOfOtherSchedule); + assertEquals(describeData(both), describeData( + dataDAO.queryData(Arrays.asList(SCHEDULE, OTHER_SCHEDULE), WINDOW_BEGIN, WINDOW_END))); + } + + /** + * The analysis asks for windows that meet at their ends: an upload on a window's end belongs to the next one. + */ + @Test + public void readsTheUploadsOnAWindowsBoundsInOneWindowOnly() throws Exception { + assertEquals(describeData(onWindowEnd), describeData(dataDAO.queryData( + Collections.singletonList(SCHEDULE), WINDOW_END, WINDOW_END + 10_000L))); + assertEquals(describeData(beforeWindow), describeData(dataDAO.queryData( + Collections.singletonList(SCHEDULE), WINDOW_BEGIN - 10_000L, WINDOW_BEGIN))); + } + + /** + * Every task lies in the daily index of the day it is created on, as the profiling page and the eBPF agents create + * them, with the minute of their creation, so the TTL deletes it with its day. + */ + @Test + public void storesEveryTaskInTheIndexOfItsDay() throws Exception { + final List expected = new ArrayList<>(); + for (final EBPFProfilingTaskRecord task : TASKS) { + expected.add(task.getLogicalId() + "|" + task.getCreateTime() + "|" + dailyIndex(task.getCreateTime())); + } + Collections.sort(expected); + final List stored = new ArrayList<>(); + for (final String logicalId : TASKS.stream().map(EBPFProfilingTaskRecord::getLogicalId).distinct() + .collect(Collectors.toList())) { + stored.addAll(storedIndices(logicalId)); + } + Collections.sort(stored); + assertEquals(expected, stored); + } + + /** + * Keeping a network task alive near its end writes a copy that runs ten minutes longer, created at that time: the + * copy carries the minute it is created in, lies in that day's index, and is read with the task. + */ + @Test + public void keepsANetworkTaskAliveWithACopyInTheIndexOfItsDay() throws Exception { + final String instanceId = IDManager.ServiceInstanceID.buildId( + IDManager.ServiceID.buildId("keep-alive", true), "keep-alive-1"); + // Created to end in thirty seconds, within the minute before its end in which a keep-alive extends it. + final EBPFProfilingTaskRecord task = networkTask( + instanceId, System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(600 - 30)); + support.writeNoneStream(tasks, new EBPFProfilingTaskRecord.Builder(), task); + + final long before = System.currentTimeMillis(); + final EBPFNetworkKeepProfilingResult result = + new EBPFProfilingMutationService(noneStreamWorkers()).keepEBPFNetworkProfiling(task.getLogicalId()); + final long after = System.currentTimeMillis(); + + assertEquals("true|null", result.isStatus() + "|" + result.getErrorReason()); + final List read = taskDAO.getTaskRecord(task.getLogicalId()); + final EBPFProfilingTaskRecord copy = read.stream() + .filter(row -> row.getCreateTime() != task.getCreateTime()) + .findFirst() + .orElseThrow(() -> new AssertionError("no copy of " + read)); + assertTrue(copy.getCreateTime() >= before && copy.getCreateTime() <= after, + "created at " + copy.getCreateTime()); + final EBPFProfilingTaskRecord expectedCopy = extended(task, copy.getCreateTime()); + assertEquals(sorted(describe(task, expectedCopy)), sorted(describe(read))); + assertEquals(sorted(Arrays.asList( + task.getLogicalId() + "|" + task.getCreateTime() + "|" + dailyIndex(task.getCreateTime()), + task.getLogicalId() + "|" + copy.getCreateTime() + "|" + dailyIndex(copy.getCreateTime()))), + sorted(storedIndices(task.getLogicalId()))); + } + + private static long minutes(final int fromMidnight) { + return MIDNIGHT + TimeUnit.MINUTES.toMillis(fromMidnight); + } + + /** + * The task worker of OAP's none-stream processor, writing through the storage's own none-stream DAO, for the + * services that write tasks. + */ + private ModuleManager noneStreamWorkers() throws Exception { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleProviderHolder storage = mock(ModuleProviderHolder.class); + final ModuleServiceHolder storageServices = mock(ModuleServiceHolder.class); + when(moduleManager.find(StorageModule.NAME)).thenReturn(storage); + when(storage.provider()).thenReturn(storageServices); + when(storageServices.getService(IEBPFProfilingTaskDAO.class)).thenReturn(taskDAO); + when(storageServices.getService(StorageDAO.class)).thenReturn(new StorageEsDAO(support.getClient())); + when(storageServices.getService(StorageBuilderFactory.class)).thenReturn(new StorageBuilderFactory.Default()); + final ModuleProviderHolder core = mock(ModuleProviderHolder.class); + final ModuleServiceHolder coreServices = mock(ModuleServiceHolder.class); + final ModelRegistry models = mock(ModelRegistry.class); + when(moduleManager.find(CoreModule.NAME)).thenReturn(core); + when(core.provider()).thenReturn(coreServices); + when(coreServices.getService(ModelRegistry.class)).thenReturn(models); + when(models.add(any(), anyInt(), any(), any())).thenReturn(tasks); + NoneStreamProcessor.getInstance().create( + moduleManager, EBPFProfilingTaskRecord.class.getAnnotation(Stream.class), EBPFProfilingTaskRecord.class); + return moduleManager; + } + + /** + * The daily index of each stored row of the task, with the row's creation time. + */ + private List storedIndices(final String logicalId) { + final SearchResponse response = support.getClient().search( + IndexController.LogicIndicesRegister.getPhysicalTableName(EBPFProfilingTaskRecord.INDEX_NAME), + Search.builder().query(Query.term(EBPFProfilingTaskRecord.LOGICAL_ID, logicalId)).size(100).build()); + return response.getHits().getHits().stream() + .map(hit -> logicalId + "|" + hit.getSource().get(EBPFProfilingTaskRecord.CREATE_TIME) + "|" + + hit.getIndex()) + .collect(Collectors.toList()); + } + + private String dailyIndex(final long createTime) { + return support.getClient().formatIndexName( + IndexNames.writeIndex(tasks, TimeBucket.getMinuteTimeBucket(createTime))); + } + + /** + * Every column of each task, as the task workers stored it, in the order read. + */ + private static List describe(final List tasks) { + final EBPFProfilingTaskRecord.Builder builder = new EBPFProfilingTaskRecord.Builder(); + return tasks.stream().map(task -> { + final HashMapConverter.ToStorage columns = new HashMapConverter.ToStorage(); + builder.entity2Storage(task, columns); + return new TreeMap<>(columns.obtain()).toString(); + }).collect(Collectors.toList()); + } + + private static List describe(final EBPFProfilingTaskRecord... tasks) { + return describe(Arrays.asList(tasks)); + } + + private static List sorted(final List described) { + return described.stream().sorted().collect(Collectors.toList()); + } + + private static List describeSchedules(final List schedules) { + return schedules.stream() + .map(schedule -> String.join("|", schedule.getScheduleId(), schedule.getTaskId(), + schedule.getProcessId(), String.valueOf(schedule.getStartTime()), + String.valueOf(schedule.getEndTime()))) + .collect(Collectors.toList()); + } + + private static List describeSchedules(final EBPFProfilingScheduleRecord... records) { + return describeSchedules(Arrays.stream(records).map(record -> { + final EBPFProfilingSchedule schedule = new EBPFProfilingSchedule(); + schedule.setScheduleId(record.getScheduleId()); + schedule.setTaskId(record.getTaskId()); + schedule.setProcessId(record.getProcessId()); + schedule.setStartTime(record.getStartTime()); + schedule.setEndTime(record.getEndTime()); + return schedule; + }).collect(Collectors.toList())); + } + + /** + * Every field of each stack the analysis uses, in no order. + */ + private static List describeData(final List data) { + return data.stream() + .map(row -> String.join("|", row.getScheduleId(), row.getTaskId(), row.getStackIdList(), + String.valueOf(row.getTargetType()), + Base64.getEncoder().encodeToString(row.getDataBinary()), + String.valueOf(row.getUploadTime()))) + .sorted() + .collect(Collectors.toList()); + } + + /** + * A fixed-time task as the profiling page creates it ({@code EBPFProfilingMutationService#createTask}). + */ + private static EBPFProfilingTaskRecord fixedTimeTask(final String serviceId, final List processLabels, + final EBPFProfilingTargetType targetType, + final long createTime, final long startTime) { + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(serviceId); + task.setProcessLabelsJson(processLabels.isEmpty() ? Const.EMPTY_STRING : GSON.toJson(processLabels)); + task.setStartTime(startTime); + task.setTriggerType(EBPFProfilingTriggerType.FIXED_TIME.value()); + task.setFixedTriggerDuration(60); + task.setTargetType(targetType.value()); + task.setCreateTime(createTime); + task.setLastUpdateTime(createTime); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(createTime)); + task.generateLogicalId(); + task.setExtensionConfigJson(Const.EMPTY_STRING); + return task; + } + + /** + * A network-profiling task of an instance, created to start at once. + */ + private static EBPFProfilingTaskRecord networkTask(final String instanceId, final long createTime) { + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(IDManager.ServiceInstanceID.analysisId(instanceId).getServiceId()); + task.setProcessLabelsJson(Const.EMPTY_STRING); + task.setInstanceId(instanceId); + task.setStartTime(createTime); + task.setTriggerType(EBPFProfilingTriggerType.FIXED_TIME.value()); + task.setFixedTriggerDuration(600); + task.setTargetType(EBPFProfilingTargetType.NETWORK.value()); + task.setCreateTime(createTime); + task.setLastUpdateTime(createTime); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(createTime)); + task.setExtensionConfigJson("{\"networkSamplings\":[{\"uriRegex\":\"/checkout\",\"minDuration\":0}]}"); + task.generateLogicalId(); + return task; + } + + /** + * The copy {@code EBPFProfilingMutationService#keepEBPFNetworkProfiling} writes to keep a network task alive: + * the same logical id, ten more minutes, created and updated at the time it is kept alive. + */ + private static EBPFProfilingTaskRecord extended(final EBPFProfilingTaskRecord task, final long keptAt) { + final EBPFProfilingTaskRecord copy = new EBPFProfilingTaskRecord(); + copy.setLogicalId(task.getLogicalId()); + copy.setServiceId(task.getServiceId()); + copy.setProcessLabelsJson(Const.EMPTY_STRING); + copy.setInstanceId(task.getInstanceId()); + copy.setStartTime(task.getStartTime()); + copy.setTriggerType(task.getTriggerType()); + copy.setFixedTriggerDuration(task.getFixedTriggerDuration() + 600); + copy.setTargetType(EBPFProfilingTargetType.NETWORK.value()); + copy.setCreateTime(keptAt); + copy.setLastUpdateTime(keptAt); + copy.setTimeBucket(TimeBucket.getMinuteTimeBucket(keptAt)); + copy.setExtensionConfigJson(Const.EMPTY_STRING); + return copy; + } + + /** + * A task as the eBPF agent reports one when a continuous-profiling policy triggers on a process of an instance. + */ + private static EBPFProfilingTaskRecord continuousProfilingTask(final String instanceId, + final EBPFProfilingTargetType targetType, + final long time) { + final EBPFProfilingTaskContinuousProfiling continuousProfiling = new EBPFProfilingTaskContinuousProfiling(); + continuousProfiling.setProcessId(IDManager.ProcessID.buildId(instanceId, "checkout")); + continuousProfiling.setProcessName("checkout"); + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(IDManager.ServiceInstanceID.analysisId(instanceId).getServiceId()); + task.setProcessLabelsJson(Const.EMPTY_STRING); + task.setInstanceId(instanceId); + task.setStartTime(time); + task.setTriggerType(EBPFProfilingTriggerType.CONTINUOUS_PROFILING.value()); + task.setFixedTriggerDuration(60); + task.setTargetType(targetType.value()); + task.setCreateTime(time); + task.setLastUpdateTime(time); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(time)); + task.setContinuousProfilingJson(GSON.toJson(continuousProfiling)); + task.generateLogicalId(); + return task; + } + + /** + * A schedule as the eBPF receiver's dispatcher stores it, at the minute of its latest report. + */ + private static EBPFProfilingScheduleRecord schedule(final String taskId, final String instanceId, + final String processName, final long startTime, + final long endTime) { + final EBPFProfilingScheduleRecord schedule = new EBPFProfilingScheduleRecord(); + schedule.setTaskId(taskId); + schedule.setProcessId(IDManager.ProcessID.buildId(instanceId, processName)); + schedule.setStartTime(startTime); + schedule.setEndTime(endTime); + schedule.setScheduleId(taskId + "-" + processName + "-" + startTime); + schedule.setTimeBucket(TimeBucket.getMinuteTimeBucket(endTime)); + return schedule; + } + + /** + * A stack of an upload, as the eBPF profiling data dispatcher writes it. + */ + private static EBPFProfilingDataRecord data(final String scheduleId, final String stack, final long uploadTime) { + final EBPFProfilingDataRecord data = new EBPFProfilingDataRecord(); + data.setScheduleId(scheduleId); + data.setTaskId(NETWORK.getLogicalId()); + data.setStackIdList("stack-" + stack); + data.setTargetType(EBPFProfilingTargetType.ON_CPU.value()); + data.setDataBinary(("stack " + stack).getBytes()); + data.setUploadTime(uploadTime); + data.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime)); + return data; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch8IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch8IT.java new file mode 100644 index 000000000000..00bfd717d9a9 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch8IT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport.Mode; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.otlp.OTLPTraceQueryEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin.ZipkinQueryEsDAOITBase; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * The storage ITs on Elasticsearch 8, in OAP's default storage mode, on one server for every nested IT. + */ +@Tag("slow") +@Tag("elastic-it") +@Testcontainers +public class Elasticsearch8IT { + @Container + private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("8.18.8"); + private static final Mode MODE = Mode.DEFAULT; + + @Nested + class AIAgentConversations extends AIAgentConversationQueryITBase { + AIAgentConversations() { + super(SERVER, MODE); + } + } + + @Nested + class Alarms extends AlarmQueryITBase { + Alarms() { + super(SERVER, MODE); + } + } + + @Nested + class BatchProcess extends BatchProcessEsDAOITBase { + BatchProcess() { + super(SERVER, MODE); + } + } + + @Nested + class BrowserLogs extends BrowserLogQueryITBase { + BrowserLogs() { + super(SERVER, MODE); + } + } + + @Nested + class EBPFProfiling extends EBPFProfilingQueryITBase { + EBPFProfiling() { + super(SERVER, MODE); + } + } + + @Nested + class Events extends EventQueryITBase { + Events() { + super(SERVER, MODE); + } + } + + @Nested + class GenAIEvaluations extends GenAIEvaluationQueryITBase { + GenAIEvaluations() { + super(SERVER, MODE); + } + } + + @Nested + class Hierarchy extends HierarchyQueryITBase { + Hierarchy() { + super(SERVER, MODE); + } + } + + @Nested + class HistoryDelete extends HistoryDeleteEsDAOITBase { + HistoryDelete() { + super(SERVER, MODE); + } + } + + @Nested + class Logs extends LogQueryITBase { + Logs() { + super(SERVER, MODE); + } + } + + @Nested + class ManagementData extends ManagementDataITBase { + ManagementData() { + super(SERVER, MODE); + } + } + + @Nested + class Metadata extends MetadataQueryITBase { + Metadata() { + super(SERVER, MODE); + } + } + + @Nested + class Metrics extends MetricsQueryITBase { + Metrics() { + super(SERVER, MODE); + } + } + + @Nested + class MetricsPersistence extends MetricsPersistenceITBase { + MetricsPersistence() { + super(SERVER, MODE); + } + } + + @Nested + class OTLPTraces extends OTLPTraceQueryEsDAOITBase { + OTLPTraces() { + super(SERVER, MODE); + } + } + + @Nested + class ProfileThreadSnapshots extends ProfileThreadSnapshotQueryITBase { + ProfileThreadSnapshots() { + super(SERVER, MODE); + } + } + + @Nested + class ProfilingTasks extends ProfilingTaskQueryITBase { + ProfilingTasks() { + super(SERVER, MODE); + } + } + + @Nested + class SpanAttachedEvents extends SpanAttachedEventQueryITBase { + SpanAttachedEvents() { + super(SERVER, MODE); + } + } + + @Nested + class TagAutocomplete extends TagAutocompleteQueryITBase { + TagAutocomplete() { + super(SERVER, MODE); + } + } + + @Nested + class Topology extends TopologyQueryITBase { + Topology() { + super(SERVER, MODE); + } + } + + @Nested + class Traces extends TraceQueryEsDAOITBase { + Traces() { + super(SERVER, MODE); + } + } + + @Nested + class Zipkin extends ZipkinQueryEsDAOITBase { + Zipkin() { + super(SERVER, MODE); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch9IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch9IT.java new file mode 100644 index 000000000000..5ce6a421314d --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch9IT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport.Mode; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.otlp.OTLPTraceQueryEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin.ZipkinQueryEsDAOITBase; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * The storage ITs on Elasticsearch 9, in OAP's default storage mode, on one server for every nested IT. + */ +@Tag("slow") +@Tag("elastic-it") +@Testcontainers +public class Elasticsearch9IT { + @Container + private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("9.3.0"); + private static final Mode MODE = Mode.DEFAULT; + + @Nested + class AIAgentConversations extends AIAgentConversationQueryITBase { + AIAgentConversations() { + super(SERVER, MODE); + } + } + + @Nested + class Alarms extends AlarmQueryITBase { + Alarms() { + super(SERVER, MODE); + } + } + + @Nested + class BatchProcess extends BatchProcessEsDAOITBase { + BatchProcess() { + super(SERVER, MODE); + } + } + + @Nested + class BrowserLogs extends BrowserLogQueryITBase { + BrowserLogs() { + super(SERVER, MODE); + } + } + + @Nested + class EBPFProfiling extends EBPFProfilingQueryITBase { + EBPFProfiling() { + super(SERVER, MODE); + } + } + + @Nested + class Events extends EventQueryITBase { + Events() { + super(SERVER, MODE); + } + } + + @Nested + class GenAIEvaluations extends GenAIEvaluationQueryITBase { + GenAIEvaluations() { + super(SERVER, MODE); + } + } + + @Nested + class Hierarchy extends HierarchyQueryITBase { + Hierarchy() { + super(SERVER, MODE); + } + } + + @Nested + class HistoryDelete extends HistoryDeleteEsDAOITBase { + HistoryDelete() { + super(SERVER, MODE); + } + } + + @Nested + class Logs extends LogQueryITBase { + Logs() { + super(SERVER, MODE); + } + } + + @Nested + class ManagementData extends ManagementDataITBase { + ManagementData() { + super(SERVER, MODE); + } + } + + @Nested + class Metadata extends MetadataQueryITBase { + Metadata() { + super(SERVER, MODE); + } + } + + @Nested + class Metrics extends MetricsQueryITBase { + Metrics() { + super(SERVER, MODE); + } + } + + @Nested + class MetricsPersistence extends MetricsPersistenceITBase { + MetricsPersistence() { + super(SERVER, MODE); + } + } + + @Nested + class OTLPTraces extends OTLPTraceQueryEsDAOITBase { + OTLPTraces() { + super(SERVER, MODE); + } + } + + @Nested + class ProfileThreadSnapshots extends ProfileThreadSnapshotQueryITBase { + ProfileThreadSnapshots() { + super(SERVER, MODE); + } + } + + @Nested + class ProfilingTasks extends ProfilingTaskQueryITBase { + ProfilingTasks() { + super(SERVER, MODE); + } + } + + @Nested + class SpanAttachedEvents extends SpanAttachedEventQueryITBase { + SpanAttachedEvents() { + super(SERVER, MODE); + } + } + + @Nested + class TagAutocomplete extends TagAutocompleteQueryITBase { + TagAutocomplete() { + super(SERVER, MODE); + } + } + + @Nested + class Topology extends TopologyQueryITBase { + Topology() { + super(SERVER, MODE); + } + } + + @Nested + class Traces extends TraceQueryEsDAOITBase { + Traces() { + super(SERVER, MODE); + } + } + + @Nested + class Zipkin extends ZipkinQueryEsDAOITBase { + Zipkin() { + super(SERVER, MODE); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch9ShardedIT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch9ShardedIT.java new file mode 100644 index 000000000000..7245ce78bc1c --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/Elasticsearch9ShardedIT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport.Mode; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.otlp.OTLPTraceQueryEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin.ZipkinQueryEsDAOITBase; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * The storage ITs on Elasticsearch 9, with logic sharding and custom routing, on one server for every nested IT. + */ +@Tag("slow") +@Tag("elastic-it") +@Testcontainers +public class Elasticsearch9ShardedIT { + @Container + private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("9.3.0"); + private static final Mode MODE = Mode.LOGIC_SHARDING_AND_CUSTOM_ROUTING; + + @Nested + class AIAgentConversations extends AIAgentConversationQueryITBase { + AIAgentConversations() { + super(SERVER, MODE); + } + } + + @Nested + class Alarms extends AlarmQueryITBase { + Alarms() { + super(SERVER, MODE); + } + } + + @Nested + class BatchProcess extends BatchProcessEsDAOITBase { + BatchProcess() { + super(SERVER, MODE); + } + } + + @Nested + class BrowserLogs extends BrowserLogQueryITBase { + BrowserLogs() { + super(SERVER, MODE); + } + } + + @Nested + class EBPFProfiling extends EBPFProfilingQueryITBase { + EBPFProfiling() { + super(SERVER, MODE); + } + } + + @Nested + class Events extends EventQueryITBase { + Events() { + super(SERVER, MODE); + } + } + + @Nested + class GenAIEvaluations extends GenAIEvaluationQueryITBase { + GenAIEvaluations() { + super(SERVER, MODE); + } + } + + @Nested + class Hierarchy extends HierarchyQueryITBase { + Hierarchy() { + super(SERVER, MODE); + } + } + + @Nested + class HistoryDelete extends HistoryDeleteEsDAOITBase { + HistoryDelete() { + super(SERVER, MODE); + } + } + + @Nested + class Logs extends LogQueryITBase { + Logs() { + super(SERVER, MODE); + } + } + + @Nested + class ManagementData extends ManagementDataITBase { + ManagementData() { + super(SERVER, MODE); + } + } + + @Nested + class Metadata extends MetadataQueryITBase { + Metadata() { + super(SERVER, MODE); + } + } + + @Nested + class Metrics extends MetricsQueryITBase { + Metrics() { + super(SERVER, MODE); + } + } + + @Nested + class MetricsPersistence extends MetricsPersistenceITBase { + MetricsPersistence() { + super(SERVER, MODE); + } + } + + @Nested + class OTLPTraces extends OTLPTraceQueryEsDAOITBase { + OTLPTraces() { + super(SERVER, MODE); + } + } + + @Nested + class ProfileThreadSnapshots extends ProfileThreadSnapshotQueryITBase { + ProfileThreadSnapshots() { + super(SERVER, MODE); + } + } + + @Nested + class ProfilingTasks extends ProfilingTaskQueryITBase { + ProfilingTasks() { + super(SERVER, MODE); + } + } + + @Nested + class SpanAttachedEvents extends SpanAttachedEventQueryITBase { + SpanAttachedEvents() { + super(SERVER, MODE); + } + } + + @Nested + class TagAutocomplete extends TagAutocompleteQueryITBase { + TagAutocomplete() { + super(SERVER, MODE); + } + } + + @Nested + class Topology extends TopologyQueryITBase { + Topology() { + super(SERVER, MODE); + } + } + + @Nested + class Traces extends TraceQueryEsDAOITBase { + Traces() { + super(SERVER, MODE); + } + } + + @Nested + class Zipkin extends ZipkinQueryEsDAOITBase { + Zipkin() { + super(SERVER, MODE); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ElasticsearchServerITsTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ElasticsearchServerITsTest.java new file mode 100644 index 000000000000..53f6c0483925 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ElasticsearchServerITsTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * The storage ITs of every area run on every server: each server IT declares them as nested classes of its own, so a + * report names the server a test ran on, and this test fails when a server IT misses an area another one runs. + */ +public class ElasticsearchServerITsTest { + private static final List> SERVER_ITS = Arrays.asList( + Elasticsearch8IT.class, Elasticsearch9IT.class, Elasticsearch9ShardedIT.class, + OpenSearch1IT.class, OpenSearch2IT.class, OpenSearch3IT.class, OpenSearch3ShardedIT.class); + + @Test + public void runsEveryAreaOnEveryServer() { + final Set expected = areas(SERVER_ITS.get(0)); + for (final Class serverIT : SERVER_ITS) { + assertEquals(expected, areas(serverIT), serverIT.getSimpleName()); + } + } + + private static Set areas(final Class serverIT) { + return Arrays.stream(serverIT.getDeclaredClasses()) + .filter(nested -> nested.isAnnotationPresent(Nested.class)) + .map(nested -> nested.getSimpleName() + " runs " + nested.getSuperclass().getSimpleName()) + .collect(Collectors.toCollection(TreeSet::new)); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EsReadIndexSelectionTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EsReadIndexSelectionTest.java index ba3604106af9..1173681107f6 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EsReadIndexSelectionTest.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EsReadIndexSelectionTest.java @@ -191,13 +191,16 @@ void queryTagAutocompleteKeys_searchesTheTagIndexRolledByDayStep() throws Except } @Test - void queryTagAutocompleteValues_withoutDuration_searchesTheWholeTagIndexFamily() throws Exception { + void queryTagAutocompleteValues_withoutDuration_scrollsTheWholeTagIndexFamily() throws Exception { IndexNames.configure(3, 1); - answerRangeSearch(); + when(client.search(any(String[].class), any(Search.class), any(SearchParams.class))) + .thenReturn(new SearchResponse()); new TagAutoCompleteQueryDAO(client).queryTagAutocompleteValues(TagType.TRACE, "http.method", 10, null); - assertThat(rangeSearchIndices()).containsExactly("tag_autocomplete"); + final ArgumentCaptor indices = ArgumentCaptor.forClass(String[].class); + verify(client).search(indices.capture(), any(Search.class), any(SearchParams.class)); + assertThat(indices.getValue()).containsExactly("tag_autocomplete"); } @Test diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EventQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EventQueryITBase.java new file mode 100644 index 000000000000..adbc6ec93ccc --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/EventQueryITBase.java @@ -0,0 +1,348 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.record.Event; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.Pagination; +import org.apache.skywalking.oap.server.core.query.type.event.EventQueryCondition; +import org.apache.skywalking.oap.server.core.query.type.event.EventType; +import org.apache.skywalking.oap.server.core.query.type.event.Events; +import org.apache.skywalking.oap.server.core.query.type.event.Source; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads events the way the event query service does on Elasticsearch and OpenSearch, through + * {@link ESEventQueryDAO}: by uuid, service, instance, endpoint, name, type and layer, newest or oldest first, page by + * page, and by several conditions, an OR of groups that each AND their fields, where the first condition's order, + * paging and duration apply to the whole read. A nested class of a server's IT names the server and the storage mode. + * + *

The events are written on both sides of midnight, so a read spans two daily indices. For each filter an event + * matches every other condition but that one, and two events of other groups match one field of each group. Events + * of their own lie on and just outside the bounds of a duration: on its first millisecond and the one before, on its + * last millisecond, on the instant it ends at ({@link Duration#getEndTimestamp()}, which a read includes) and the + * millisecond after. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class EventQueryITBase { + /** A fixed local midnight, so the daily indices the events are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final long MINUTE = 60_000L; + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + + private static final String CHECKOUT = "checkout"; + private static final String PAYMENT = "payment"; + private static final String BOUNDS = "bounds"; + private static final String POST_ORDERS = "POST /orders"; + + /** + * An event that has ended is stored at its end time, one that has only started at its start time, as the event + * analyzer stores them. + */ + private static final List EVENTS = Arrays.asList( + new EventRow("ev1", MIDNIGHT - 150_000L, CHECKOUT, "checkout-1", POST_ORDERS, "Upgrade", EventType.Normal, + Layer.GENERAL, true), + new EventRow("ev2", MIDNIGHT - 90_000L, CHECKOUT, "checkout-2", POST_ORDERS, "Reboot", EventType.Error, + Layer.MESH, false), + new EventRow("ev3", MIDNIGHT - 30_000L, CHECKOUT, "checkout-1", "GET /orders", "Upgrade", EventType.Error, + Layer.GENERAL, true), + new EventRow("ev4", MIDNIGHT + 30_000L, PAYMENT, "payment-1", "POST /pay", "Upgrade", EventType.Normal, + Layer.MESH, true), + new EventRow("ev5", MIDNIGHT + 90_000L, CHECKOUT, "checkout-1", POST_ORDERS, "Reboot", EventType.Normal, + Layer.GENERAL, true), + new EventRow("ev6", MIDNIGHT + 150_000L, PAYMENT, "payment-1", "POST /pay", "Reboot", EventType.Error, + Layer.GENERAL, false), + new EventRow("before-start", BOUNDED.getStartTimestamp() - 1, BOUNDS, "bounds-1", "/", "Scale", + EventType.Normal, Layer.GENERAL, false), + new EventRow("on-start", BOUNDED.getStartTimestamp(), BOUNDS, "bounds-1", "/", "Scale", + EventType.Normal, Layer.GENERAL, false), + new EventRow("last-millisecond", BOUNDED.getEndTimestamp() - 1, BOUNDS, "bounds-1", "/", "Scale", + EventType.Normal, Layer.GENERAL, false), + new EventRow("on-end", BOUNDED.getEndTimestamp(), BOUNDS, "bounds-1", "/", "Scale", + EventType.Normal, Layer.GENERAL, false), + new EventRow("after-end", BOUNDED.getEndTimestamp() + 1, BOUNDS, "bounds-1", "/", "Scale", + EventType.Normal, Layer.GENERAL, false) + ); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private ESEventQueryDAO dao; + + EventQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeEvents() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final Model model = support.installRecord(Event.class); + support.writeRecords(model, new Event.Builder(), EVENTS.stream() + .map(EventRow::record).collect(Collectors.toList())); + dao = new ESEventQueryDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheEventsOfAServiceNewestOrOldestFirst() throws Exception { + assertEquals(Arrays.asList("ev5", "ev3", "ev2", "ev1"), uuids(dao.queryEvents( + condition().source(CHECKOUT, null, null).build()))); + assertEquals(Arrays.asList("ev1", "ev2", "ev3", "ev5"), uuids(dao.queryEvents( + condition().source(CHECKOUT, null, null).order(Order.ASC).build()))); + } + + @Test + public void readsTheEventsOfAnInstance() throws Exception { + assertEquals(Arrays.asList("ev5", "ev3", "ev1"), uuids(dao.queryEvents( + condition().source(null, "checkout-1", null).build()))); + } + + @Test + public void readsTheEventsOfAnEndpoint() throws Exception { + assertEquals(Arrays.asList("ev5", "ev2", "ev1"), uuids(dao.queryEvents( + condition().source(null, null, POST_ORDERS).build()))); + } + + @Test + public void readsAnEventByUuidWithOrWithoutADuration() throws Exception { + assertEquals(Collections.singletonList("ev3"), uuids(dao.queryEvents(condition().uuid("ev3").build()))); + assertEquals(Collections.singletonList("ev3"), uuids(dao.queryEvents( + condition().uuid("ev3").during(null).build()))); + } + + @Test + public void readsTheEventsOfANameATypeAndALayer() throws Exception { + assertEquals(Arrays.asList("ev4", "ev3", "ev1"), uuids(dao.queryEvents(condition().name("Upgrade").build()))); + assertEquals(Arrays.asList("ev6", "ev3", "ev2"), uuids(dao.queryEvents( + condition().type(EventType.Error).build()))); + assertEquals(Arrays.asList("ev4", "ev2"), uuids(dao.queryEvents(condition().layer(Layer.MESH).build()))); + } + + @Test + public void readsTheEventsOfEitherGroupOnly() throws Exception { + assertEquals(Arrays.asList("ev5", "ev4", "ev2"), uuids(dao.queryEvents(Arrays.asList( + condition().source(CHECKOUT, null, null).name("Reboot").build(), + condition().source(PAYMENT, null, null).name("Upgrade").build())))); + assertEquals(Arrays.asList("ev2", "ev4", "ev5"), uuids(dao.queryEvents(Arrays.asList( + condition().source(CHECKOUT, null, null).name("Reboot").order(Order.ASC).build(), + condition().source(PAYMENT, null, null).name("Upgrade").build())))); + } + + @Test + public void pagesThroughTheEventsOfAService() throws Exception { + assertEquals(Arrays.asList("ev5", "ev3"), uuids(dao.queryEvents( + condition().source(CHECKOUT, null, null).page(1, 2).build()))); + assertEquals(Arrays.asList("ev2", "ev1"), uuids(dao.queryEvents( + condition().source(CHECKOUT, null, null).page(2, 2).build()))); + assertEquals(Collections.emptyList(), uuids(dao.queryEvents( + condition().source(CHECKOUT, null, null).page(3, 2).build()))); + } + + @Test + public void pagesThroughTheEventsOfTheGroups() throws Exception { + final List> pages = new ArrayList<>(); + for (int page = 1; page <= 3; page++) { + pages.add(uuids(dao.queryEvents(Arrays.asList( + condition().source(CHECKOUT, null, null).name("Reboot").page(page, 2).build(), + condition().source(PAYMENT, null, null).name("Upgrade").page(page, 2).build())))); + } + assertEquals(Arrays.asList(Arrays.asList("ev5", "ev4"), Collections.singletonList("ev2"), + Collections.emptyList()), pages); + } + + @Test + public void readsEachEventWhole() throws Exception { + final List expected = new ArrayList<>(); + for (int i = 5; i >= 0; i--) { + expected.add(EVENTS.get(i).render()); + } + assertEquals(expected, dao.queryEvents(condition().build()).getEvents().stream() + .map(EventQueryITBase::render).collect(Collectors.toList())); + } + + @Test + public void readsTheEventsOnAndWithinTheBoundsOfTheDuration() throws Exception { + assertEquals(Arrays.asList("on-end", "last-millisecond", "on-start"), uuids(dao.queryEvents( + condition().source(BOUNDS, null, null).during(BOUNDED).build()))); + } + + private static Condition condition() { + return new Condition(); + } + + private static List uuids(final Events events) { + return events.getEvents().stream().map(event -> event.getUuid()).collect(Collectors.toList()); + } + + private static String render(final org.apache.skywalking.oap.server.core.query.type.event.Event event) { + return String.join("|", event.getUuid(), event.getSource().getService(), + event.getSource().getServiceInstance(), event.getSource().getEndpoint(), event.getName(), + event.getType().name(), event.getMessage(), event.getParameters().stream() + .map(kv -> kv.getKey() + "=" + kv.getValue()).collect(Collectors.joining(",")), + String.valueOf(event.getStartTime()), String.valueOf(event.getEndTime()), + String.valueOf(event.getTimestamp()), event.getLayer()); + } + + private static Duration minutes(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.MINUTE); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmm")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmm")); + return duration; + } + + /** + * An event query condition as the event query service passes it, newest first over the events around midnight + * unless told otherwise. + */ + private static final class Condition { + private final EventQueryCondition condition = new EventQueryCondition(); + + Condition() { + condition.setTime(AROUND_MIDNIGHT); + condition.setOrder(Order.DES); + condition.setPaging(new Pagination(1, 20)); + } + + Condition uuid(final String uuid) { + condition.setUuid(uuid); + return this; + } + + Condition source(final String service, final String instance, final String endpoint) { + condition.setSource(new Source(service, instance, endpoint)); + return this; + } + + Condition name(final String name) { + condition.setName(name); + return this; + } + + Condition type(final EventType type) { + condition.setType(type); + return this; + } + + Condition layer(final Layer layer) { + condition.setLayer(layer.name()); + return this; + } + + Condition order(final Order order) { + condition.setOrder(order); + return this; + } + + Condition page(final int pageNum, final int pageSize) { + condition.setPaging(new Pagination(pageNum, pageSize)); + return this; + } + + Condition during(final Duration duration) { + condition.setTime(duration); + return this; + } + + EventQueryCondition build() { + return condition; + } + } + + /** + * An event as the event analyzer hands it to the record stream. + */ + private static final class EventRow { + private final String uuid; + private final long time; + private final String service; + private final String instance; + private final String endpoint; + private final String name; + private final EventType type; + private final Layer layer; + private final long startTime; + private final long endTime; + + EventRow(final String uuid, final long time, final String service, final String instance, + final String endpoint, final String name, final EventType type, final Layer layer, + final boolean ended) { + this.uuid = uuid; + this.time = time; + this.service = service; + this.instance = instance; + this.endpoint = endpoint; + this.name = name; + this.type = type; + this.layer = layer; + this.startTime = ended ? time - 5_000L : time; + this.endTime = ended ? time : 0; + } + + Event record() { + final Event event = new Event(); + event.setUuid(uuid); + event.setService(service); + event.setServiceInstance(instance); + event.setEndpoint(endpoint); + event.setName(name); + event.setType(type.name()); + event.setMessage(name + " of " + uuid); + event.setParameters("{\"replicas\":\"3\"}"); + event.setStartTime(startTime); + event.setEndTime(endTime); + event.setLayer(layer); + event.setTimestamp(time); + event.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + return event; + } + + String render() { + return String.join("|", uuid, service, instance, endpoint, name, type.name(), name + " of " + uuid, + "replicas=3", String.valueOf(startTime), String.valueOf(endTime), + String.valueOf(time), layer.name()); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationQueryITBase.java new file mode 100644 index 000000000000..fab5f2333c3e --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/GenAIEvaluationQueryITBase.java @@ -0,0 +1,440 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; +import org.apache.skywalking.oap.server.core.query.enumeration.GenAIEvaluationRecordSortBy; +import org.apache.skywalking.oap.server.core.query.enumeration.GenAIEvaluationValueType; +import org.apache.skywalking.oap.server.core.query.enumeration.GenAITraceRefType; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.GenAITraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.GenAITraceRef; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads GenAI evaluation records the way the GenAI evaluation query service does on Elasticsearch and OpenSearch, + * through {@link GenAIEvaluationRecordQueryEsDAO}: by service, provider, model, value type, score bounds, boolean + * value, task, evaluation level, judge model and the trace, native segment and span index, or OTLP and Zipkin span + * it was produced from, newest, oldest, highest or lowest score first, page by page. A nested class of a server's IT + * names the server and the storage mode. + * + *

The records are written as the span evaluation writes them, on both sides of midnight, so a read spans two + * daily indices. For each filter a record matches every other condition but that one, and the score bounds lie on a + * record's score. Records of their own lie on and just outside the bounds of a duration: on its first millisecond and + * the one before, on its last millisecond, on the instant it ends at ({@link Duration#getEndTimestamp()}, which a + * read includes) and the millisecond after. + * One more lies on the end instant of a duration ending in a day's last minute: the next midnight, in the next + * day's index. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class GenAIEvaluationQueryITBase { + /** A fixed local midnight, so the daily indices the records are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final long MINUTE = 60_000L; + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + private static final Duration ENDING = minutes(MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE); + /** The midnight a day before {@link #MIDNIGHT}. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = minutes(PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE); + + private static final String CHAT = "chat-app"; + private static final String OPENAI = "openai"; + private static final String TRACE_1 = "trace-1"; + private static final String TRACE_2 = "trace-2"; + private static final String OTLP_SPAN = "a1b2c3d4e5f60718"; + + private static final List EVALUATIONS = Arrays.asList( + new Evaluation("g1", MIDNIGHT - 150_000L, CHAT, OPENAI, "gpt-4o", "relevance", "judge-a") + .score(800_000L, "SPAN").nativeSpan(TRACE_1, "segment-1", 0), + new Evaluation("g2", MIDNIGHT - 110_000L, CHAT, OPENAI, "gpt-4o", "safety", "judge-a") + .bool(true, "TRACE").nativeSpan(TRACE_1, "segment-1", 1), + new Evaluation("g3", MIDNIGHT - 70_000L, CHAT, OPENAI, "gpt-4o-mini", "relevance", "judge-b") + .score(300_000L, "SPAN").nativeSpan(TRACE_1, "segment-2", 0), + new Evaluation("g4", MIDNIGHT - 30_000L, "search-app", "anthropic", "claude", "relevance", "judge-a") + .score(950_000L, "SPAN").span(GenAITraceRefType.OTLP, TRACE_2, OTLP_SPAN), + new Evaluation("g5", MIDNIGHT + 30_000L, CHAT, "anthropic", "claude", "safety", "judge-b") + .bool(false, "SPAN").span(GenAITraceRefType.ZIPKIN, "trace-3", "0000000000000001"), + new Evaluation("g6", MIDNIGHT + 70_000L, CHAT, OPENAI, "gpt-4o", "tone", "judge-a") + .string("polite", "SPAN").span(GenAITraceRefType.OTLP, TRACE_2, "a1b2c3d4e5f60719"), + new Evaluation("g7", MIDNIGHT + 110_000L, CHAT, OPENAI, "gpt-4o", "relevance", "judge-a") + .score(600_000L, "SPAN").nativeSpan("trace-4", "segment-4", 0), + bound("before-start", BOUNDED.getStartTimestamp() - 1), + bound("on-start", BOUNDED.getStartTimestamp()), + bound("last-millisecond", BOUNDED.getEndTimestamp() - 1), + bound("after-end", BOUNDED.getEndTimestamp() + 1), + bound("on-end", ENDING.getEndTimestamp()), + bound("on-midnight", ENDING_AT_MIDNIGHT.getEndTimestamp()) + ); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private GenAIEvaluationRecordQueryEsDAO dao; + + GenAIEvaluationQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeEvaluations() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final Model model = support.installRecord(GenAIEvaluationRecord.class); + support.writeRecords(model, new GenAIEvaluationRecord.Builder(), EVALUATIONS.stream() + .map(Evaluation::record).collect(Collectors.toList())); + dao = new GenAIEvaluationRecordQueryEsDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheEvaluationsOfAServiceNewestOrOldestFirst() throws IOException { + assertEquals(Arrays.asList("g7", "g6", "g5", "g3", "g2", "g1"), ids(read().service(CHAT))); + assertEquals(Arrays.asList("g1", "g2", "g3", "g5", "g6", "g7"), ids(read().service(CHAT).order(Order.ASC))); + } + + @Test + public void readsTheEvaluationsOfAProviderAndOfAModel() throws IOException { + assertEquals(Arrays.asList("g7", "g6", "g3", "g2", "g1"), ids(read().provider(OPENAI))); + assertEquals(Arrays.asList("g7", "g6", "g2", "g1"), ids(read().model(OPENAI, "gpt-4o"))); + } + + @Test + public void readsTheEvaluationsOfAValueType() throws IOException { + assertEquals(Arrays.asList("g7", "g4", "g3", "g1"), ids(read().valueType(GenAIEvaluationValueType.SCORE))); + assertEquals(Collections.singletonList("g6"), ids(read().valueType(GenAIEvaluationValueType.STRING))); + } + + /** + * The query service reads score bounds and the score order of scores only, and a boolean value of booleans + * only. A bound lies on a score. + */ + @Test + public void readsTheScoresWithinTheBoundsAndTheBooleansOfAValue() throws IOException { + assertEquals(Arrays.asList("g7", "g4", "g1"), ids(read().scores(600_000L, null))); + assertEquals(Arrays.asList("g7", "g3", "g1"), ids(read().scores(null, 800_000L))); + assertEquals(Arrays.asList("g7", "g1"), ids(read().scores(500_000L, 900_000L))); + assertEquals(Collections.singletonList("g2"), ids(read().bool(true))); + assertEquals(Collections.singletonList("g5"), ids(read().bool(false))); + } + + @Test + public void readsTheScoresHighestOrLowestFirst() throws IOException { + assertEquals(Arrays.asList("g4", "g1", "g7", "g3"), ids(read().scores(null, null).byScore())); + assertEquals(Arrays.asList("g3", "g7", "g1", "g4"), ids(read().scores(null, null).byScore() + .order(Order.ASC))); + } + + @Test + public void readsTheEvaluationsOfATaskALevelAndAJudgeModel() throws IOException { + assertEquals(Arrays.asList("g7", "g4", "g3", "g1"), ids(read().task("relevance"))); + assertEquals(Collections.singletonList("g2"), ids(read().level("TRACE"))); + assertEquals(Arrays.asList("g5", "g3"), ids(read().judge("judge-b"))); + } + + @Test + public void readsTheEvaluationsOfANativeTraceSegmentAndSpan() throws IOException { + assertEquals(Arrays.asList("g3", "g2", "g1"), ids(read().trace(GenAITraceRefType.SKYWALKING_NATIVE, TRACE_1, + null, null, null))); + assertEquals(Arrays.asList("g2", "g1"), ids(read().trace(GenAITraceRefType.SKYWALKING_NATIVE, TRACE_1, + "segment-1", null, null))); + assertEquals(Collections.singletonList("g2"), ids(read().trace(GenAITraceRefType.SKYWALKING_NATIVE, TRACE_1, + "segment-1", 1, null))); + } + + /** + * The trace's store is part of the reference: a Zipkin trace is not found as an OTLP one. + */ + @Test + public void readsTheEvaluationsOfAnOtlpOrZipkinTraceAndSpan() throws IOException { + assertEquals(Arrays.asList("g6", "g4"), ids(read().trace(GenAITraceRefType.OTLP, TRACE_2, null, null, null))); + assertEquals(Collections.singletonList("g4"), ids(read().trace(GenAITraceRefType.OTLP, TRACE_2, null, null, + OTLP_SPAN))); + assertEquals(Collections.singletonList("g5"), ids(read().trace(GenAITraceRefType.ZIPKIN, "trace-3", null, + null, null))); + assertEquals(Collections.emptyList(), ids(read().trace(GenAITraceRefType.OTLP, "trace-3", null, null, null))); + } + + @Test + public void pagesThroughTheEvaluationsOfAService() throws IOException { + assertEquals(Arrays.asList("g7", "g6"), ids(read().service(CHAT).page(0, 2))); + assertEquals(Arrays.asList("g5", "g3"), ids(read().service(CHAT).page(2, 2))); + assertEquals(Arrays.asList("g2", "g1"), ids(read().service(CHAT).page(4, 2))); + assertEquals(Collections.emptyList(), ids(read().service(CHAT).page(6, 2))); + } + + @Test + public void readsEachEvaluationWhole() throws IOException { + final List expected = new ArrayList<>(); + for (int i = 6; i >= 0; i--) { + expected.add(render(EVALUATIONS.get(i).record())); + } + assertEquals(expected, read().run(dao).stream().map(GenAIEvaluationQueryITBase::render) + .collect(Collectors.toList())); + } + + @Test + public void readsTheEvaluationsOfEveryDayWithoutADuration() throws IOException { + assertEquals(Arrays.asList("g3", "g2", "g1"), ids(read().trace(GenAITraceRefType.SKYWALKING_NATIVE, TRACE_1, + null, null, null).during(null))); + } + + @Test + public void keepsTheEvaluationsWithinTheDuration() throws IOException { + assertEquals(Arrays.asList("last-millisecond", "on-start"), ids(read().service("bounds-app").during(BOUNDED))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's index. + */ + @Test + public void readsAnEvaluationOnTheInstantTheDurationEndsAt() throws IOException { + assertEquals(Collections.singletonList("on-end"), ids(read().service("bounds-app").during(ENDING))); + assertEquals(Collections.singletonList("on-midnight"), ids(read().service("bounds-app") + .during(ENDING_AT_MIDNIGHT))); + } + + private List ids(final EvaluationRead read) throws IOException { + return read.run(dao).stream().map(GenAIEvaluationRecord::getUniqueId).collect(Collectors.toList()); + } + + private static EvaluationRead read() { + return new EvaluationRead(); + } + + private static String render(final GenAIEvaluationRecord record) { + final GenAITraceRef ref = record.getTraceRef(); + return String.join("|", record.getUniqueId(), record.getServiceId(), record.getProviderId(), + record.getModelId(), record.getOperationName(), String.valueOf(record.getValueType()), + String.valueOf(record.getEvalNumberValue()), record.getEvalStringValue(), + record.getTaskName(), record.getEvaluationLevel(), record.getReason(), + record.getJudgeModel(), String.valueOf(record.getEvaluationTime()), + String.valueOf(ref.getType()), ref.getTraceId(), ref.getSegmentId(), + String.valueOf(ref.getSpanIndex()), ref.getSpanId()); + } + + private static Duration minutes(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.MINUTE); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmm")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmm")); + return duration; + } + + private static Evaluation bound(final String id, final long time) { + return new Evaluation(id, time, "bounds-app", OPENAI, "gpt-4o", "relevance", "judge-a") + .score(500_000L, "SPAN").nativeSpan("trace-" + id, "segment-" + id, 0); + } + + /** + * The arguments of one read, as the GenAI evaluation query service passes them after its checks: score bounds + * and the score order ask for scores, a boolean value for booleans. + */ + private static final class EvaluationRead { + private String serviceId; + private String providerId; + private String modelId; + private GenAIEvaluationValueType valueType; + private Long minScore; + private Long maxScore; + private Boolean booleanValue; + private GenAIEvaluationRecordSortBy sortBy = GenAIEvaluationRecordSortBy.EVALUATION_TIME; + private String taskName; + private String evaluationLevel; + private String judgeModel; + private GenAITraceScopeCondition trace; + private Order order = Order.DES; + private int from = 0; + private int limit = 100; + private Duration duration = AROUND_MIDNIGHT; + + EvaluationRead service(final String service) { + serviceId = GenAIEvaluationRecord.toServiceId(service); + return this; + } + + EvaluationRead provider(final String provider) { + providerId = GenAIEvaluationRecord.toEntityId(provider); + return this; + } + + EvaluationRead model(final String provider, final String model) { + modelId = IDManager.ServiceInstanceID.buildId(GenAIEvaluationRecord.toEntityId(provider), model); + return this; + } + + EvaluationRead valueType(final GenAIEvaluationValueType valueType) { + this.valueType = valueType; + return this; + } + + EvaluationRead scores(final Long min, final Long max) { + valueType = GenAIEvaluationValueType.SCORE; + minScore = min; + maxScore = max; + return this; + } + + EvaluationRead bool(final boolean value) { + valueType = GenAIEvaluationValueType.BOOLEAN; + booleanValue = value; + return this; + } + + EvaluationRead byScore() { + sortBy = GenAIEvaluationRecordSortBy.SCORE_VALUE; + return this; + } + + EvaluationRead task(final String taskName) { + this.taskName = taskName; + return this; + } + + EvaluationRead level(final String evaluationLevel) { + this.evaluationLevel = evaluationLevel; + return this; + } + + EvaluationRead judge(final String judgeModel) { + this.judgeModel = judgeModel; + return this; + } + + EvaluationRead trace(final GenAITraceRefType type, final String traceId, final String segmentId, + final Integer spanIndex, final String spanId) { + trace = new GenAITraceScopeCondition(); + trace.setType(type); + trace.setTraceId(traceId); + trace.setSegmentId(segmentId); + trace.setSpanIndex(spanIndex); + trace.setSpanId(spanId); + return this; + } + + EvaluationRead order(final Order order) { + this.order = order; + return this; + } + + EvaluationRead page(final int from, final int limit) { + this.from = from; + this.limit = limit; + return this; + } + + EvaluationRead during(final Duration duration) { + this.duration = duration; + return this; + } + + List run(final GenAIEvaluationRecordQueryEsDAO dao) throws IOException { + return dao.queryGenAIEvaluationRecord(serviceId, providerId, modelId, valueType, minScore, maxScore, + booleanValue, sortBy, taskName, evaluationLevel, judgeModel, trace, + order, from, limit, duration) + .getGenAIEvaluationRecordList(); + } + } + + /** + * A GenAI evaluation as the span evaluation hands it to the record stream. + */ + private static final class Evaluation { + private final GenAIEvaluationRecord record; + + Evaluation(final String id, final long time, final String service, final String provider, final String model, + final String task, final String judge) { + record = GenAIEvaluationRecord.create(provider, model); + record.setUniqueId(id); + record.setServiceName(service); + record.setOperationName("chat"); + record.setTaskName(task); + record.setJudgeModel(judge); + record.setReason("reason of " + id); + record.setEvaluationTime(time); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + } + + Evaluation score(final long scorePpm, final String level) { + record.setValueType(GenAIEvaluationValueType.SCORE.name()); + record.setEvalNumberValue(scorePpm); + record.setEvaluationLevel(level); + return this; + } + + Evaluation bool(final boolean value, final String level) { + record.setValueType(GenAIEvaluationValueType.BOOLEAN.name()); + record.setEvalNumberValue(value ? GenAIEvaluationRecord.SCORE_SCALE : 0L); + record.setEvaluationLevel(level); + return this; + } + + Evaluation string(final String value, final String level) { + record.setValueType(GenAIEvaluationValueType.STRING.name()); + record.setEvalStringValue(value); + record.setEvaluationLevel(level); + return this; + } + + Evaluation nativeSpan(final String traceId, final String segmentId, final int spanIndex) { + record.setRefType(GenAITraceRefType.SKYWALKING_NATIVE.name()); + record.setTraceId(traceId); + record.setSegmentId(segmentId); + record.setSpanIndex(spanIndex); + return this; + } + + Evaluation span(final GenAITraceRefType type, final String traceId, final String spanId) { + record.setRefType(type.name()); + record.setTraceId(traceId); + record.setSpanId(spanId); + return this; + } + + GenAIEvaluationRecord record() { + return record; + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/HierarchyQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/HierarchyQueryITBase.java new file mode 100644 index 000000000000..7ac6381c8092 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/HierarchyQueryITBase.java @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.hierarchy.instance.InstanceHierarchyRelationTraffic; +import org.apache.skywalking.oap.server.core.hierarchy.service.ServiceHierarchyRelationTraffic; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Reads service and instance hierarchy relations the way the hierarchy query service does, through + * {@link HierarchyQueryEsDAO}: every service relation, and the relations of an instance in the layer it is read in, + * found from either side of the relation, both scrolled page by page. A nested class of a server's IT names the + * server and the storage mode. + * + *

A hierarchy relation is traffic: it stays in the daily index it was first stored in, and the relations here + * were first stored on both sides of midnight, so every read spans two daily indices. One instance id is in three + * layers, as a service of one name is in each layer it reports to, so it is the upper instance of relations in two + * layers and the lower instance of a relation in a third. The near misses are an instance of the same service with + * relations of its own, and reads in a layer the instance has relations in on the other side only. One pod lies + * below instances in more layers than a search returns by default. In the default mode the service and the instance + * relations share the merged metrics index; neither model has a routing column, so the sharded mode routes them by + * id over both shards. Neither read promises an order, so the relations are compared sorted. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class HierarchyQueryITBase { + /** A fixed local midnight, so the daily indices the relations are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + /** Fewer than the service relations, so reading them scrolls through pages. */ + private static final int SCROLLING_BATCH_SIZE = 2; + /** Readable names of the ids the fixture builds, to print the relations read. */ + private static final Map NAMES = new HashMap<>(); + + private static final String CHECKOUT = service("checkout"); + private static final String CHECKOUT_POD = service("checkout.default"); + private static final String MYSQL = service("mysql::orders-db"); + private static final String MYSQL_POD = service("orders-db.default"); + private static final String ORDERS_DB = service("orders-db.default:3306"); + + private static final String CHECKOUT_0 = instance(CHECKOUT, "pod-0"); + private static final String CHECKOUT_1 = instance(CHECKOUT, "pod-1"); + private static final String CHECKOUT_POD_0 = instance(CHECKOUT_POD, "pod-0"); + private static final String CHECKOUT_POD_1 = instance(CHECKOUT_POD, "pod-1"); + private static final String EDGE_0 = instance(service("edge"), "pod-0"); + private static final String EDGE_POD_0 = instance(service("edge.default"), "pod-0"); + /** Every layer the edge service reports to, each above K8S_SERVICE: more than a search returns by default. */ + private static final List EDGE_LAYERS = Arrays.asList( + Layer.GENERAL, Layer.MESH, Layer.MESH_DP, Layer.MYSQL, Layer.POSTGRESQL, Layer.APISIX, Layer.NGINX, + Layer.SO11Y_OAP, Layer.ROCKETMQ, Layer.RABBITMQ, Layer.KAFKA, Layer.CLICKHOUSE); + + private static final List SERVICE_RELATIONS = sorted( + "checkout GENERAL -> checkout.default K8S_SERVICE", + "checkout MESH -> checkout MESH_DP", + "checkout MESH -> checkout.default K8S_SERVICE", + "mysql::orders-db MYSQL -> orders-db.default K8S_SERVICE", + "orders-db.default:3306 VIRTUAL_DATABASE -> mysql::orders-db MYSQL"); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private HierarchyQueryEsDAO dao; + + HierarchyQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeRelations() throws Exception { + support = ElasticSearchQueryITSupport.connect( + server, mode, getClass(), config -> config.setScrollingBatchSize(SCROLLING_BATCH_SIZE)); + // As at boot, both models are installed before anything is written: a daily index is created from the + // index template at its first write, and maps only the columns the template had then. + final Model serviceModel = support.installMetrics(ServiceHierarchyRelationTraffic.class); + final Model instanceModel = support.installMetrics(InstanceHierarchyRelationTraffic.class); + final long yesterday = MIDNIGHT - 2 * 3_600_000L; + final long today = MIDNIGHT + 20 * 60_000L; + + support.writeMetrics( + serviceModel, + new ServiceHierarchyRelationTraffic.Builder(), + Arrays.asList( + services(CHECKOUT, Layer.GENERAL, CHECKOUT_POD, Layer.K8S_SERVICE, yesterday), + services(CHECKOUT, Layer.MESH, CHECKOUT_POD, Layer.K8S_SERVICE, today), + services(CHECKOUT, Layer.MESH, CHECKOUT, Layer.MESH_DP, today), + services(MYSQL, Layer.MYSQL, MYSQL_POD, Layer.K8S_SERVICE, today), + services(ORDERS_DB, Layer.VIRTUAL_DATABASE, MYSQL, Layer.MYSQL, yesterday))); + + support.writeMetrics( + instanceModel, + new InstanceHierarchyRelationTraffic.Builder(), + Arrays.asList( + instances(CHECKOUT_0, Layer.GENERAL, CHECKOUT_POD_0, Layer.K8S_SERVICE, yesterday), + instances(CHECKOUT_0, Layer.MESH, CHECKOUT_POD_0, Layer.K8S_SERVICE, today), + instances(CHECKOUT_0, Layer.MESH, CHECKOUT_0, Layer.MESH_DP, today), + instances(CHECKOUT_1, Layer.GENERAL, CHECKOUT_POD_1, Layer.K8S_SERVICE, today))); + final List edge = new ArrayList<>(); + for (int i = 0; i < EDGE_LAYERS.size(); i++) { + edge.add(instances( + EDGE_0, EDGE_LAYERS.get(i), EDGE_POD_0, Layer.K8S_SERVICE, i % 2 == 0 ? yesterday : today)); + } + support.writeMetrics(instanceModel, new InstanceHierarchyRelationTraffic.Builder(), edge); + + dao = new HierarchyQueryEsDAO(support.getClient(), support.getConfig()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + /** + * A scroll context left open is held for the scroll's keep-alive, and a node refuses every new scroll, of any + * query, past {@code search.max_open_scroll_context}. + */ + @Test + public void scrollsThroughEveryServiceRelationOfEveryDailyIndex() throws Exception { + final long openBefore = support.openScrollContexts(); + assertEquals(SERVICE_RELATIONS, serviceRelations(dao)); + final long openAfter = support.openScrollContexts(); + assertTrue(openAfter <= openBefore, + "scroll contexts open before the read: " + openBefore + ", after it: " + openAfter); + } + + /** + * The limit falls inside the second page. + */ + @Test + public void readsNoMoreRelationsThanTheLimit() throws Exception { + final StorageModuleElasticsearchConfig config = new StorageModuleElasticsearchConfig(); + config.setMetadataQueryMaxSize(3); + config.setScrollingBatchSize(SCROLLING_BATCH_SIZE); + final HierarchyQueryEsDAO limitedDAO = new HierarchyQueryEsDAO(support.getClient(), config); + final long openBefore = support.openScrollContexts(); + final List services = serviceRelations(limitedDAO); + final List instances = instanceRelations(limitedDAO, EDGE_POD_0, Layer.K8S_SERVICE); + final long openAfter = support.openScrollContexts(); + + assertEquals(3, services.size(), services.toString()); + assertEquals(3, services.stream().distinct().count(), services.toString()); + assertTrue(SERVICE_RELATIONS.containsAll(services), services.toString()); + assertEquals(3, instances.size(), instances.toString()); + assertEquals(3, instances.stream().distinct().count(), instances.toString()); + assertTrue(edgeRelations().containsAll(instances), instances.toString()); + assertTrue(openAfter <= openBefore, + "scroll contexts open before the read: " + openBefore + ", after it: " + openAfter); + } + + @Test + public void readsTheRelationsOfAnInstanceFromEitherSideInItsLayer() { + assertEquals( + sorted("checkout/pod-0 GENERAL -> checkout.default/pod-0 K8S_SERVICE"), + instanceRelations(CHECKOUT_0, Layer.GENERAL)); + assertEquals( + sorted( + "checkout/pod-0 MESH -> checkout.default/pod-0 K8S_SERVICE", + "checkout/pod-0 MESH -> checkout/pod-0 MESH_DP"), + instanceRelations(CHECKOUT_0, Layer.MESH)); + assertEquals( + sorted("checkout/pod-0 MESH -> checkout/pod-0 MESH_DP"), + instanceRelations(CHECKOUT_0, Layer.MESH_DP)); + assertEquals( + sorted( + "checkout/pod-0 GENERAL -> checkout.default/pod-0 K8S_SERVICE", + "checkout/pod-0 MESH -> checkout.default/pod-0 K8S_SERVICE"), + instanceRelations(CHECKOUT_POD_0, Layer.K8S_SERVICE)); + } + + @Test + public void scrollsThroughEveryRelationOfAnInstanceBelowManyLayers() throws Exception { + final long openBefore = support.openScrollContexts(); + assertEquals(edgeRelations(), instanceRelations(EDGE_POD_0, Layer.K8S_SERVICE)); + assertEquals( + sorted("edge/pod-0 KAFKA -> edge.default/pod-0 K8S_SERVICE"), instanceRelations(EDGE_0, Layer.KAFKA)); + final long openAfter = support.openScrollContexts(); + assertTrue(openAfter <= openBefore, + "scroll contexts open before the read: " + openBefore + ", after it: " + openAfter); + } + + @Test + public void readsNoRelationOfAnInstanceInTheLayerOfItsOtherSide() { + assertEquals(Collections.emptyList(), instanceRelations(CHECKOUT_0, Layer.K8S_SERVICE)); + assertEquals(Collections.emptyList(), instanceRelations(CHECKOUT_POD_0, Layer.GENERAL)); + } + + private List instanceRelations(final String instanceId, final Layer layer) { + return instanceRelations(dao, instanceId, layer); + } + + private static List instanceRelations(final HierarchyQueryEsDAO dao, final String instanceId, + final Layer layer) { + return dao.readInstanceHierarchyRelations(instanceId, layer.name()).stream() + .map(r -> name(r.getInstanceId()) + " " + r.getServiceLayer() + " -> " + + name(r.getRelatedInstanceId()) + " " + r.getRelatedServiceLayer()) + .sorted() + .collect(Collectors.toList()); + } + + private static List serviceRelations(final HierarchyQueryEsDAO dao) { + return dao.readAllServiceHierarchyRelations().stream() + .map(r -> name(r.getServiceId()) + " " + r.getServiceLayer() + " -> " + + name(r.getRelatedServiceId()) + " " + r.getRelatedServiceLayer()) + .sorted() + .collect(Collectors.toList()); + } + + private static List edgeRelations() { + return EDGE_LAYERS.stream() + .map(layer -> "edge/pod-0 " + layer + " -> edge.default/pod-0 K8S_SERVICE") + .sorted() + .collect(Collectors.toList()); + } + + private static List sorted(final String... relations) { + return Arrays.stream(relations).sorted().collect(Collectors.toList()); + } + + private static String name(final String id) { + return NAMES.getOrDefault(id, id); + } + + private static String service(final String name) { + final String id = IDManager.ServiceID.buildId(name, true); + NAMES.put(id, name); + return id; + } + + private static String instance(final String serviceId, final String name) { + final String id = IDManager.ServiceInstanceID.buildId(serviceId, name); + NAMES.put(id, NAMES.get(serviceId) + "/" + name); + return id; + } + + private static ServiceHierarchyRelationTraffic services(final String upper, final Layer upperLayer, + final String lower, final Layer lowerLayer, + final long firstStored) { + final ServiceHierarchyRelationTraffic relation = new ServiceHierarchyRelationTraffic(); + relation.setServiceId(upper); + relation.setServiceLayer(upperLayer); + relation.setRelatedServiceId(lower); + relation.setRelatedServiceLayer(lowerLayer); + relation.setTimeBucket(TimeBucket.getMinuteTimeBucket(firstStored)); + return relation; + } + + private static InstanceHierarchyRelationTraffic instances(final String upper, final Layer upperLayer, + final String lower, final Layer lowerLayer, + final long firstStored) { + final InstanceHierarchyRelationTraffic relation = new InstanceHierarchyRelationTraffic(); + relation.setInstanceId(upper); + relation.setServiceLayer(upperLayer); + relation.setRelatedInstanceId(lower); + relation.setRelatedServiceLayer(lowerLayer); + relation.setTimeBucket(TimeBucket.getMinuteTimeBucket(firstStored)); + return relation; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/JFRDataQueryEsDAOTest.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/JFRDataQueryEsDAOTest.java index 6376bd75556a..129e3cf8c4d2 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/JFRDataQueryEsDAOTest.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/JFRDataQueryEsDAOTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.skywalking.library.elasticsearch.requests.search.Search; +import org.apache.skywalking.library.elasticsearch.requests.search.SearchParams; import org.apache.skywalking.library.elasticsearch.response.search.SearchResponse; import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.JFRProfilingDataRecord; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; @@ -69,28 +70,29 @@ void tearDown() throws Exception { @Test void getByTaskIdAndInstancesAndEvent_inMergedTable_shouldIncludeTableNameFilter() throws Exception { - when(client.search(anyString(), any(Search.class))).thenReturn(new SearchResponse()); + when(client.search(anyString(), any(Search.class), any(SearchParams.class))).thenReturn(new SearchResponse()); dao.getByTaskIdAndInstancesAndEvent("task-id-1", Collections.emptyList(), "EXECUTION_SAMPLE"); final ArgumentCaptor searchCaptor = ArgumentCaptor.forClass(Search.class); - verify(client).search(anyString(), searchCaptor.capture()); + verify(client).search(anyString(), searchCaptor.capture(), any(SearchParams.class)); final String json = objectMapper.writeValueAsString(searchCaptor.getValue()); assertThat(json).contains(IndexController.LogicIndicesRegister.RECORD_TABLE_NAME); assertThat(json).contains(JFRProfilingDataRecord.INDEX_NAME); assertThat(json).contains(JFRProfilingDataRecord.TASK_ID); assertThat(json).contains(JFRProfilingDataRecord.EVENT_TYPE); + assertThat(searchCaptor.getValue().getSize()).isEqualTo(1); } @Test void getByTaskIdAndInstancesAndEvent_withInstanceIds_shouldIncludeInstanceIdFilter() throws Exception { - when(client.search(anyString(), any(Search.class))).thenReturn(new SearchResponse()); + when(client.search(anyString(), any(Search.class), any(SearchParams.class))).thenReturn(new SearchResponse()); dao.getByTaskIdAndInstancesAndEvent("task-id-1", Arrays.asList("inst-1", "inst-2"), "EXECUTION_SAMPLE"); final ArgumentCaptor searchCaptor = ArgumentCaptor.forClass(Search.class); - verify(client).search(anyString(), searchCaptor.capture()); + verify(client).search(anyString(), searchCaptor.capture(), any(SearchParams.class)); final String json = objectMapper.writeValueAsString(searchCaptor.getValue()); assertThat(json).contains(JFRProfilingDataRecord.INSTANCE_ID); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryITBase.java new file mode 100644 index 000000000000..97b182ef3861 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/LogQueryITBase.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.analysis.record.LongText; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.TraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.ContentType; +import org.apache.skywalking.oap.server.core.query.type.Log; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Reads logs the way the log query service does on Elasticsearch and OpenSearch, through {@link LogQueryEsDAO}: by + * service, instance, endpoint, trace, native span, OTLP span, searchable tag, and keywords the content has or must + * not have, newest or oldest first, page by page. A nested class of a server's IT names the server and the storage + * mode. + * + *

The lines are written as the log analyzer writes them, on both sides of midnight, so a read spans two daily + * indices. For each filter a line matches every other condition but that one. Lines of their own lie on and just + * outside the bounds of a duration: on its first millisecond and the one before, on its last millisecond, on the + * instant it ends at ({@link Duration#getEndTimestamp()}, which a read includes) and the millisecond after. + * One more lies on the end instant of a duration ending in a day's last minute: the next midnight, in the next + * day's index. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class LogQueryITBase { + /** A fixed local midnight, so the daily indices the lines are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final long MINUTE = 60_000L; + /** Every line of the filter reads, which lie within three minutes of midnight. */ + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + private static final Duration ENDING = minutes(MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE); + /** The midnight a day before {@link #MIDNIGHT}. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = minutes(PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE); + /** The tag keys OAP's default {@code searchableLogsTags} indexes; the others are kept in the raw tags only. */ + private static final List SEARCHABLE_TAG_KEYS = Arrays.asList("level", "http.status_code"); + + private static final String CHECKOUT = "checkout"; + private static final String PAYMENT = "payment"; + private static final String BOUNDS = "bounds"; + private static final String POST_ORDERS = "POST /orders"; + private static final String TRACE_A = "trace-a"; + private static final String TRACE_B = "trace-b"; + private static final String OTLP_SPAN = "00f067aa0ba902b7"; + + private static final List LINES = Arrays.asList( + new Line("l1", MIDNIGHT - 150_000L, CHECKOUT, "checkout-1", POST_ORDERS) + .nativeSpan(TRACE_A, "segment-a1", 1).text("order created").tags("level=INFO", "thread=main"), + new Line("l2", MIDNIGHT - 90_000L, CHECKOUT, "checkout-1", POST_ORDERS) + .nativeSpan(TRACE_A, "segment-a1", 2).text("payment timeout for order") + .tags("level=ERROR", "http.status_code=500"), + new Line("l3", MIDNIGHT - 30_000L, CHECKOUT, "checkout-2", POST_ORDERS) + .nativeSpan(TRACE_A, "segment-a2", 1).text("order was created").tags("level=ERROR"), + new Line("l4", MIDNIGHT + 30_000L, CHECKOUT, "checkout-1", "GET /orders") + .otlpSpan(TRACE_B, OTLP_SPAN).text("order created again").tags("level=ERROR", "http.status_code=500"), + new Line("l5", MIDNIGHT + 90_000L, CHECKOUT, "checkout-1", null) + .otlpSpan(TRACE_B, "00f067aa0ba902b8").json("{\"event\":\"order created\"}"), + new Line("l6", MIDNIGHT + 150_000L, PAYMENT, "payment-1", "POST /pay") + .nativeSpan("trace-c", "segment-c1", 1).text("order created").tags("level=ERROR", "http.status_code=500"), + new Line("before-start", BOUNDED.getStartTimestamp() - 1, BOUNDS, "bounds-1", null).text("health check"), + new Line("on-start", BOUNDED.getStartTimestamp(), BOUNDS, "bounds-1", null).text("health check"), + new Line("last-millisecond", BOUNDED.getEndTimestamp() - 1, BOUNDS, "bounds-1", null).text("health check"), + new Line("after-end", BOUNDED.getEndTimestamp() + 1, BOUNDS, "bounds-1", null).text("health check"), + new Line("on-end", ENDING.getEndTimestamp(), BOUNDS, "bounds-1", null).text("health check"), + new Line("on-midnight", ENDING_AT_MIDNIGHT.getEndTimestamp(), BOUNDS, "bounds-1", null).text("health check") + ); + private static final Map ID_BY_TIME = new HashMap<>(); + + static { + LINES.forEach(line -> ID_BY_TIME.put(line.time, line.id)); + } + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private LogQueryEsDAO dao; + + LogQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeLines() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final Model model = support.installRecord(LogRecord.class); + support.writeRecords(model, new LogRecord.Builder(), LINES.stream() + .map(Line::record).collect(Collectors.toList())); + dao = new LogQueryEsDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheLogsOfAServiceNewestOrOldestFirst() throws IOException { + assertEquals(Arrays.asList("l5", "l4", "l3", "l2", "l1"), ids(read().service(CHECKOUT))); + assertEquals(Arrays.asList("l1", "l2", "l3", "l4", "l5"), ids(read().service(CHECKOUT).order(Order.ASC))); + assertEquals(Collections.singletonList("l6"), ids(read().service(PAYMENT))); + } + + @Test + public void readsTheLogsOfAnInstance() throws IOException { + assertEquals(Arrays.asList("l5", "l4", "l2", "l1"), ids(read().instance(CHECKOUT, "checkout-1"))); + } + + @Test + public void readsTheLogsOfAnEndpoint() throws IOException { + assertEquals(Arrays.asList("l3", "l2", "l1"), ids(read().endpoint(CHECKOUT, POST_ORDERS))); + } + + @Test + public void readsTheLogsOfATrace() throws IOException { + assertEquals(Arrays.asList("l3", "l2", "l1"), ids(read().trace(TRACE_A, null, null, null))); + assertEquals(Arrays.asList("l5", "l4"), ids(read().trace(TRACE_B, null, null, null))); + } + + /** + * A native span is its segment and its number within the segment: the trace's other span of the segment, and + * the span of the same number in its other segment, stay out. + */ + @Test + public void readsTheLogsOfANativeSpan() throws IOException { + assertEquals(Collections.singletonList("l1"), ids(read().trace(TRACE_A, "segment-a1", 1, null))); + assertEquals(Collections.singletonList("l2"), ids(read().trace(TRACE_A, "segment-a1", 2, null))); + } + + @Test + public void readsTheLogsOfAnOtlpSpan() throws IOException { + assertEquals(Collections.singletonList("l4"), ids(read().trace(TRACE_B, null, null, OTLP_SPAN))); + } + + /** + * Only the searchable tags are indexed, so a tag kept in the raw tags alone finds nothing. + */ + @Test + public void readsTheLogsHavingEverySearchableTag() throws IOException { + assertEquals(Arrays.asList("l6", "l4", "l3", "l2"), ids(read().tags("level=ERROR"))); + assertEquals(Arrays.asList("l6", "l4", "l2"), ids(read().tags("level=ERROR", "http.status_code=500"))); + assertEquals(Collections.emptyList(), ids(read().tags("thread=main"))); + } + + /** + * A keyword is a phrase of the content: {@code order was created} does not have {@code order created}. + */ + @Test + public void readsTheLogsHavingEveryKeywordAndNoExcludedOne() throws IOException { + assertTrue(dao.supportQueryLogsByKeywords()); + assertEquals(Arrays.asList("l6", "l5", "l4", "l1"), ids(read().keywords("order created"))); + assertEquals(Collections.singletonList("l2"), ids(read().keywords("order", "timeout"))); + assertEquals(Arrays.asList("l6", "l5", "l1"), ids(read().keywords("order created").excluding("again"))); + assertEquals(Collections.singletonList("l2"), ids(read().service(CHECKOUT).excluding("created"))); + } + + @Test + public void pagesThroughTheLogsOfAService() throws IOException { + assertEquals(Arrays.asList("l5", "l4"), ids(read().service(CHECKOUT).page(0, 2))); + assertEquals(Arrays.asList("l3", "l2"), ids(read().service(CHECKOUT).page(2, 2))); + assertEquals(Collections.singletonList("l1"), ids(read().service(CHECKOUT).page(4, 2))); + assertEquals(Collections.emptyList(), ids(read().service(CHECKOUT).page(6, 2))); + } + + @Test + public void readsEachLogWhole() throws IOException { + final List expected = new ArrayList<>(); + for (int i = 5; i >= 0; i--) { + expected.add(LINES.get(i).render()); + } + assertEquals(expected, read().run(dao).stream().map(LogQueryITBase::render).collect(Collectors.toList())); + } + + @Test + public void readsTheLogsOfEveryDayWithoutADuration() throws IOException { + assertEquals(Arrays.asList("l3", "l2", "l1"), ids(read().trace(TRACE_A, null, null, null).during(null))); + } + + @Test + public void keepsTheLogsWithinTheDuration() throws IOException { + assertEquals(Arrays.asList("last-millisecond", "on-start"), ids(read().service(BOUNDS).during(BOUNDED))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's index. + */ + @Test + public void readsALogOnTheInstantTheDurationEndsAt() throws IOException { + assertEquals(Collections.singletonList("on-end"), ids(read().service(BOUNDS).during(ENDING))); + assertEquals(Collections.singletonList("on-midnight"), ids(read().service(BOUNDS).during(ENDING_AT_MIDNIGHT))); + } + + private List ids(final LogRead read) throws IOException { + return read.run(dao).stream().map(log -> ID_BY_TIME.get(log.getTimestamp())).collect(Collectors.toList()); + } + + private static LogRead read() { + return new LogRead(); + } + + private static String render(final Log log) { + return String.join("|", log.getServiceId(), log.getServiceInstanceId(), log.getEndpointId(), + log.getEndpointName(), log.getTraceId(), String.valueOf(log.getTimestamp()), + log.getContentType().name(), log.getContent(), log.getTags().stream() + .map(tag -> tag.getKey() + "=" + tag.getValue()).collect(Collectors.joining(","))); + } + + /** + * A duration as the query protocol writes it for minute steps. + */ + private static Duration minutes(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.MINUTE); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmm")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmm")); + return duration; + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + /** + * The arguments of one read, as the log query service passes them. + */ + private static final class LogRead { + private String serviceId; + private String instanceId; + private String endpointId; + private TraceScopeCondition trace; + private Order order = Order.DES; + private int from = 0; + private int limit = 100; + private Duration duration = AROUND_MIDNIGHT; + private List tags; + private List keywords; + private List excluding; + + LogRead service(final String service) { + serviceId = serviceId(service); + return this; + } + + LogRead instance(final String service, final String instance) { + instanceId = IDManager.ServiceInstanceID.buildId(serviceId(service), instance); + return this; + } + + LogRead endpoint(final String service, final String endpoint) { + endpointId = IDManager.EndpointID.buildId(serviceId(service), endpoint); + return this; + } + + LogRead trace(final String traceId, final String segmentId, final Integer spanId, final String otlpSpanId) { + trace = new TraceScopeCondition(); + trace.setTraceId(traceId); + trace.setSegmentId(segmentId); + trace.setSpanId(spanId); + trace.setOtlpSpanId(otlpSpanId); + return this; + } + + LogRead order(final Order order) { + this.order = order; + return this; + } + + LogRead page(final int from, final int limit) { + this.from = from; + this.limit = limit; + return this; + } + + LogRead during(final Duration duration) { + this.duration = duration; + return this; + } + + LogRead tags(final String... tags) { + this.tags = Arrays.stream(tags).map(tag -> tag.split("=")).map(kv -> new Tag(kv[0], kv[1])) + .collect(Collectors.toList()); + return this; + } + + LogRead keywords(final String... keywords) { + this.keywords = Arrays.asList(keywords); + return this; + } + + LogRead excluding(final String... keywords) { + this.excluding = Arrays.asList(keywords); + return this; + } + + List run(final LogQueryEsDAO dao) throws IOException { + return dao.queryLogs(serviceId, instanceId, endpointId, trace, order, from, limit, duration, tags, + keywords, excluding).getLogs(); + } + } + + /** + * A log line as the log analyzer hands it to the record stream. + */ + private static final class Line { + private final String id; + private final long time; + private final String serviceId; + private final String instanceId; + private final String endpointId; + private final String endpoint; + private String traceId; + private String segmentId; + private int spanId; + private String otlpSpanId; + private ContentType contentType; + private String content; + private List tags = Collections.emptyList(); + + Line(final String id, final long time, final String service, final String instance, final String endpoint) { + this.id = id; + this.time = time; + this.serviceId = serviceId(service); + this.instanceId = IDManager.ServiceInstanceID.buildId(serviceId, instance); + this.endpointId = endpoint == null ? null : IDManager.EndpointID.buildId(serviceId, endpoint); + this.endpoint = endpoint; + } + + Line nativeSpan(final String traceId, final String segmentId, final int spanId) { + this.traceId = traceId; + this.segmentId = segmentId; + this.spanId = spanId; + return this; + } + + Line otlpSpan(final String traceId, final String otlpSpanId) { + this.traceId = traceId; + this.otlpSpanId = otlpSpanId; + return this; + } + + Line text(final String content) { + this.contentType = ContentType.TEXT; + this.content = content; + return this; + } + + Line json(final String content) { + this.contentType = ContentType.JSON; + this.content = content; + return this; + } + + Line tags(final String... tags) { + this.tags = Arrays.asList(tags); + return this; + } + + LogRecord record() { + final LogRecord record = new LogRecord(); + record.setUniqueId(id); + record.setServiceId(serviceId); + record.setServiceInstanceId(instanceId); + record.setEndpointId(endpointId); + record.setTraceId(traceId); + record.setTraceSegmentId(segmentId); + record.setSpanId(spanId); + record.setOtlpSpanId(otlpSpanId); + record.setContentType(contentType.value()); + record.setContent(new LongText(content)); + record.setTimestamp(time); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + if (!tags.isEmpty()) { + final LogTags.Builder raw = LogTags.newBuilder(); + tags.forEach(tag -> raw.addData(KeyStringValuePair.newBuilder() + .setKey(tag.split("=")[0]) + .setValue(tag.split("=")[1]))); + record.setTagsRawData(raw.build().toByteArray()); + } + record.setTagsInString(tags.stream() + .filter(tag -> SEARCHABLE_TAG_KEYS.contains(tag.split("=")[0])) + .collect(Collectors.toList())); + return record; + } + + String render() { + return String.join("|", serviceId, instanceId, endpointId, endpoint, traceId, String.valueOf(time), + contentType.name(), content, String.join(",", tags)); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ManagementDataITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ManagementDataITBase.java new file mode 100644 index 000000000000..eb40bfd69b44 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ManagementDataITBase.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.management.runtimerule.RuntimeRule; +import org.apache.skywalking.oap.server.core.management.ui.template.UITemplate; +import org.apache.skywalking.oap.server.core.profiling.continuous.storage.ContinuousProfilingPolicy; +import org.apache.skywalking.oap.server.core.query.input.DashboardSetting; +import org.apache.skywalking.oap.server.core.query.type.DashboardConfiguration; +import org.apache.skywalking.oap.server.core.query.type.TemplateChangeStatus; +import org.apache.skywalking.oap.server.core.storage.management.RuntimeRuleManagementDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Keeps the management data OAP stores in Elasticsearch or OpenSearch through their own DAOs: UI templates the way the + * UI management API manages them ({@link UITemplateManagementEsDAO}), continuous-profiling policies the way the policy + * page saves them and the eBPF agents read them ({@link ContinuousProfilingPolicyEsDAO}), and runtime rules the way + * the runtime-rule API and its reconciler keep them ({@link RuntimeRuleManagementEsDAO}). A nested class of a + * server's IT names the server and the storage mode. + * + *

All management data shares one index, in either storage mode, so each kind is read beside rows of the others. + * A template list is longer than a search's default page (10). Each change has its refusal: adding an id that + * exists, changing or disabling one that does not. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class ManagementDataITBase { + /** More than a search returns without an explicit size. */ + private static final int LISTED_TEMPLATES = 12; + private static final String CHECKOUT_ID = IDManager.ServiceID.buildId("checkout", true); + private static final String PAYMENT_ID = IDManager.ServiceID.buildId("payment", true); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private UITemplateManagementEsDAO templateDAO; + private RuntimeRuleManagementEsDAO ruleDAO; + private ContinuousProfilingPolicyEsDAO policyDAO; + + ManagementDataITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void install() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + support.installManagement(UITemplate.class); + support.installManagement(RuntimeRule.class); + support.installManagement(ContinuousProfilingPolicy.class); + templateDAO = new UITemplateManagementEsDAO(support.getClient(), new UITemplate.Builder()); + ruleDAO = new RuntimeRuleManagementEsDAO(support.getClient(), new RuntimeRule.Builder()); + policyDAO = new ContinuousProfilingPolicyEsDAO(support.getClient(), new ContinuousProfilingPolicy.Builder()); + + // A row of each kind beside the ones the tests read. + assertTrue(templateDAO.addTemplate(setting("neighbour", configuration("neighbour", 1))).isStatus()); + ruleDAO.save(rule("lal", "neighbour", "rules: []", RuntimeRule.STATUS_ACTIVE, 500L)); + policyDAO.savePolicy(policy(IDManager.ServiceID.buildId("neighbour", true), "{}", "uuid-neighbour")); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void addsATemplateOnlyWhenItsIdIsNew() throws Exception { + final TemplateChangeStatus added = templateDAO.addTemplate(setting("added", configuration("added", 1))); + assertEquals("added|true|null", describe(added)); + + final TemplateChangeStatus again = templateDAO.addTemplate(setting("added", configuration("added", 2))); + assertEquals("added|false|Template already exists", describe(again)); + assertEquals(template("added", configuration("added", 1), false), describe(templateDAO.getTemplate("added"))); + } + + /** + * The template list of the UI, with and without the disabled templates, is every template whole and nothing + * else of the index. + */ + @Test + public void readsEveryTemplateWithOrWithoutTheDisabledOnes() throws Exception { + final List all = new ArrayList<>(); + final List enabled = new ArrayList<>(); + for (int i = 0; i < LISTED_TEMPLATES; i++) { + final String id = String.format("listed-%02d", i); + assertTrue(templateDAO.addTemplate(setting(id, configuration(id, 1))).isStatus()); + final boolean disabled = i % 5 == 3; + if (disabled) { + assertTrue(templateDAO.disableTemplate(id).isStatus()); + } else { + enabled.add(template(id, configuration(id, 1), false)); + } + all.add(template(id, configuration(id, 1), disabled)); + } + + final List includingDisabled = templateDAO.getAllTemplates(true); + assertEquals(all, listed(includingDisabled)); + assertEquals(enabled, listed(templateDAO.getAllTemplates(false))); + assertTrue(includingDisabled.stream().allMatch(template -> template.getId() != null + && template.getConfiguration() != null), + "a rule or a policy read as a template"); + } + + @Test + public void disablesAnAddedTemplateOnly() throws Exception { + assertTrue(templateDAO.addTemplate(setting("disabled", configuration("disabled", 1))).isStatus()); + + assertEquals("disabled|true|null", describe(templateDAO.disableTemplate("disabled"))); + assertEquals(template("disabled", configuration("disabled", 1), true), + describe(templateDAO.getTemplate("disabled"))); + assertEquals("disabled|true|null", describe(templateDAO.disableTemplate("disabled"))); + + assertEquals("never-added|false|Can't find the template", describe(templateDAO.disableTemplate("never-added"))); + assertNull(templateDAO.getTemplate("never-added")); + } + + /** + * A change replaces the whole template, so it also enables a disabled one, the only way the API has to. + */ + @Test + public void changesAnAddedTemplateOnly() throws Exception { + assertEquals("not-added|false|Can't find the template", + describe(templateDAO.changeTemplate(setting("not-added", configuration("not-added", 2))))); + assertNull(templateDAO.getTemplate("not-added")); + + assertTrue(templateDAO.addTemplate(setting("changed", configuration("changed", 1))).isStatus()); + assertTrue(templateDAO.disableTemplate("changed").isStatus()); + assertEquals("changed|true|null", + describe(templateDAO.changeTemplate(setting("changed", configuration("changed", 2))))); + assertEquals(template("changed", configuration("changed", 2), false), + describe(templateDAO.getTemplate("changed"))); + } + + /** + * A service has one policy: saving it again, as the policy page does on every change, replaces the whole policy. + * The eBPF agents ask for the policies of their services, some of which have none. + */ + @Test + public void savesAPolicyPerServiceAndReplacesItOnTheNextSave() throws Exception { + final String withoutPolicy = IDManager.ServiceID.buildId("policy-less", true); + final ContinuousProfilingPolicy checkout = policy(CHECKOUT_ID, "{\"targets\":[\"ON_CPU\"]}", "uuid-1"); + final ContinuousProfilingPolicy payment = policy(PAYMENT_ID, "{\"targets\":[\"OFF_CPU\"]}", "uuid-2"); + policyDAO.savePolicy(checkout); + policyDAO.savePolicy(payment); + assertEquals(policies(checkout), policies(policyDAO.queryPolicies(Collections.singletonList(CHECKOUT_ID)))); + + final ContinuousProfilingPolicy changed = policy(CHECKOUT_ID, "{\"targets\":[\"NETWORK\"]}", "uuid-3"); + policyDAO.savePolicy(changed); + assertEquals(policies(changed, payment), + policies(policyDAO.queryPolicies(Arrays.asList(CHECKOUT_ID, PAYMENT_ID, withoutPolicy)))); + assertEquals(policies(payment), policies(policyDAO.queryPolicies(Collections.singletonList(PAYMENT_ID)))); + assertEquals(Collections.emptyList(), policyDAO.queryPolicies(Collections.singletonList(withoutPolicy))); + } + + /** + * Runtime rules are kept by catalog and name, the same name in two catalogs being two rules. A save replaces the + * whole rule, as {@code /addOrUpdate} and {@code /inactivate} do, and a delete removes the rule of one catalog, + * silently when there is none, as a retried {@code /delete} does. The reconciler reads every rule, so each read + * is compared whole, with the rules already kept. + */ + @Test + public void savesReplacesAndDeletesRulesByCatalogAndName() throws Exception { + final List kept = rules(ruleDAO.getAll()); + final String active = RuntimeRule.STATUS_ACTIVE; + final RuntimeRule vm = rule("otel-rules", "vm", "metricPrefix: vm", active, 1_000L); + final RuntimeRule logVm = rule("log-mal-rules", "vm", "metricPrefix: log_vm", active, 1_001L); + final RuntimeRule mysql = rule("otel-rules", "mysql", "metricPrefix: mysql", active, 1_002L); + for (final RuntimeRule rule : Arrays.asList(vm, logVm, mysql)) { + ruleDAO.save(rule); + } + assertEquals(rules(kept, vm, logVm, mysql), rules(ruleDAO.getAll())); + + final RuntimeRule inactiveVm = + rule("otel-rules", "vm", "metricPrefix: vm_v2", RuntimeRule.STATUS_INACTIVE, 2_000L); + ruleDAO.save(inactiveVm); + assertEquals(rules(kept, inactiveVm, logVm, mysql), rules(ruleDAO.getAll())); + + ruleDAO.delete("log-mal-rules", "vm"); + assertEquals(rules(kept, inactiveVm, mysql), rules(ruleDAO.getAll())); + ruleDAO.delete("log-mal-rules", "vm"); + ruleDAO.delete("otel-rules", "mysql"); + assertEquals(rules(kept, inactiveVm), rules(ruleDAO.getAll())); + } + + /** + * A rule's name is its path under the catalog, so it may hold a '/'. + */ + @Test + public void savesAndDeletesARuleNamedByAPath() throws Exception { + final List kept = rules(ruleDAO.getAll()); + final RuntimeRule gateway = rule("otel-rules", "aws-gateway/gateway-service", "metricPrefix: aws_gateway", + RuntimeRule.STATUS_ACTIVE, 3_000L); + ruleDAO.save(gateway); + assertEquals(rules(kept, gateway), rules(ruleDAO.getAll())); + + ruleDAO.delete("otel-rules", "aws-gateway/gateway-service"); + assertEquals(kept, rules(ruleDAO.getAll())); + } + + private static List listed(final List templates) { + return templates.stream() + .filter(template -> template.getId() != null && template.getId().startsWith("listed-")) + .map(ManagementDataITBase::describe) + .sorted() + .collect(Collectors.toList()); + } + + private static String describe(final DashboardConfiguration template) { + return template(template.getId(), template.getConfiguration(), template.isDisabled()); + } + + private static String template(final String id, final String configuration, final boolean disabled) { + return id + "|" + configuration + "|" + disabled; + } + + private static String describe(final TemplateChangeStatus status) { + return status.getId() + "|" + status.isStatus() + "|" + status.getMessage(); + } + + private static DashboardSetting setting(final String id, final String configuration) { + final DashboardSetting setting = new DashboardSetting(); + setting.setId(id); + setting.setConfiguration(configuration); + return setting; + } + + private static String configuration(final String name, final int version) { + return "{\"name\":\"" + name + "\",\"version\":" + version + ",\"children\":[]}"; + } + + private static ContinuousProfilingPolicy policy(final String serviceId, final String configuration, + final String uuid) { + final ContinuousProfilingPolicy policy = new ContinuousProfilingPolicy(); + policy.setServiceId(serviceId); + policy.setConfigurationJson(configuration); + policy.setUuid(uuid); + return policy; + } + + private static List policies(final ContinuousProfilingPolicy... policies) { + return policies(Arrays.asList(policies)); + } + + private static List policies(final List policies) { + return policies.stream() + .map(policy -> String.join("|", policy.getServiceId(), policy.getConfigurationJson(), + policy.getUuid())) + .sorted() + .collect(Collectors.toList()); + } + + private static RuntimeRule rule(final String catalog, final String name, final String content, + final String status, final long updateTime) { + final RuntimeRule rule = new RuntimeRule(); + rule.setCatalog(catalog); + rule.setName(name); + rule.setContent(content); + rule.setStatus(status); + rule.setUpdateTime(updateTime); + return rule; + } + + private static List rules(final List kept, final RuntimeRule... rules) { + final List all = new ArrayList<>(kept); + all.addAll(rules(rules)); + Collections.sort(all); + return all; + } + + private static List rules(final RuntimeRule... rules) { + return Arrays.stream(rules) + .map(rule -> String.join("|", rule.getCatalog(), rule.getName(), rule.getContent(), + rule.getStatus(), String.valueOf(rule.getUpdateTime()))) + .sorted() + .collect(Collectors.toList()); + } + + private static List rules(final List files) { + return files.stream() + .map(file -> String.join("|", file.getCatalog(), file.getName(), file.getContent(), + file.getStatus(), String.valueOf(file.getUpdateTime()))) + .sorted() + .collect(Collectors.toList()); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryITBase.java new file mode 100644 index 000000000000..fc4f206df036 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetadataQueryITBase.java @@ -0,0 +1,702 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.networkalias.NetworkAddressAlias; +import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessDetectType; +import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.process.ServiceLabelRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; +import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; +import org.apache.skywalking.oap.server.core.query.enumeration.ProfilingSupportStatus; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.Attribute; +import org.apache.skywalking.oap.server.core.query.type.Endpoint; +import org.apache.skywalking.oap.server.core.query.type.Process; +import org.apache.skywalking.oap.server.core.query.type.Service; +import org.apache.skywalking.oap.server.core.query.type.ServiceInstance; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.cache.NetworkAddressAliasEsDAO; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Reads services, instances, endpoints, the processes an eBPF agent reports, the labels of a service's processes, + * network address aliases and searchable tags on Elasticsearch and OpenSearch, the way the metadata query, the eBPF + * receiver and profiling, the alias cache and tag autocompletion do. A nested class of a server's IT names the server + * and the storage mode. + * + *

Traffic is stored in the daily index of the minute it was first seen, on both sides of midnight here, and read + * through the alias of every day; a document written again (an instance that pings, an alias that changes) is updated + * in that index. A row is created at one minute and last seen at another, so a read by a window applies both bounds: + * created no later than the window's end, last seen no earlier than its start. The window most reads use crosses + * midnight; rows sit on each of its bounds and one minute outside, and rows of another service, instance or agent + * match every other condition. Processes are read by an instance at minute, hour and day steps, as the query + * protocol allows any step. Endpoints are found newest first. The lists are scrolled two documents a page. + * + *

A tag is stored once for each day it is seen, and autocompletion reads the indices of the days the window + * touches, whatever its minutes: tags seen on both days, only on one, three days before and of the other tag type. A + * window ends with the day of its last step, though its end instant, {@link Duration#getEndTimestamp()}, opens the + * next day: honoring it would read all of that day's tags. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class MetadataQueryITBase { + private static final Gson GSON = new Gson(); + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + /** Fewer than every list holds, so each list is read over more than one scroll page. */ + private static final int SCROLLING_BATCH_SIZE = 2; + private static final String ORDERS = "orders"; + private static final String ORDERS_ID = IDManager.ServiceID.buildId(ORDERS, true); + private static final String PAYMENT = "finance::payment"; + private static final String PAYMENT_ID = IDManager.ServiceID.buildId(PAYMENT, true); + private static final String DATABASE = "mysql.orders:3306"; + private static final String ORDERS_INSTANCE = "orders-1"; + private static final String ORDERS_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(ORDERS_ID, ORDERS_INSTANCE); + private static final String PAYMENT_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(PAYMENT_ID, "payment-1"); + private static final String ORDERS_AGENT = "rover-node-2"; + /** The first and the last minute, counted from midnight, of the window most reads use. */ + private static final int WINDOW_START = -10; + private static final int WINDOW_END = 10; + /** Searchable tags as type, key, value and the minute, counted from midnight, they were seen. */ + private static final String[][] TAGS = { + {"TRACE", "http.method", "GET", "-30"}, + {"TRACE", "http.status_code", "200", "-30"}, + {"TRACE", "rpc.method", "Get", "-30"}, + {"TRACE", "http.route", "/a", "-30"}, + {"TRACE", "http.route", "/b", "-30"}, + {"TRACE", "http.method", "GET", "20"}, + {"TRACE", "http.method", "POST", "20"}, + {"TRACE", "http.status_code", "500", "20"}, + {"TRACE", "db.type", "MySQL", "20"}, + {"TRACE", "http.route", "/a", "20"}, + {"LOG", "level", "ERROR", "20"}, + {"LOG", "http.method", "PUT", "20"}, + {"TRACE", "old.key", "old", String.valueOf(-3 * 24 * 60)}, + }; + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private final Map instanceRows = new HashMap<>(); + private final Map processRows = new HashMap<>(); + private ElasticSearchQueryITSupport support; + private MetadataQueryEsDAO metadataDAO; + private TagAutoCompleteQueryDAO tagDAO; + private NetworkAddressAliasEsDAO aliasDAO; + private ServiceLabelEsDAO labelDAO; + private Duration window; + private Duration aroundMidnight; + private Duration afterMidnight; + + MetadataQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeTraffic() throws Exception { + support = ElasticSearchQueryITSupport.connect( + server, mode, getClass(), config -> config.setScrollingBatchSize(SCROLLING_BATCH_SIZE)); + // Every model is installed before the first write, as at boot: in the default mode they share the daily + // indices of one physical index, and an index created before a model was installed lacks its mappings. + final Model serviceModel = support.installMetrics(ServiceTraffic.class); + final Model instanceModel = support.installMetrics(InstanceTraffic.class); + final Model endpointModel = support.installMetrics(EndpointTraffic.class); + final Model processModel = support.installMetrics(ProcessTraffic.class); + final Model tagModel = support.installMetrics(TagAutocompleteData.class); + final Model aliasModel = support.installMetrics(NetworkAddressAlias.class); + final Model labelModel = support.installMetrics(ServiceLabelRecord.class); + + support.writeMetrics(serviceModel, new ServiceTraffic.Builder(), Arrays.asList( + service(ORDERS, Layer.GENERAL, -60), + service(ORDERS, Layer.MESH, 5), + service(PAYMENT, Layer.GENERAL, 5), + service(DATABASE, Layer.VIRTUAL_DATABASE, -60))); + + support.writeMetrics(instanceModel, new InstanceTraffic.Builder(), Arrays.asList( + ordersInstance("seen-across-midnight", -60, 30, "java", "hostname", "orders-host-0"), + ordersInstance("pinged-at-start", -60, WINDOW_START, "java"), + ordersInstance("pinged-before-start", -60, WINDOW_START - 1, "java"), + ordersInstance("created-at-end", WINDOW_END, 30, null), + ordersInstance("created-after-end", WINDOW_END + 1, 30, null), + ordersInstance("updated", -60, -30, "java"), + instance(PAYMENT_ID, "payment-1", -60, 30, properties("python")))); + // The metrics worker writes an instance again when it pings, with its first minute kept. + support.updateMetrics(instanceModel, new InstanceTraffic.Builder(), + Collections.singletonList(ordersInstance("updated", -60, 5, "java"))); + + support.writeMetrics(endpointModel, new EndpointTraffic.Builder(), Arrays.asList( + endpoint(ORDERS_ID, "GET:/api/orders", -60, 30), + endpoint(ORDERS_ID, "GET:/api/orders/archive", -61, WINDOW_START - 1), + endpoint(ORDERS_ID, "PATCH:/api/orders", -59, WINDOW_START), + endpoint(ORDERS_ID, "POST:/api/orders", -5, 30), + endpoint(ORDERS_ID, "GET:/api/users", 5, 30), + endpoint(ORDERS_ID, "DELETE:/api/orders", WINDOW_END, WINDOW_END), + endpoint(ORDERS_ID, "PUT:/api/orders", WINDOW_END + 1, 30), + endpoint(PAYMENT_ID, "GET:/api/orders", -60, 30))); + + final JsonObject host = new JsonObject(); + host.addProperty("host_name", "node-0"); + host.addProperty("pid", "1001"); + support.writeMetrics(processModel, new ProcessTraffic.Builder(), Arrays.asList( + ordersProcess("first-seen-yesterday", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + -60, 30, host, "env:prod", "team:a"), + ordersProcess("first-seen-today", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + 5, 30, null), + ordersProcess("virtual", ProcessDetectType.VIRTUAL, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + -60, 30, null), + ordersProcess("not-supported", ProcessDetectType.KUBERNETES, ProfilingSupportStatus.NOT_SUPPORT, + -60, 30, null), + ordersProcess("pinged-at-start", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + -60, WINDOW_START, null), + ordersProcess("pinged-before-start", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + -60, WINDOW_START - 1, null), + ordersProcess("created-at-end", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + WINDOW_END, WINDOW_END, null), + ordersProcess("created-after-end", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + WINDOW_END + 1, 30, null), + process(PAYMENT_ID, PAYMENT_INSTANCE_ID, "rover-node-3", "payment-process", ProcessDetectType.VM, + ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, -60, 30, null))); + + final List tags = new ArrayList<>(); + for (final String[] tag : TAGS) { + tags.add(tag(tag)); + } + support.writeMetrics(tagModel, new TagAutocompleteData.Builder(), tags); + + support.writeMetrics(aliasModel, new NetworkAddressAlias.Builder(), Arrays.asList( + alias("10.0.0.1:80", ORDERS_INSTANCE_ID, -60, -30), + alias("10.0.0.2:80", ORDERS_INSTANCE_ID, -60, WINDOW_START), + alias("10.0.0.3:80", ORDERS_INSTANCE_ID, -60, WINDOW_START - 1), + alias("10.0.0.4:80", PAYMENT_INSTANCE_ID, 5, 5))); + // An alias set up again by another instance is updated in place, with its first minute kept. + support.updateMetrics(aliasModel, new NetworkAddressAlias.Builder(), + Collections.singletonList(alias("10.0.0.1:80", PAYMENT_INSTANCE_ID, -60, 5))); + + support.writeMetrics(labelModel, new ServiceLabelRecord.Builder(), Arrays.asList( + label(ORDERS_ID, "env:prod", -60), label(ORDERS_ID, "team:a", 5), label(PAYMENT_ID, "env:test", -60))); + + metadataDAO = new MetadataQueryEsDAO(support.getClient(), support.getConfig()); + tagDAO = new TagAutoCompleteQueryDAO(support.getClient()); + aliasDAO = new NetworkAddressAliasEsDAO(support.getClient(), support.getConfig()); + labelDAO = new ServiceLabelEsDAO(support.getClient(), support.getConfig()); + window = duration(minute(WINDOW_START), minute(WINDOW_END), Step.MINUTE); + aroundMidnight = duration(minute(-30), minute(30), Step.MINUTE); + afterMidnight = duration(minute(10), minute(30), Step.MINUTE); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void listsEveryServiceOfEveryDay() { + assertEquals( + new TreeSet<>(Arrays.asList( + ORDERS_ID + "|orders|orders|null|GENERAL|true", + ORDERS_ID + "|orders|orders|null|MESH|true", + PAYMENT_ID + "|finance::payment|payment|finance|GENERAL|true", + IDManager.ServiceID.buildId(DATABASE, false) + "|" + DATABASE + "|" + DATABASE + + "|null|VIRTUAL_DATABASE|false")), + serviceRows(metadataDAO.listServices())); + } + + @Test + public void limitsTheServicesToTheMetadataQueryMaxSize() { + final StorageModuleElasticsearchConfig config = new StorageModuleElasticsearchConfig(); + config.setMetadataQueryMaxSize(3); + config.setScrollingBatchSize(SCROLLING_BATCH_SIZE); + final Set limited = serviceRows( + new MetadataQueryEsDAO(support.getClient(), config).listServices()); + assertEquals(3, limited.size()); + assertTrue(serviceRows(metadataDAO.listServices()).containsAll(limited)); + } + + @Test + public void listsTheInstancesOfAServiceAliveInTheWindow() { + assertEquals(expectedInstances("seen-across-midnight", "pinged-at-start", "created-at-end", "updated"), + instanceRows(metadataDAO.listInstances(window, ORDERS_ID))); + } + + @Test + public void listsEveryInstanceOfAServiceWithoutAWindow() { + assertEquals(expectedInstances("seen-across-midnight", "pinged-at-start", "pinged-before-start", + "created-at-end", "created-after-end", "updated"), + instanceRows(metadataDAO.listInstances(null, ORDERS_ID))); + } + + @Test + public void readsAnInstanceByIdFromEitherDay() { + assertEquals(instanceRows.get("seen-across-midnight"), + instanceRow(metadataDAO.getInstance(ordersInstanceId("seen-across-midnight")))); + assertEquals(instanceRows.get("created-at-end"), + instanceRow(metadataDAO.getInstance(ordersInstanceId("created-at-end")))); + assertNull(metadataDAO.getInstance(ordersInstanceId("missing"))); + } + + @Test + public void readsTheInstancesOfTheIds() { + assertEquals(expectedInstances("seen-across-midnight", "created-at-end", "updated"), + instanceRows(metadataDAO.getInstances(Arrays.asList( + ordersInstanceId("seen-across-midnight"), ordersInstanceId("created-at-end"), + ordersInstanceId("updated"), ordersInstanceId("missing"))))); + } + + @Test + public void findsTheEndpointsMatchingAKeywordNewestFirst() { + assertEquals(endpointRows("DELETE:/api/orders", "POST:/api/orders", "PATCH:/api/orders", "GET:/api/orders"), + endpointRows(metadataDAO.findEndpoint("orders", ORDERS_ID, 10, window))); + assertEquals(endpointRows("PUT:/api/orders", "DELETE:/api/orders", "POST:/api/orders", "PATCH:/api/orders", + "GET:/api/orders", "GET:/api/orders/archive"), + endpointRows(metadataDAO.findEndpoint("orders", ORDERS_ID, 10, null))); + // The keyword is matched against the words of the analyzed name. + assertEquals(endpointRows("GET:/api/orders/archive"), + endpointRows(metadataDAO.findEndpoint("archive", ORDERS_ID, 10, null))); + } + + @Test + public void findsTheNewestEndpointsUpToTheLimit() { + assertEquals(endpointRows("DELETE:/api/orders", "GET:/api/users"), + endpointRows(metadataDAO.findEndpoint(null, ORDERS_ID, 2, window))); + assertEquals(endpointRows("DELETE:/api/orders", "GET:/api/users", "POST:/api/orders"), + endpointRows(metadataDAO.findEndpoint(null, ORDERS_ID, 3, window))); + } + + /** + * As the eBPF profiling and continuous profiling read them: profiling-supported or every non-virtual process + * alive in the window, or, without an end, every one pinging since the start. + */ + @Test + public void listsTheProcessesOfAServiceAliveInTheWindow() { + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "pinged-at-start", "created-at-end"), + processRows(metadataDAO.listProcesses( + ORDERS_ID, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, bucket(WINDOW_START), + bucket(WINDOW_END)))); + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "not-supported", "pinged-at-start", + "created-at-end"), + processRows(metadataDAO.listProcesses(ORDERS_ID, null, bucket(WINDOW_START), bucket(WINDOW_END)))); + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "not-supported", "pinged-at-start", + "created-at-end", "created-after-end"), + processRows(metadataDAO.listProcesses(ORDERS_ID, null, bucket(WINDOW_START), 0))); + } + + @Test + public void listsTheProcessesOfAnInstanceAliveInTheWindow() { + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "not-supported", "pinged-at-start", + "created-at-end"), + processRows(metadataDAO.listProcesses(ORDERS_INSTANCE_ID, window, false))); + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "virtual", "not-supported", + "pinged-at-start", "created-at-end"), + processRows(metadataDAO.listProcesses(ORDERS_INSTANCE_ID, window, true))); + } + + /** + * The process list of the query protocol takes a duration of any step; the documents' minutes are compared with + * the minutes the duration covers, the first hour after midnight or the last before it, and that day or the day + * before. + */ + @Test + public void listsTheProcessesOfAnInstanceInHourAndDayWindows() { + final Set today = expectedProcesses( + "first-seen-yesterday", "first-seen-today", "virtual", "not-supported", "created-at-end", + "created-after-end"); + final Set yesterday = expectedProcesses( + "first-seen-yesterday", "virtual", "not-supported", "pinged-at-start", "pinged-before-start"); + assertEquals(today, processRows(metadataDAO.listProcesses( + ORDERS_INSTANCE_ID, duration(MIDNIGHT, MIDNIGHT, Step.HOUR), true))); + assertEquals(yesterday, processRows(metadataDAO.listProcesses( + ORDERS_INSTANCE_ID, duration(minute(-1), minute(-1), Step.HOUR), true))); + assertEquals(today, processRows(metadataDAO.listProcesses( + ORDERS_INSTANCE_ID, duration(MIDNIGHT, MIDNIGHT, Step.DAY), true))); + assertEquals(yesterday, processRows(metadataDAO.listProcesses( + ORDERS_INSTANCE_ID, duration(minute(-1), minute(-1), Step.DAY), true))); + } + + @Test + public void listsTheProcessesOfAnAgentAliveInTheWindow() { + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "not-supported", "pinged-at-start", + "created-at-end"), + processRows(metadataDAO.listProcesses(ORDERS_AGENT, bucket(WINDOW_START), bucket(WINDOW_END)))); + } + + @Test + public void countsTheProcessesOfAServiceAndOfAnInstance() { + assertEquals(4, metadataDAO.getProcessCount( + ORDERS_ID, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, bucket(WINDOW_START), bucket(WINDOW_END))); + // Every non-virtual process of the instance, whenever it pinged. + assertEquals(7, metadataDAO.getProcessCount(ORDERS_INSTANCE_ID)); + } + + @Test + public void readsAProcessByIdFromEitherDay() { + assertEquals( + String.join("|", IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, "first-seen-yesterday"), + "first-seen-yesterday", ORDERS_ID, ORDERS, ORDERS_INSTANCE_ID, ORDERS_INSTANCE, ORDERS_AGENT, + "VM", "SUPPORT_EBPF_PROFILING", "host_name=node-0,pid=1001", "env:prod,team:a"), + processRow(metadataDAO.getProcess( + IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, "first-seen-yesterday")))); + assertEquals(processRows.get("first-seen-today"), + processRow(metadataDAO.getProcess( + IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, "first-seen-today")))); + assertNull(metadataDAO.getProcess(IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, "missing"))); + } + + /** + * As each node's alias cache reloads them: every alias whose last update is on or after a minute, from the index + * of the day it was first set up. + */ + @Test + public void loadsTheAliasesUpdatedSinceAMinute() { + assertEquals( + Arrays.asList( + "10.0.0.1:80|" + PAYMENT_ID + "|" + PAYMENT_INSTANCE_ID + "|" + bucket(5) + "|" + bucket(-60), + "10.0.0.2:80|" + ORDERS_ID + "|" + ORDERS_INSTANCE_ID + "|" + bucket(WINDOW_START) + "|" + bucket(-60), + "10.0.0.4:80|" + PAYMENT_ID + "|" + PAYMENT_INSTANCE_ID + "|" + bucket(5) + "|" + bucket(5)), + aliasRows(aliasDAO.loadLastUpdate(bucket(WINDOW_START)))); + assertEquals(Arrays.asList("10.0.0.1:80", "10.0.0.4:80"), + aliasDAO.loadLastUpdate(bucket(5)).stream().map(NetworkAddressAlias::getAddress).sorted() + .collect(Collectors.toList())); + assertEquals(Collections.emptyList(), aliasDAO.loadLastUpdate(bucket(6))); + } + + @Test + public void readsTheLabelsOfAServiceFromEveryDay() { + assertEquals(Arrays.asList("env:prod", "team:a"), + labelDAO.queryAllLabels(ORDERS_ID).stream().sorted().collect(Collectors.toList())); + assertEquals(Collections.emptyList(), labelDAO.queryAllLabels(IDManager.ServiceID.buildId("missing", true))); + } + + @Test + public void autocompletesTheTagKeysOfTheDaysOfTheWindow() throws Exception { + assertEquals(new TreeSet<>(Arrays.asList("db.type", "http.method", "http.route", "http.status_code", + "rpc.method")), + new TreeSet<>(tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 10, aroundMidnight))); + assertEquals(new TreeSet<>(Arrays.asList("db.type", "http.method", "http.route", "http.status_code")), + new TreeSet<>(tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 10, afterMidnight))); + assertEquals(new TreeSet<>(Arrays.asList("http.method", "level")), + new TreeSet<>(tagDAO.queryTagAutocompleteKeys(TagType.LOG, 10, aroundMidnight))); + + final Set limited = tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 2, aroundMidnight); + assertEquals(2, limited.size()); + assertTrue(Arrays.asList("db.type", "http.method", "http.route", "http.status_code", "rpc.method") + .containsAll(limited)); + } + + @Test + public void autocompletesTheTagValuesOfTheDaysOfTheWindow() throws Exception { + assertEquals(new TreeSet<>(Arrays.asList("GET", "POST")), + new TreeSet<>(tagDAO.queryTagAutocompleteValues( + TagType.TRACE, "http.method", 10, aroundMidnight))); + assertEquals(new TreeSet<>(Arrays.asList("200", "500")), + new TreeSet<>(tagDAO.queryTagAutocompleteValues( + TagType.TRACE, "http.status_code", 10, aroundMidnight))); + assertEquals(Collections.singleton("500"), + tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 10, afterMidnight)); + assertEquals(Collections.singleton("PUT"), + tagDAO.queryTagAutocompleteValues(TagType.LOG, "http.method", 10, aroundMidnight)); + assertEquals(Collections.emptySet(), + tagDAO.queryTagAutocompleteValues(TagType.TRACE, "old.key", 10, aroundMidnight)); + } + + /** + * A window ending in the last minute before midnight, and a window of one day, end at the next midnight; the tags + * of that next day stay out. + */ + @Test + public void autocompletesNoTagOfTheDayAfterTheLastStep() throws Exception { + for (final Duration yesterday : Arrays.asList(duration(minute(-30), minute(-1), Step.MINUTE), + duration(minute(-30), minute(-30), Step.DAY))) { + assertEquals(new TreeSet<>(Arrays.asList("http.method", "http.route", "http.status_code", "rpc.method")), + new TreeSet<>(tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 10, yesterday))); + assertEquals(Collections.singleton("200"), + tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 10, yesterday)); + } + final Duration today = duration(minute(20), minute(20), Step.DAY); + assertEquals(new TreeSet<>(Arrays.asList("db.type", "http.method", "http.route", "http.status_code")), + new TreeSet<>(tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 10, today))); + assertEquals(Collections.singleton("500"), + tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 10, today)); + } + + /** + * A value seen on both days is stored once per day; the limit is of values, not of the documents read for them. + * Which day's documents are read first is up to the server, so one key has more values yesterday and the other + * today. + */ + @Test + public void limitsTheTagValuesOfBothDaysCountingEachValueOnce() throws Exception { + assertEquals(new TreeSet<>(Arrays.asList("GET", "POST")), + new TreeSet<>(tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.method", 2, aroundMidnight))); + assertEquals(new TreeSet<>(Arrays.asList("/a", "/b")), + new TreeSet<>(tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.route", 2, aroundMidnight))); + assertEquals(1, tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.route", 1, aroundMidnight).size()); + } + + private Set expectedInstances(final String... names) { + return Arrays.stream(names).map(instanceRows::get).collect(Collectors.toCollection(TreeSet::new)); + } + + private Set expectedProcesses(final String... names) { + return Arrays.stream(names).map(processRows::get).collect(Collectors.toCollection(TreeSet::new)); + } + + private static Set serviceRows(final List services) { + return services.stream() + .map(s -> String.join("|", s.getId(), s.getName(), s.getShortName(), + String.valueOf(s.getGroup()), + String.join(",", new TreeSet<>(s.getLayers())), + String.valueOf(s.isNormal()))) + .collect(Collectors.toCollection(TreeSet::new)); + } + + private static Set instanceRows(final List instances) { + return instances.stream().map(MetadataQueryITBase::instanceRow) + .collect(Collectors.toCollection(TreeSet::new)); + } + + private static String instanceRow(final ServiceInstance instance) { + return String.join("|", instance.getId(), instance.getName(), instance.getInstanceUUID(), + instance.getLanguage().name(), attributes(instance.getAttributes())); + } + + private static List endpointRows(final String... names) { + return Arrays.stream(names) + .map(name -> IDManager.EndpointID.buildId(ORDERS_ID, name) + "|" + name) + .collect(Collectors.toList()); + } + + private static List endpointRows(final List endpoints) { + return endpoints.stream().map(e -> e.getId() + "|" + e.getName()).collect(Collectors.toList()); + } + + private static Set processRows(final List processes) { + return processes.stream().map(MetadataQueryITBase::processRow) + .collect(Collectors.toCollection(TreeSet::new)); + } + + private static String processRow(final Process process) { + return String.join("|", process.getId(), process.getName(), process.getServiceId(), process.getServiceName(), + process.getInstanceId(), process.getInstanceName(), process.getAgentId(), + process.getDetectType(), process.getProfilingSupportStatus(), + attributes(process.getAttributes()), String.join(",", process.getLabels())); + } + + private static List aliasRows(final List aliases) { + return aliases.stream() + .map(a -> String.join("|", a.getAddress(), a.getRepresentServiceId(), + a.getRepresentServiceInstanceId(), + String.valueOf(a.getLastUpdateTimeBucket()), + String.valueOf(a.getTimeBucket()))) + .sorted() + .collect(Collectors.toList()); + } + + private static String attributes(final Collection attributes) { + return attributes.stream().map(a -> a.getName() + "=" + a.getValue()).collect(Collectors.joining(",")); + } + + private static long minute(final int minutesFromMidnight) { + return MIDNIGHT + minutesFromMidnight * 60_000L; + } + + private static long bucket(final int minutesFromMidnight) { + return TimeBucket.getMinuteTimeBucket(minute(minutesFromMidnight)); + } + + /** + * A query duration from {@code start} to {@code end}, as the query protocol writes it for {@code step}. + */ + private static Duration duration(final long start, final long end, final Step step) { + final String pattern; + switch (step) { + case MINUTE: + pattern = "yyyy-MM-dd HHmm"; + break; + case HOUR: + pattern = "yyyy-MM-dd HH"; + break; + default: + pattern = "yyyy-MM-dd"; + break; + } + final Duration duration = new Duration(); + duration.setStep(step); + duration.setStart(new DateTime(start).toString(pattern)); + duration.setEnd(new DateTime(end).toString(pattern)); + return duration; + } + + private static String ordersInstanceId(final String name) { + return IDManager.ServiceInstanceID.buildId(ORDERS_ID, name); + } + + private static ServiceTraffic service(final String name, final Layer layer, final int created) { + final ServiceTraffic service = new ServiceTraffic(); + service.setName(name); + service.setLayer(layer); + service.setTimeBucket(bucket(created)); + return service; + } + + private static InstanceTraffic instance(final String serviceId, final String name, final int created, + final int lastPing, final JsonObject properties) { + final InstanceTraffic instance = new InstanceTraffic(); + instance.setServiceId(serviceId); + instance.setName(name); + instance.setProperties(properties); + instance.setTimeBucket(bucket(created)); + instance.setLastPingTimestamp(bucket(lastPing)); + return instance; + } + + /** + * An instance of the orders service, and the row the metadata query reads for it. + * + * @param language the language the agent reports, or null for an instance that reports no properties + * @param attributes the other properties, as key and value + */ + private InstanceTraffic ordersInstance(final String name, final int created, final int lastPing, + final String language, final String... attributes) { + final List expected = new ArrayList<>(); + for (int i = 0; i < attributes.length; i += 2) { + expected.add(new Attribute(attributes[i], attributes[i + 1])); + } + final String id = ordersInstanceId(name); + instanceRows.put(name, String.join("|", id, name, id, language == null ? "UNKNOWN" : language.toUpperCase(), + attributes(expected))); + return instance(ORDERS_ID, name, created, lastPing, language == null ? null : properties(language, attributes)); + } + + private static JsonObject properties(final String language, final String... attributes) { + final JsonObject properties = new JsonObject(); + properties.addProperty(InstanceTraffic.PropertyUtil.LANGUAGE, language); + for (int i = 0; i < attributes.length; i += 2) { + properties.addProperty(attributes[i], attributes[i + 1]); + } + return properties; + } + + private static EndpointTraffic endpoint(final String serviceId, final String name, final int created, + final int lastPing) { + final EndpointTraffic endpoint = new EndpointTraffic(); + endpoint.setServiceId(serviceId); + endpoint.setName(name); + endpoint.setTimeBucket(bucket(created)); + endpoint.setLastPingTimestamp(bucket(lastPing)); + return endpoint; + } + + private static ProcessTraffic process(final String serviceId, final String instanceId, final String agentId, + final String name, final ProcessDetectType detectType, + final ProfilingSupportStatus status, final int created, final int lastPing, + final JsonObject properties, final String... labels) { + final ProcessTraffic process = new ProcessTraffic(); + process.setServiceId(serviceId); + process.setInstanceId(instanceId); + process.setName(name); + process.setAgentId(agentId); + process.setDetectType(detectType.value()); + process.setProfilingSupportStatus(status.value()); + process.setProperties(properties); + process.setLabelsJson(labels.length == 0 ? Const.EMPTY_STRING : GSON.toJson(labels)); + process.setTimeBucket(bucket(created)); + process.setLastPingTimestamp(bucket(lastPing)); + return process; + } + + /** + * A process of the orders instance, reported by its agent, and the row the metadata query reads for it. + */ + private ProcessTraffic ordersProcess(final String name, final ProcessDetectType detectType, + final ProfilingSupportStatus status, final int created, final int lastPing, + final JsonObject properties, final String... labels) { + final List expected = new ArrayList<>(); + if (properties != null) { + properties.entrySet().forEach(p -> expected.add(new Attribute(p.getKey(), p.getValue().getAsString()))); + } + processRows.put(name, String.join("|", IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, name), name, ORDERS_ID, + ORDERS, ORDERS_INSTANCE_ID, ORDERS_INSTANCE, ORDERS_AGENT, detectType.name(), + status.name(), attributes(expected), String.join(",", labels))); + return process(ORDERS_ID, ORDERS_INSTANCE_ID, ORDERS_AGENT, name, detectType, status, created, lastPing, + properties, labels); + } + + /** + * A tag as the autocompletion dispatcher stores it, under the first minute of the day it was seen. + */ + private static TagAutocompleteData tag(final String[] tag) { + final TagAutocompleteData data = new TagAutocompleteData(); + data.setTagType(tag[0]); + data.setTagKey(tag[1]); + data.setTagValue(tag[2]); + data.setTimeBucket(TimeBucket.retainToDay4MinuteBucket(bucket(Integer.parseInt(tag[3])))); + return data; + } + + private static NetworkAddressAlias alias(final String address, final String instanceId, final int created, + final int lastUpdate) { + final NetworkAddressAlias alias = new NetworkAddressAlias(); + alias.setAddress(address); + alias.setRepresentServiceId(IDManager.ServiceInstanceID.analysisId(instanceId).getServiceId()); + alias.setRepresentServiceInstanceId(instanceId); + alias.setLastUpdateTimeBucket(bucket(lastUpdate)); + alias.setTimeBucket(bucket(created)); + return alias; + } + + private static ServiceLabelRecord label(final String serviceId, final String label, final int created) { + final ServiceLabelRecord record = new ServiceLabelRecord(); + record.setServiceId(serviceId); + record.setLabel(label); + record.setTimeBucket(bucket(created)); + return record; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsPersistenceITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsPersistenceITBase.java new file mode 100644 index 000000000000..9f19812a299f --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsPersistenceITBase.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; +import org.apache.skywalking.oap.server.core.analysis.metrics.LongAvgMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.analysis.worker.MetricsSessionCache; +import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; +import org.apache.skywalking.oap.server.core.storage.SessionCacheCallback; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import org.apache.skywalking.oap.server.library.client.request.PrepareRequest; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITExt; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsDAO; +import org.joda.time.DateTime; +import org.joda.time.LocalDate; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Persists metrics the way the metrics workers do, through the metrics DAO the storage DAO hands out and the batch + * DAO, built as the storage provider builds them: a persistence round's metrics are inserted and cached once + * Elasticsearch has answered, the next round reads the stored metrics back by id ({@code multiGet}) and writes the + * combined ones as updates, and a cached metric expires with the day the TTL deletes ({@code isExpiredCache}). A + * nested class of a server's IT names the server and the storage mode. + * + *

The metrics lie on both sides of midnight, so in two daily indices. Beside them, a metric of another name of + * the same function has the same entities and minutes: OAP's default mode merges every metric into one index, and + * logic sharding merges the metrics of one function. A metadata metric, whose id has no time bucket, is stored + * before midnight and read with a later minute, as the metrics worker asks for it. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class MetricsPersistenceITBase { + /** A fixed local midnight, so the daily indices the metrics are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final String RESP_TIME = "service_resp_time"; + private static final String OTHER_RESP_TIME = "service_relation_server_resp_time"; + private static final String SERVICE_ID = IDManager.ServiceID.buildId("checkout", true); + private static final String OTHER_SERVICE_ID = IDManager.ServiceID.buildId("payment", true); + private static final int[] MINUTES = {-1, 0, 1}; + private static final int METRICS_TTL_DAYS = 7; + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private BatchProcessEsDAO batchDAO; + private StorageDAO storageDAO; + private Model respTime; + private Model traffic; + private StorageBuilder respTimeBuilder; + + MetricsPersistenceITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void install() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final Class respTimeClass = OALTestMetrics.metricsClass(RESP_TIME); + final Class otherRespTimeClass = OALTestMetrics.metricsClass(OTHER_RESP_TIME); + respTime = support.installMetrics(respTimeClass); + final Model otherRespTime = support.installMetrics(otherRespTimeClass); + traffic = support.installMetrics(ServiceTraffic.class); + respTimeBuilder = OALTestMetrics.builder(respTimeClass); + + final List sameEntities = new ArrayList<>(); + for (final int minute : MINUTES) { + for (final String entityId : Arrays.asList(SERVICE_ID, OTHER_SERVICE_ID)) { + sameEntities.add(respTime(OTHER_RESP_TIME, entityId, minute, 7L)); + } + } + support.writeMetrics(otherRespTime, OALTestMetrics.builder(otherRespTimeClass), sameEntities); + + final StorageModuleElasticsearchConfig config = support.getConfig(); + batchDAO = new BatchProcessEsDAO( + support.getClient(), config.getBulkActions(), config.getFlushInterval(), config.getConcurrentRequests(), + config.getBatchOfBytes()); + storageDAO = new StorageEsDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (batchDAO != null) { + BatchProcessEsDAOITExt.close(batchDAO); + } + if (support != null) { + support.close(); + } + } + + /** + * The metrics worker caches each metric of a round once its insert is answered, reads back by id the metrics it + * does not hold, and writes a metric combined with what came in since as an update of the stored one. + */ + @Test + public void writesTheMetricsOfAPersistenceRoundAndReadsThemBackById() throws Exception { + final IMetricsDAO dao = storageDAO.newMetricsDao(respTimeBuilder); + final MetricsSessionCache sessionCache = new MetricsSessionCache(TimeUnit.MINUTES.toMillis(1), true); + final List inserted = new ArrayList<>(); + final List round = new ArrayList<>(); + for (final int minute : MINUTES) { + for (final String entityId : Arrays.asList(SERVICE_ID, OTHER_SERVICE_ID)) { + final Metrics metrics = respTime(RESP_TIME, entityId, minute, 100L + minute); + inserted.add(metrics); + round.add(dao.prepareBatchInsert(respTime, metrics, new SessionCacheCallback(sessionCache, metrics))); + } + } + persistenceRound(round); + + assertEquals(columns(inserted), columns(inserted.stream().map(sessionCache::remove) + .collect(Collectors.toList())), + "every metric of the round is cached once Elasticsearch has answered"); + final List asked = new ArrayList<>(inserted); + asked.add(respTime(RESP_TIME, IDManager.ServiceID.buildId("never-written", true), 0, 1L)); + assertEquals(columns(inserted), columns(dao.multiGet(respTime, asked))); + + final Metrics combined = respTime(RESP_TIME, SERVICE_ID, 0, 100L, 300L); + persistenceRound(Collections.singletonList(dao.prepareBatchUpdate(respTime, combined, null))); + final List afterUpdate = inserted.stream() + .map(metrics -> metrics.id().build().equals(combined.id().build()) ? combined : metrics) + .collect(Collectors.toList()); + assertEquals(columns(afterUpdate), columns(dao.multiGet(respTime, inserted))); + } + + /** + * A metadata metric has one row for all time. The metrics worker asks for it with the minute it sees it in again, + * not the minute it was stored with. + */ + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void readsAMetadataMetricBackWhicheverDayItIsStoredIn() throws Exception { + final StorageBuilder builder = (StorageBuilder) new ServiceTraffic.Builder(); + final ServiceTraffic stored = service("checkout", Layer.GENERAL, -1); + support.writeMetrics(traffic, builder, Arrays.asList(stored, service("checkout", Layer.MESH, -1))); + + final List read = storageDAO.newMetricsDao(builder).multiGet(traffic, Arrays.asList( + service("checkout", Layer.GENERAL, 1), service("never-seen", Layer.GENERAL, 1))); + + assertEquals(columns(builder, Collections.singletonList(stored)), columns(builder, read)); + } + + /** + * The TTL deletes the daily indices up to the local calendar day {@code ttl} days before today, so a metric of + * that day is expired however late in the day it is, and a metric of the next day is not, however early. + */ + @Test + public void expiresAMetricWithTheDayTheTTLDeletes() throws Exception { + final IMetricsDAO dao = storageDAO.newMetricsDao(respTimeBuilder); + final DateTime now = new DateTime(MIDNIGHT).withHourOfDay(10); + final LocalDate lastDeletedDay = now.toLocalDate().minusDays(METRICS_TTL_DAYS); + final long firstKeptDayStart = lastDeletedDay.plusDays(1).toDateTimeAtStartOfDay().getMillis(); + assertEquals( + Arrays.asList(true, true, false, false), + Arrays.asList( + expired(dao, lastDeletedDay.toDateTimeAtStartOfDay().getMillis(), now.getMillis()), + expired(dao, firstKeptDayStart - TimeUnit.MINUTES.toMillis(1), now.getMillis()), + expired(dao, firstKeptDayStart, now.getMillis()), + expired(dao, now.getMillis(), now.getMillis()))); + } + + private boolean expired(final IMetricsDAO dao, final long metricTime, final long now) throws Exception { + final Metrics metrics = OALTestMetrics.newMetrics( + OALTestMetrics.metricsClass(RESP_TIME), "entityId", SERVICE_ID, + "timeBucket", TimeBucket.getMinuteTimeBucket(metricTime)); + return dao.isExpiredCache(respTime, metrics, now, METRICS_TTL_DAYS); + } + + /** + * One worker's share of a persistence round, as the persistence timer writes it: the flush, the end of the round, + * and the wait for the answer. + */ + private void persistenceRound(final List requests) throws Exception { + final CompletableFuture answered = batchDAO.flush(requests); + batchDAO.endOfFlush(); + answered.get(30, TimeUnit.SECONDS); + support.refresh(); + } + + private List columns(final List metrics) { + return columns(respTimeBuilder, metrics); + } + + /** + * Every column of each metric, as the metrics workers store and read them, in no order. + */ + private static List columns(final StorageBuilder builder, final List metrics) { + return metrics.stream().map(metric -> { + final HashMapConverter.ToStorage columns = new HashMapConverter.ToStorage(); + builder.entity2Storage(metric, columns); + return new TreeMap<>(columns.obtain()).toString(); + }).sorted().collect(Collectors.toList()); + } + + /** + * The response time of a minute of an entity, one call of each latency. + */ + private static Metrics respTime(final String metric, final String entityId, final int minute, + final long... latencies) throws Exception { + final LongAvgMetrics metrics = (LongAvgMetrics) OALTestMetrics.newMetrics( + OALTestMetrics.metricsClass(metric), "entityId", entityId, + "timeBucket", TimeBucket.getMinuteTimeBucket(MIDNIGHT + TimeUnit.MINUTES.toMillis(minute))); + for (final long latency : latencies) { + metrics.combine(latency, 1); + } + metrics.calculate(); + return metrics; + } + + private static ServiceTraffic service(final String name, final Layer layer, final int minute) { + final ServiceTraffic service = new ServiceTraffic(); + service.setName(name); + service.setLayer(layer); + service.setTimeBucket(TimeBucket.getMinuteTimeBucket(MIDNIGHT + TimeUnit.MINUTES.toMillis(minute))); + return service; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryITBase.java new file mode 100644 index 000000000000..f4ba04f1a485 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/MetricsQueryITBase.java @@ -0,0 +1,973 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import com.google.gson.Gson; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.cache.TopNCacheReadCommand; +import org.apache.skywalking.oap.server.core.analysis.manual.database.TopNDatabaseStatement; +import org.apache.skywalking.oap.server.core.analysis.meter.MeterEntity; +import org.apache.skywalking.oap.server.core.analysis.meter.function.AcceptableValue; +import org.apache.skywalking.oap.server.core.analysis.meter.function.BucketedValues; +import org.apache.skywalking.oap.server.core.analysis.meter.function.avg.AvgFunction; +import org.apache.skywalking.oap.server.core.analysis.meter.function.avg.AvgHistogramFunction; +import org.apache.skywalking.oap.server.core.analysis.meter.function.avg.AvgLabeledFunction; +import org.apache.skywalking.oap.server.core.analysis.metrics.ApdexMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.CPMMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; +import org.apache.skywalking.oap.server.core.analysis.metrics.LongAvgMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.PercentileMetrics2; +import org.apache.skywalking.oap.server.core.analysis.topn.TopN; +import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.apache.skywalking.oap.server.core.query.DurationUtils; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.enumeration.Scope; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.AttrCondition; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.Entity; +import org.apache.skywalking.oap.server.core.query.input.MetricsCondition; +import org.apache.skywalking.oap.server.core.query.input.RecordCondition; +import org.apache.skywalking.oap.server.core.query.input.TopNCondition; +import org.apache.skywalking.oap.server.core.query.type.HeatMap; +import org.apache.skywalking.oap.server.core.query.type.KVInt; +import org.apache.skywalking.oap.server.core.query.type.KeyValue; +import org.apache.skywalking.oap.server.core.query.type.MetricsValues; +import org.apache.skywalking.oap.server.core.query.type.Record; +import org.apache.skywalking.oap.server.core.query.type.SelectedRecord; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads metrics the way the metrics query service, the top-N ranking and the top-N records do on Elasticsearch and + * OpenSearch, over ranges that cross midnight, so over two daily indices. The metrics are the classes OAP generates + * from the shipped {@code core.oal} and MAL meters of their function classes, so each value column is read under the + * name its function stores it by, which logic sharding replaces with a legacy name for some: a long + * ({@code service_instance_resp_time}, minute and hour), an int ({@code service_apdex}), the percentile labels + * ({@code service_percentile}), a MAL histogram, MAL labeled meters, the call rate whose entities are listed, and MAL + * {@code vm.yaml} meters ranked across hosts. A nested class of a server's IT names the server and the storage mode. + * + *

The edge cases: a minute and an hour without a point, read as empty, beside another instance that has every + * point; a labeled row without one of the labels; points just outside each range that is filtered by time; metrics of + * the same function, which share an index and are told apart by {@code metric_table}, on the same entities and + * minutes; more labeled rows than the read of every entity returns; entities ranked by the mean of the whole range, + * which no single minute or day ranks the same way; endpoints of another service; statements on and just outside the + * bounds of the range, its end instant included, on the end instant of a range ending at midnight, of another + * database, and cache commands of the same entity, which share an index in the default mode and are told apart by + * {@code record_table}. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class MetricsQueryITBase { + private static final String LATENCY_HISTOGRAM = "meter_checkout_latency_histogram"; + private static final String CPU = "meter_vm_cpu_total_percentage"; + private static final String MEMORY = "meter_vm_memory_used"; + private static final String LOAD = "meter_vm_load"; + private static final String INSTANCE_RESP_TIME = "service_instance_resp_time"; + private static final String RESP_TIME = "service_resp_time"; + private static final String MQ_LATENCY = "service_mq_consume_latency"; + private static final String ENDPOINT_RESP_TIME = "endpoint_resp_time"; + private static final String APDEX = "service_apdex"; + private static final String PERCENTILE = "service_percentile"; + private static final String CPM = "service_cpm"; + private static final String ENDPOINT_CPM = "endpoint_cpm"; + private static final String POOL_SIZE = "meter_pool_size"; + private static final String POOL_WAIT = "meter_pool_wait"; + private static final String SERVICE = "checkout"; + private static final String OTHER_SERVICE = "payment"; + private static final long[] HISTOGRAM_BUCKETS = {0, 50, 100}; + /** Minute from midnight, then the count of each bucket. Minute -1 has no histogram. */ + private static final long[][] HISTOGRAMS = { + {-2, 3, 5, 1}, + {0, 4, 6, 2}, + {1, 5, 7, 3}, + }; + /** + * Host, then the CPU percentage of each minute from -3 to 2. web-1 is steady at 50; batch-1 is busy until + * midnight and backup-1 from midnight, each with one quiet minute on the other day, so each tops one day and + * the whole range averages them to 46 and 47. A missing minute is -1. + */ + private static final Object[][] CPU_MINUTES = { + {"web-1", new long[] {-1, -1, 50, 50, 50, 50}}, + {"batch-1", new long[] {60, 60, 60, 5, -1, -1}}, + {"backup-1", new long[] {-1, -1, 5, 62, 62, 62}}, + }; + /** + * Hosts on each side of midnight that top web-1 on their own day, more than the 11 buckets a shard returns for a + * top 1 by default. web-1 is steady at 50 from minute -3 to 2; each day host is 60 until midnight and 5 at + * minute 0, each night host 5 at minute -1 and 62 from midnight, so they average 46 and 47. + */ + private static final int HOSTS_PER_DAY = 12; + private static final long GIB = 1L << 30; + /** Host, then the used memory in bytes of each minute from -2 to 0: db-1 averages 26 GiB, web-1 4 GiB. */ + private static final Object[][] MEMORY_MINUTES = { + {"db-1", new long[] {30 * GIB, 30 * GIB, 18 * GIB}}, + {"web-1", new long[] {-1, 3 * GIB, 5 * GIB}}, + }; + /** Minute from midnight, then checkout-1's response time. Minute -1 has none; checkout-2 has one every minute. */ + private static final long[][] INSTANCE_MINUTES = {{-3, 110}, {-2, 120}, {0, 140}, {1, 150}, {2, 160}}; + /** Hour from midnight, then checkout-1's response time. Hour -1 has none; checkout-2 has one every hour. */ + private static final long[][] INSTANCE_HOURS = {{-2, 1000}, {0, 3000}, {1, 4000}}; + /** Service, then its apdex at minutes -1, 0 and 1; -1 is no point. Over minutes -1 and 0: 10000, 5000, 2500. */ + private static final Object[][] APDEX_MINUTES = { + {SERVICE, new long[] {10000, -1, 5000}}, + {OTHER_SERVICE, new long[] {7500, 2500, -1}}, + {"auth", new long[] {-1, 2500, 10000}}, + }; + /** + * Service, its layer (attr0), then its response time at minutes -1 and 0. The means, 50, 60 and 10, rank them + * unlike either minute does. resp-c also has 1000 at minutes -2 and 1, just outside the range ranked, and resp-a + * has 1 at both minutes in {@code service_mq_consume_latency}, a metric of the same function. + */ + private static final Object[][] RANKED_SERVICES = { + {"resp-a", "GENERAL", new long[] {10, 90}}, + {"resp-b", "GENERAL", new long[] {60, 60}}, + {"resp-c", "MESH", new long[] {5, 15}}, + }; + /** Endpoint of checkout, then its response time at minutes -1 and 0; payment's /order is 500 at both. */ + private static final Object[][] RANKED_ENDPOINTS = { + {"/order", new long[] {10, 90}}, + {"/pay", new long[] {60, 60}}, + {"/cart", new long[] {5, 15}}, + }; + /** + * Service, then the minutes it has a call rate at. The range listed is minutes -3 to 2: cpm-e and cpm-f are just + * outside it, cpm-g and cpm-h on its bounds, and an endpoint of checkout has {@code endpoint_cpm} at minute 2. + */ + private static final Object[][] CPM_MINUTES = { + {"cpm-a", new long[] {-3, -2}}, + {"cpm-b", new long[] {-1}}, + {"cpm-c", new long[] {-3, 0}}, + {"cpm-d", new long[] {1}}, + {"cpm-e", new long[] {3}}, + {"cpm-f", new long[] {-4}}, + {"cpm-g", new long[] {2}}, + {"cpm-h", new long[] {-3}}, + }; + /** + * Pool, minute, then its active and idle connections; -1 is a label the row does not have. The range read is + * minutes -1 and 0: pool-3 at minute -2 and pool-0 at minute 1 are just outside it. + */ + private static final long[][] POOLS = { + {0, -1, 10, 100}, + {1, -1, 20, 200}, + {2, -1, 30, -1}, + {0, 0, 11, 101}, + {1, 0, 21, 201}, + {3, -2, 98, 998}, + {0, 1, 99, 999}, + }; + /** Waiting pools at each of minutes -1 and 0: more rows than a read of every entity returns. */ + private static final int WAITING_POOLS = 6; + private static final String DATABASE = "orders-db:3306"; + /** + * Second from midnight, latency in ms. The range read, minutes -2 to 1, is read from second -120 through its end + * instant, second 120, so the first and last statements sit on its bounds. + */ + private static final long[][] STATEMENTS = { + {-120, 600}, + {-110, 1200}, + {-80, 300}, + {-40, 900}, + {15, 1500}, + {45, 100}, + {90, 700}, + {119, 50}, + {120, 20}, + }; + /** The slowest and the fastest statements of the database, each a millisecond outside the range. */ + private static final long[][] MILLIS_OUTSIDE = {{-120_001, 5000}, {120_001, 10}}; + /** Its range ends in the last minute before midnight, so at midnight, the first instant of the next day. */ + private static final String MIDNIGHT_DATABASE = "midnight-db:3306"; + + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private MetricsQueryEsDAO metricsDAO; + private AggregationQueryEsDAO aggregationDAO; + private RecordsQueryEsDAO recordsDAO; + private final Map models = new HashMap<>(); + private final Map waiting = new TreeMap<>(); + + MetricsQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeMetrics() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + metricsDAO = new MetricsQueryEsDAO(support.getClient()); + aggregationDAO = new AggregationQueryEsDAO(support.getClient()); + recordsDAO = new RecordsQueryEsDAO(support.getClient()); + MeterEntity.setNamingControl(new NamingControl(70, 100, 150, new EndpointNameGrouping())); + installModels(); + + final List histograms = new ArrayList<>(); + for (final long[] point : HISTOGRAMS) { + final HistogramMeter meter = new HistogramMeter(); + meter.setTimeBucket(minuteBucket(point[0])); + meter.accept(MeterEntity.newService(SERVICE, Layer.GENERAL), + new BucketedValues(HISTOGRAM_BUCKETS, Arrays.copyOfRange(point, 1, point.length))); + meter.calculate(); + histograms.add(meter); + } + support.writeMetrics(model(LATENCY_HISTOGRAM, DownSampling.Minute), + new AvgHistogramFunction.AvgHistogramFunctionBuilder(), histograms); + + writeHostMetric(CPU, CPU_MINUTES, -3); + writeHostMetric(MEMORY, MEMORY_MINUTES, -2); + final List loads = new ArrayList<>(); + loads.add(new Object[] {"web-1", new long[] {50, 50, 50, 50, 50, 50}}); + for (int i = 0; i < HOSTS_PER_DAY; i++) { + loads.add(new Object[] {"day-" + i, new long[] {60, 60, 60, 5, -1, -1}}); + loads.add(new Object[] {"night-" + i, new long[] {-1, -1, 5, 62, 62, 62}}); + } + writeHostMetric(LOAD, loads.toArray(new Object[0][]), -3); + writeInstanceResponseTimes(); + writeApdex(); + writePercentiles(); + writeRankedResponseTimes(); + writeCallRates(); + writePools(); + writeStatements(); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + /** + * Every minute of the range in minute order, each from whichever daily index holds it, and the minute none holds + * as empty. + */ + @Test + public void readsEveryMinuteOfARangeAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("110", "120", "empty", "140", "150", "160"), + values(INSTANCE_RESP_TIME, instance("checkout-1"), minutes(-3, 2))); + assertEquals(Arrays.asList("empty", "empty"), + values(INSTANCE_RESP_TIME, instance("checkout-9"), minutes(-1, 0))); + } + + @Test + public void readsEveryHourOfARangeAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("1000", "empty", "3000", "4000"), + values(INSTANCE_RESP_TIME, instance("checkout-1"), hours(-2, 1))); + } + + @Test + public void readsTheIntValuesOfAMetric() throws Exception { + assertEquals(Arrays.asList("10000", "empty", "5000"), values(APDEX, service(SERVICE), minutes(-1, 1))); + } + + /** + * Each percentile label over every minute of the range, the minute without a point as empty. + */ + @Test + public void readsEveryLabelOfALabeledMetricAcrossMidnight() throws Exception { + final Map> expected = new LinkedHashMap<>(); + expected.put("{p=50}", Arrays.asList("500", "empty", "1000")); + expected.put("{p=75}", Arrays.asList("750", "empty", "1500")); + expected.put("{p=90}", Arrays.asList("900", "empty", "1800")); + expected.put("{p=95}", Arrays.asList("950", "empty", "1900")); + expected.put("{p=99}", Arrays.asList("990", "empty", "1980")); + assertEquals(expected, labeled(Collections.emptyList())); + } + + @Test + public void readsTheLabelsAskedFor() throws Exception { + final Map> expected = new LinkedHashMap<>(); + expected.put("{p=50}", Arrays.asList("500", "empty", "1000")); + expected.put("{p=99}", Arrays.asList("990", "empty", "1980")); + assertEquals(expected, labeled(Collections.singletonList(new KeyValue("p", "50,99")))); + assertEquals(Collections.emptyMap(), labeled(Collections.singletonList(new KeyValue("p", "42")))); + } + + /** + * One column per minute of the range, in minute order, each from whichever daily index holds it, and an empty + * column for the minute none holds. + */ + @Test + public void readsOneHeatMapColumnPerMinuteAcrossMidnight() throws Exception { + final HeatMap heatMap = metricsDAO.readHeatMap( + condition(LATENCY_HISTOGRAM, service(SERVICE)), valueColumn(LATENCY_HISTOGRAM), minutes(-2, 1)); + assertEquals( + Arrays.asList(Arrays.asList(3L, 5L, 1L), Arrays.asList(0L, 0L, 0L), Arrays.asList(4L, 6L, 2L), + Arrays.asList(5L, 7L, 3L)), + heatMap.getValues().stream().map(HeatMap.HeatMapColumn::getValues).collect(Collectors.toList())); + assertEquals("[{\"min\":\"0\",\"max\":\"50\"},{\"min\":\"50\",\"max\":\"100\"}," + + "{\"min\":\"100\",\"max\":\"infinite+\"}]", + new Gson().toJson(heatMap.getBuckets())); + } + + /** + * Every document of the metric in the range, from both daily indices, each under every label; a document without + * a label reads as empty under it. + */ + @Test + public void readsTheLabeledValuesOfEveryEntityAcrossMidnight() throws Exception { + final Map> expected = new TreeMap<>(); + expected.put("{pool=active}", new TreeMap<>()); + expected.put("{pool=idle}", new TreeMap<>()); + for (final long[] pool : POOLS) { + if (pool[1] < -1 || pool[1] > 0) { + continue; + } + final String id = poolRowId(POOL_SIZE, pool[0], pool[1]); + expected.get("{pool=active}").put(id, String.valueOf(pool[2])); + expected.get("{pool=idle}").put(id, pool[3] < 0 ? "empty" : String.valueOf(pool[3])); + } + assertEquals(expected, everyEntity(POOL_SIZE, Collections.emptyList())); + + expected.remove("{pool=active}"); + assertEquals(expected, everyEntity(POOL_SIZE, Collections.singletonList(new KeyValue("pool", "idle")))); + } + + @Test + public void capsTheLabeledValuesOfEveryEntity() throws Exception { + final Map> read = everyEntity(POOL_WAIT, Collections.emptyList()); + assertEquals(Collections.singleton("{pool=waiting}"), read.keySet()); + final Map rows = read.get("{pool=waiting}"); + assertEquals(IMetricsQueryDAO.METRICS_VALUES_WITHOUT_ENTITY_LIMIT, rows.size()); + for (final Map.Entry row : rows.entrySet()) { + assertEquals(String.valueOf(waiting.get(row.getKey())), row.getValue(), row.getKey()); + } + } + + /** + * The entities with a point in the range, by their latest point, newest first: the latest of one is on the other + * day of the range, and the ones just outside the range or of another metric of the same function are not. + */ + @Test + public void listsTheEntitiesOfARangeNewestFirstAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("cpm-g", "cpm-d", "cpm-c", "cpm-b", "cpm-a", "cpm-h"), entities(minutes(-3, 2), 10)); + assertEquals(Arrays.asList("cpm-g", "cpm-d", "cpm-c"), entities(minutes(-3, 2), 3)); + assertEquals(Collections.emptyList(), entities(minutes(-6, -5), 10)); + } + + /** + * Each host's mean over the whole range, not the mean of its two daily means (32 for batch-1, 33 for backup-1). + */ + @Test + public void ranksTheMeanOfTheWholeRangeAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("web-1=50", "backup-1=47", "batch-1=46"), + rank(top(CPU, 3, Order.DES), minutes(-3, 2))); + } + + /** + * web-1 tops neither day, as batch-1 and backup-1 each top one, yet tops the range. + */ + @Test + public void keepsTheTopOfTheRangeThatTopsNoDay() throws Exception { + assertEquals(Collections.singletonList("web-1=50"), rank(top(CPU, 1, Order.DES), minutes(-3, 2))); + } + + /** + * Elasticsearch's documented approximation of terms ordered by a sub-aggregation, which OAP accepts: each daily + * index returns its own top hosts, and web-1, the top of the range, is in neither, so it is missing, and night-0's + * mean is that of today's index alone, 62 rather than 47. The JDBC IT asserts the exact answer, web-1=50, for the + * same hosts. A range in one index of one shard is ranked exactly, see {@link #ranksAMetricAboveTheIntRange}. + */ + @Test + public void ranksTheTopOfEachShardWhenNoShardHoldsTheRangesTop() throws Exception { + assertEquals(Collections.singletonList("night-0=62"), rank(top(LOAD, 1, Order.DES), minutes(-3, 2))); + } + + @Test + public void ranksAMetricAboveTheIntRange() throws Exception { + assertEquals(Arrays.asList("db-1=" + 26 * GIB, "web-1=" + 4 * GIB), + rank(top(MEMORY, 2, Order.DES), minutes(-2, 0))); + assertEquals(Arrays.asList("db-1=" + 18 * GIB, "web-1=" + 5 * GIB), + rank(top(MEMORY, 2, Order.DES), minutes(0, 0))); + } + + @Test + public void ranksTheServicesEitherWayAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("resp-b=60", "resp-a=50", "resp-c=10"), + rank(top(RESP_TIME, 10, Order.DES), minutes(-1, 0))); + assertEquals(Arrays.asList("resp-c=10", "resp-a=50", "resp-b=60"), + rank(top(RESP_TIME, 10, Order.ASC), minutes(-1, 0))); + assertEquals(Arrays.asList("resp-b=60", "resp-a=50"), rank(top(RESP_TIME, 2, Order.DES), minutes(-1, 0))); + assertEquals(Collections.singletonList("resp-c=10"), rank(top(RESP_TIME, 1, Order.ASC), minutes(-1, 0))); + assertEquals(Collections.emptyList(), rank(top(RESP_TIME, 10, Order.DES), minutes(-6, -5))); + } + + @Test + public void ranksTheServicesOfAnAttribute() throws Exception { + final TopNCondition general = top(RESP_TIME, 10, Order.DES); + general.setAttributes(Collections.singletonList(new AttrCondition("attr0", "GENERAL", true))); + assertEquals(Arrays.asList("resp-b=60", "resp-a=50"), rank(general, minutes(-1, 0))); + final TopNCondition notGeneral = top(RESP_TIME, 10, Order.DES); + notGeneral.setAttributes(Collections.singletonList(new AttrCondition("attr0", "GENERAL", false))); + assertEquals(Collections.singletonList("resp-c=10"), rank(notGeneral, minutes(-1, 0))); + } + + /** + * The endpoints of one service, as the top-N of a service's endpoints asks with its {@code service_id}. + */ + @Test + public void ranksTheEndpointsOfAService() throws Exception { + final List ofService = Collections.singletonList( + new KeyValue("service_id", IDManager.ServiceID.buildId(SERVICE, true))); + final TopNCondition descending = top(ENDPOINT_RESP_TIME, 10, Order.DES); + descending.setScope(Scope.Endpoint); + assertEquals(Arrays.asList("/pay=60", "/order=50", "/cart=10"), rank(descending, minutes(-1, 0), ofService)); + final TopNCondition ascending = top(ENDPOINT_RESP_TIME, 2, Order.ASC); + ascending.setScope(Scope.Endpoint); + assertEquals(Arrays.asList("/cart=10", "/order=50"), rank(ascending, minutes(-1, 0), ofService)); + } + + @Test + public void ranksAnIntMetricAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("checkout=10000", "payment=5000", "auth=2500"), + rank(top(APDEX, 10, Order.DES), minutes(-1, 0))); + assertEquals(Arrays.asList("auth=2500", "payment=5000"), rank(top(APDEX, 2, Order.ASC), minutes(-1, 0))); + } + + @Test + public void readsTheSlowestStatementsAcrossMidnight() throws Exception { + assertEquals(Arrays.asList(statementRecord(4, 1500), statementRecord(1, 1200), statementRecord(3, 900)), + records(DATABASE, 3, Order.DES)); + } + + @Test + public void readsTheFastestStatementsAcrossMidnight() throws Exception { + assertEquals(Arrays.asList(statementRecord(8, 20), statementRecord(7, 50), statementRecord(5, 100)), + records(DATABASE, 3, Order.ASC)); + assertEquals(Collections.emptyList(), records("idle-db:3306", 3, Order.ASC)); + } + + /** + * A range ending in a day's last minute ends at the next midnight, which is in the next day's index. + */ + @Test + public void readsTheStatementOnTheEndInstantOfARangeEndingAtMidnight() throws Exception { + assertEquals(Collections.singletonList(statementRecord(40, 30)), + records(MIDNIGHT_DATABASE, 10, Order.DES, minutes(-2, -1))); + } + + /** + * Installs every model before the first write, as OAP does at boot. A column that a model installed later adds to + * a shared index reaches its template and today's index only, so a daily index written before would map it by + * its first value. + */ + private void installModels() throws Exception { + installMeter(HistogramMeter.class, LATENCY_HISTOGRAM); + for (final String hostMetric : Arrays.asList(CPU, MEMORY, LOAD)) { + installMeter(AvgMeter.class, hostMetric); + } + installMeter(LabeledMeter.class, POOL_SIZE); + installMeter(LabeledMeter.class, POOL_WAIT); + for (final String metric : Arrays.asList(INSTANCE_RESP_TIME, APDEX, PERCENTILE, RESP_TIME, MQ_LATENCY, + ENDPOINT_RESP_TIME, CPM, ENDPOINT_CPM)) { + installOAL(metric, DownSampling.Minute); + } + installOAL(INSTANCE_RESP_TIME, DownSampling.Hour); + models.put(key(TopNDatabaseStatement.INDEX_NAME, DownSampling.Second), + support.installRecord(TopNDatabaseStatement.class)); + models.put(key(TopNCacheReadCommand.INDEX_NAME, DownSampling.Second), + support.installRecord(TopNCacheReadCommand.class)); + } + + private void installMeter(final Class meterClass, final String name) throws Exception { + models.put(key(name, DownSampling.Minute), + support.installMeter(meterClass, DefaultScopeDefine.SERVICE, name)); + } + + private void installOAL(final String metric, final DownSampling downSampling) throws Exception { + final Class metricsClass = OALTestMetrics.metricsClass(metric); + models.put(key(metric, downSampling), + support.installMetrics(metricsClass, downSampling)); + } + + private Model model(final String name, final DownSampling downSampling) { + return models.get(key(name, downSampling)); + } + + private static String key(final String name, final DownSampling downSampling) { + return name + "/" + downSampling; + } + + private void writeHostMetric(final String name, final Object[][] hosts, final long firstMinute) throws Exception { + final List metrics = new ArrayList<>(); + for (final Object[] host : hosts) { + final long[] values = (long[]) host[1]; + for (int i = 0; i < values.length; i++) { + if (values[i] < 0) { + continue; + } + final AvgMeter meter = new AvgMeter(); + meter.setTimeBucket(minuteBucket(firstMinute + i)); + meter.accept(MeterEntity.newService((String) host[0], Layer.OS_LINUX), values[i]); + meter.calculate(); + metrics.add(meter); + } + } + support.writeMetrics(model(name, DownSampling.Minute), new AvgFunction.AvgStorageBuilder(), metrics); + } + + private void writeInstanceResponseTimes() throws Exception { + final List minutes = new ArrayList<>(); + for (final long[] point : INSTANCE_MINUTES) { + minutes.add(instanceResponseTime("checkout-1", minuteBucket(point[0]), point[1])); + } + for (long minute = -3; minute <= 2; minute++) { + minutes.add(instanceResponseTime("checkout-2", minuteBucket(minute), 7)); + } + writeOAL(INSTANCE_RESP_TIME, DownSampling.Minute, minutes); + + final List hours = new ArrayList<>(); + for (final long[] point : INSTANCE_HOURS) { + hours.add(instanceResponseTime("checkout-1", hourBucket(point[0]), point[1])); + } + for (long hour = -2; hour <= 1; hour++) { + hours.add(instanceResponseTime("checkout-2", hourBucket(hour), 7)); + } + writeOAL(INSTANCE_RESP_TIME, DownSampling.Hour, hours); + } + + private void writeApdex() throws Exception { + final List metrics = new ArrayList<>(); + for (final Object[] service : APDEX_MINUTES) { + final long[] values = (long[]) service[1]; + for (int i = 0; i < values.length; i++) { + if (values[i] < 0) { + continue; + } + // An apdex of n x 2500 is n satisfied calls of four. + final ApdexMetrics apdex = (ApdexMetrics) oal( + APDEX, serviceId((String) service[0]), minuteBucket(i - 1), + "totalNum", 4L, "sNum", values[i] / 2500, "tNum", 0L); + apdex.calculate(); + metrics.add(apdex); + } + } + writeOAL(APDEX, DownSampling.Minute, metrics); + } + + private void writePercentiles() throws Exception { + final List metrics = new ArrayList<>(); + // checkout's latencies are 10 to 1000 ms at minute -1 and twice that at minute 1. + metrics.add(percentile(SERVICE, -1, 1)); + metrics.add(percentile(SERVICE, 1, 2)); + for (long minute = -1; minute <= 1; minute++) { + metrics.add(percentile(OTHER_SERVICE, minute, 3)); + } + writeOAL(PERCENTILE, DownSampling.Minute, metrics); + } + + private void writeRankedResponseTimes() throws Exception { + final List services = new ArrayList<>(); + for (final Object[] service : RANKED_SERVICES) { + final long[] values = (long[]) service[2]; + for (int i = 0; i < values.length; i++) { + services.add(responseTime(RESP_TIME, serviceId((String) service[0]), minuteBucket(i - 1), values[i], + "attr0", service[1])); + } + } + services.add(responseTime(RESP_TIME, serviceId("resp-c"), minuteBucket(-2), 1000, "attr0", "MESH")); + services.add(responseTime(RESP_TIME, serviceId("resp-c"), minuteBucket(1), 1000, "attr0", "MESH")); + writeOAL(RESP_TIME, DownSampling.Minute, services); + + final List mq = new ArrayList<>(); + for (long minute = -1; minute <= 0; minute++) { + mq.add(responseTime(MQ_LATENCY, serviceId("resp-a"), minuteBucket(minute), 1, "attr0", "GENERAL")); + } + writeOAL(MQ_LATENCY, DownSampling.Minute, mq); + + final List endpoints = new ArrayList<>(); + for (final Object[] endpoint : RANKED_ENDPOINTS) { + final long[] values = (long[]) endpoint[1]; + for (int i = 0; i < values.length; i++) { + endpoints.add(responseTime( + ENDPOINT_RESP_TIME, endpointId(SERVICE, (String) endpoint[0]), minuteBucket(i - 1), values[i], + "serviceId", serviceId(SERVICE))); + } + } + for (long minute = -1; minute <= 0; minute++) { + endpoints.add(responseTime(ENDPOINT_RESP_TIME, endpointId(OTHER_SERVICE, "/order"), minuteBucket(minute), + 500, "serviceId", serviceId(OTHER_SERVICE))); + } + writeOAL(ENDPOINT_RESP_TIME, DownSampling.Minute, endpoints); + } + + private void writeCallRates() throws Exception { + final List services = new ArrayList<>(); + for (final Object[] service : CPM_MINUTES) { + for (final long minute : (long[]) service[1]) { + services.add(callRate(CPM, serviceId((String) service[0]), minute)); + } + } + writeOAL(CPM, DownSampling.Minute, services); + writeOAL(ENDPOINT_CPM, DownSampling.Minute, + Collections.singletonList(callRate(ENDPOINT_CPM, endpointId(SERVICE, "/order"), 2, "serviceId", + serviceId(SERVICE)))); + } + + private void writePools() throws Exception { + final List sizes = new ArrayList<>(); + for (final long[] pool : POOLS) { + final DataTable value = new DataTable(); + value.put("{pool=active}", pool[2]); + if (pool[3] >= 0) { + value.put("{pool=idle}", pool[3]); + } + sizes.add(labeledMeter("pool-" + pool[0], pool[1], value)); + } + support.writeMetrics(model(POOL_SIZE, DownSampling.Minute), new AvgLabeledFunction.AvgLabeledStorageBuilder(), + sizes); + + final List waits = new ArrayList<>(); + for (long minute = -1; minute <= 0; minute++) { + for (int pool = 0; pool < WAITING_POOLS; pool++) { + final DataTable value = new DataTable(); + final long waiters = pool * 10L + minute + 2; + value.put("{pool=waiting}", waiters); + waits.add(labeledMeter("pool-" + pool, minute, value)); + waiting.put(poolRowId(POOL_WAIT, pool, minute), waiters); + } + } + support.writeMetrics(model(POOL_WAIT, DownSampling.Minute), new AvgLabeledFunction.AvgLabeledStorageBuilder(), + waits); + } + + private void writeStatements() throws Exception { + final List statements = new ArrayList<>(); + for (int i = 0; i < STATEMENTS.length; i++) { + statements.add(statement(DATABASE, i, STATEMENTS[i][0], STATEMENTS[i][1])); + } + for (int i = 0; i < MILLIS_OUTSIDE.length; i++) { + statements.add(statementAt(DATABASE, 10 + i, MIDNIGHT + MILLIS_OUTSIDE[i][0], MILLIS_OUTSIDE[i][1])); + } + statements.add(statementAt(MIDNIGHT_DATABASE, 40, MIDNIGHT, 30)); + statements.add(statementAt(MIDNIGHT_DATABASE, 41, MIDNIGHT + 1, 40)); + statements.add(statement("inventory-db:3306", 20, 0, 9000)); + statements.add(statement("inventory-db:3306", 21, 10, 1)); + support.writeRecords(model(TopNDatabaseStatement.INDEX_NAME, DownSampling.Second), + new TopNDatabaseStatement.Builder(), statements); + support.writeRecords(model(TopNCacheReadCommand.INDEX_NAME, DownSampling.Second), + new TopNCacheReadCommand.Builder(), + Arrays.asList(cacheCommand(DATABASE, 30, 5, 8000), cacheCommand(DATABASE, 31, 6, 2))); + } + + private void writeOAL(final String metric, final DownSampling downSampling, + final List metrics) throws Exception { + support.writeMetrics(model(metric, downSampling), + OALTestMetrics.builder(OALTestMetrics.metricsClass(metric)), metrics); + } + + private List values(final String metric, final Entity entity, final Duration duration) throws Exception { + return metricsDAO.readMetricsValues(condition(metric, entity), valueColumn(metric), duration) + .getValues().getValues().stream() + .map(MetricsQueryITBase::value) + .collect(Collectors.toList()); + } + + /** checkout's percentiles over minutes -1 to 1, as label to values. */ + private Map> labeled(final List labels) throws Exception { + final Map> byLabel = new LinkedHashMap<>(); + for (final MetricsValues values : metricsDAO.readLabeledMetricsValues( + condition(PERCENTILE, service(SERVICE)), valueColumn(PERCENTILE), labels, minutes(-1, 1))) { + byLabel.put(values.getLabel(), values.getValues().getValues().stream() + .map(MetricsQueryITBase::value) + .collect(Collectors.toList())); + } + return byLabel; + } + + /** The labeled values of every entity over minutes -1 and 0, as label to row id to value. */ + private Map> everyEntity(final String metric, + final List labels) throws Exception { + final Map> byLabel = new TreeMap<>(); + for (final MetricsValues values : metricsDAO.readLabeledMetricsValuesWithoutEntity( + metric, valueColumn(metric), labels, minutes(-1, 0))) { + final Map rows = new TreeMap<>(); + for (final KVInt kv : values.getValues().getValues()) { + rows.put(kv.getId(), value(kv)); + } + byLabel.put(values.getLabel(), rows); + } + return byLabel; + } + + /** The services listed, by name. */ + private List entities(final Duration duration, final int limit) throws Exception { + return metricsDAO.listEntityIdsInRange(CPM, valueColumn(CPM), null, duration, limit) + .stream() + .map(id -> IDManager.ServiceID.analysisId(id).getName()) + .collect(Collectors.toList()); + } + + private List rank(final TopNCondition condition, final Duration duration) throws Exception { + return rank(condition, duration, null); + } + + /** The ranked entities as {@code name=value}, an endpoint by its own name. */ + private List rank(final TopNCondition condition, final Duration duration, + final List additionalConditions) throws Exception { + final List ranked = aggregationDAO.sortMetrics( + condition, valueColumn(condition.getName()), duration, additionalConditions); + return ranked.stream() + .map(record -> (condition.getScope() == Scope.Endpoint + ? IDManager.EndpointID.analysisId(record.getId()).getEndpointName() + : IDManager.ServiceID.analysisId(record.getId()).getName()) + "=" + record.getValue()) + .collect(Collectors.toList()); + } + + /** The records as {@code name|refId|id|value}. */ + private List records(final String database, final int topN, final Order order) throws Exception { + return records(database, topN, order, minutes(-2, 1)); + } + + private List records(final String database, final int topN, final Order order, + final Duration duration) throws Exception { + final Entity entity = new Entity(); + entity.setScope(Scope.Service); + entity.setServiceName(database); + entity.setNormal(false); + final RecordCondition condition = new RecordCondition(); + condition.setName(TopNDatabaseStatement.INDEX_NAME); + condition.setParentEntity(entity); + condition.setTopN(topN); + condition.setOrder(order); + final List records = recordsDAO.readRecords(condition, TopN.LATENCY, duration); + return records.stream() + .map(record -> record.getName() + "|" + record.getRefId() + "|" + record.getId() + "|" + + record.getValue()) + .collect(Collectors.toList()); + } + + private static String statementRecord(final int i, final long latency) { + return "select * from orders where id = " + i + "|trace-" + i + "|trace-" + i + "|" + latency; + } + + private static TopNCondition top(final String metric, final int topN, final Order order) { + final TopNCondition condition = new TopNCondition(); + condition.setName(metric); + condition.setScope(Scope.Service); + condition.setTopN(topN); + condition.setOrder(order); + return condition; + } + + private static String value(final KVInt kv) { + return kv.isEmptyValue() ? "empty" : String.valueOf(kv.getValue()); + } + + private static String valueColumn(final String metric) { + return ValueColumnMetadata.INSTANCE.getValueCName(metric); + } + + private static Entity service(final String name) { + final Entity entity = new Entity(); + entity.setScope(Scope.Service); + entity.setServiceName(name); + entity.setNormal(true); + return entity; + } + + private static Entity instance(final String name) { + final Entity entity = service(SERVICE); + entity.setScope(Scope.ServiceInstance); + entity.setServiceInstanceName(name); + return entity; + } + + private static MetricsCondition condition(final String metric, final Entity entity) { + final MetricsCondition condition = new MetricsCondition(); + condition.setName(metric); + condition.setEntity(entity); + return condition; + } + + private static Duration minutes(final long first, final long last) { + final Duration duration = new Duration(); + duration.setStep(Step.MINUTE); + duration.setStart(DurationUtils.YYYY_MM_DD_HHMM.print(MIDNIGHT + first * 60_000L)); + duration.setEnd(DurationUtils.YYYY_MM_DD_HHMM.print(MIDNIGHT + last * 60_000L)); + return duration; + } + + private static Duration hours(final long first, final long last) { + final Duration duration = new Duration(); + duration.setStep(Step.HOUR); + duration.setStart(DurationUtils.YYYY_MM_DD_HH.print(MIDNIGHT + first * 3_600_000L)); + duration.setEnd(DurationUtils.YYYY_MM_DD_HH.print(MIDNIGHT + last * 3_600_000L)); + return duration; + } + + private static long minuteBucket(final long minute) { + return TimeBucket.getMinuteTimeBucket(MIDNIGHT + minute * 60_000L); + } + + private static long hourBucket(final long hour) { + return TimeBucket.getTimeBucket(MIDNIGHT + hour * 3_600_000L, DownSampling.Hour); + } + + private static String serviceId(final String name) { + return IDManager.ServiceID.buildId(name, true); + } + + private static String endpointId(final String service, final String endpoint) { + return IDManager.EndpointID.buildId(serviceId(service), endpoint); + } + + /** The id of a pool's row: the metric, the minute and the entity, as OAP stores a MAL metric. */ + private static String poolRowId(final String metric, final long pool, final long minute) { + return metric + "_" + minuteBucket(minute) + "_" + serviceId("pool-" + pool); + } + + private static Metrics oal(final String metric, final String entityId, final long timeBucket, + final Object... properties) throws Exception { + final Object[] all = Arrays.copyOf(properties, properties.length + 4); + all[properties.length] = "entityId"; + all[properties.length + 1] = entityId; + all[properties.length + 2] = "timeBucket"; + all[properties.length + 3] = timeBucket; + return OALTestMetrics.newMetrics(OALTestMetrics.metricsClass(metric), all); + } + + private static Metrics responseTime(final String metric, final String entityId, final long timeBucket, + final long value, final Object... properties) throws Exception { + // Two calls around the value, so the stored value is the average the function computes. + final LongAvgMetrics metrics = (LongAvgMetrics) oal(metric, entityId, timeBucket, properties); + metrics.combine(value - 1, 1); + metrics.combine(value + 1, 1); + metrics.calculate(); + return metrics; + } + + private static Metrics instanceResponseTime(final String instance, final long timeBucket, + final long value) throws Exception { + return responseTime(INSTANCE_RESP_TIME, IDManager.ServiceInstanceID.buildId(serviceId(SERVICE), instance), + timeBucket, value, "serviceId", serviceId(SERVICE)); + } + + private static Metrics callRate(final String metric, final String entityId, final long minute, + final Object... properties) throws Exception { + final CPMMetrics metrics = (CPMMetrics) oal(metric, entityId, minuteBucket(minute), properties); + metrics.combine(1); + metrics.calculate(); + return metrics; + } + + /** A service's percentiles of 100 calls, of {@code scale} x 10 to {@code scale} x 1000 ms. */ + private static Metrics percentile(final String service, final long minute, final int scale) throws Exception { + final PercentileMetrics2 metrics = + (PercentileMetrics2) oal(PERCENTILE, serviceId(service), minuteBucket(minute)); + for (int latency = 10; latency <= 1000; latency += 10) { + metrics.combine(latency * scale, 10); + } + metrics.calculate(); + return metrics; + } + + private static Metrics labeledMeter(final String service, final long minute, final DataTable value) { + final LabeledMeter meter = new LabeledMeter(); + meter.setTimeBucket(minuteBucket(minute)); + meter.accept(MeterEntity.newService(service, Layer.GENERAL), value); + meter.calculate(); + return meter; + } + + private static TopNDatabaseStatement statement(final String database, final int i, + final long second, final long latency) { + return statementAt(database, i, MIDNIGHT + second * 1000L, latency); + } + + private static TopNDatabaseStatement statementAt(final String database, final int i, + final long timestamp, final long latency) { + final TopNDatabaseStatement statement = new TopNDatabaseStatement(); + statement.setId("segment-" + i + "-1"); + statement.setEntityId(IDManager.ServiceID.buildId(database, false)); + statement.setStatement("select * from orders where id = " + i); + statement.setLatency(latency); + statement.setTraceId("trace-" + i); + statement.setTimestamp(timestamp); + statement.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + return statement; + } + + private static TopNCacheReadCommand cacheCommand(final String database, final int i, + final long second, final long latency) { + final long timestamp = MIDNIGHT + second * 1000L; + final TopNCacheReadCommand command = new TopNCacheReadCommand(); + command.setId("segment-" + i + "-1"); + command.setEntityId(IDManager.ServiceID.buildId(database, false)); + command.setCommand("GET order:" + i); + command.setLatency(latency); + command.setTraceId("trace-" + i); + command.setTimestamp(timestamp); + command.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + return command; + } + + /** A MAL histogram meter: the class MAL generates adds nothing to its function but {@code createNew}. */ + static class HistogramMeter extends AvgHistogramFunction { + @Override + public AcceptableValue createNew() { + return new HistogramMeter(); + } + } + + /** A MAL meter of the default {@code avg} function. */ + static class AvgMeter extends AvgFunction { + @Override + public AcceptableValue createNew() { + return new AvgMeter(); + } + } + + /** A MAL labeled meter of the {@code avgLabeled} function. */ + static class LabeledMeter extends AvgLabeledFunction { + @Override + public AcceptableValue createNew() { + return new LabeledMeter(); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OALTestMetrics.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OALTestMetrics.java new file mode 100644 index 000000000000..89cfe807efa8 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OALTestMetrics.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.File; +import java.io.FileReader; +import java.io.Reader; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javassist.ClassPool; +import org.apache.skywalking.oal.v2.generator.CodeGenModel; +import org.apache.skywalking.oal.v2.generator.MetricDefinitionEnricher; +import org.apache.skywalking.oal.v2.generator.OALClassGeneratorV2; +import org.apache.skywalking.oal.v2.model.MetricDefinition; +import org.apache.skywalking.oal.v2.parser.OALScriptParserV2; +import org.apache.skywalking.oap.server.core.analysis.SourceDecoratorManager; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +/** + * The metrics classes OAP generates from the shipped {@code core.oal}, generated the same way at test time, so an IT + * reads a metric through the schema, the value columns and the name a real OAP has. + */ +final class OALTestMetrics { + private static final String SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.source."; + private static final String METRICS_PACKAGE = "org.apache.skywalking.oap.server.core.source.oal.rt.metrics."; + private static final String CORE_OAL = "oal/core.oal"; + private static final String[] STARTER_RESOURCES = { + "../../server-starter/src/main/resources/", + "oap-server/server-starter/src/main/resources/", + "../server-starter/src/main/resources/" + }; + /** The sources {@code core.oal} reads. */ + private static final String[] SOURCES = { + "Service", "ServiceInstance", "Endpoint", "ServiceRelation", "ServiceInstanceRelation", "EndpointRelation", + "DatabaseAccess", "CacheAccess", "MQAccess", "MQEndpointAccess" + }; + private static final String[] DECORATORS = {"ServiceDecorator", "EndpointDecorator"}; + + /** Generated once per JVM: a class name can be defined only once in a class loader. */ + private static Map> CLASSES; + + private OALTestMetrics() { + } + + /** The class OAP generates for the {@code core.oal} metric of this name. */ + static synchronized Class metricsClass(String name) throws Exception { + if (CLASSES == null) { + CLASSES = generate(); + } + final Class metricsClass = CLASSES.get(name); + if (metricsClass == null) { + throw new IllegalArgumentException("core.oal has no metric " + name); + } + return metricsClass; + } + + static int scopeId(Class metricsClass) { + return metricsClass.getAnnotation(Stream.class).scopeId(); + } + + @SuppressWarnings("unchecked") + static StorageBuilder builder(Class metricsClass) throws Exception { + return (StorageBuilder) metricsClass.getAnnotation(Stream.class).builder() + .getDeclaredConstructor().newInstance(); + } + + /** A new metrics object with each named property set through its setter. */ + static Metrics newMetrics(Class metricsClass, Object... properties) throws Exception { + final Metrics metrics = metricsClass.getDeclaredConstructor().newInstance(); + for (int i = 0; i < properties.length; i += 2) { + set(metrics, (String) properties[i], properties[i + 1]); + } + return metrics; + } + + private static void set(Object target, String property, Object value) throws Exception { + final String setter = "set" + Character.toUpperCase(property.charAt(0)) + property.substring(1); + for (Class c = target.getClass(); c != null; c = c.getSuperclass()) { + for (final Method method : c.getDeclaredMethods()) { + if (method.getName().equals(setter) && method.getParameterCount() == 1) { + method.setAccessible(true); + method.invoke(target, value); + return; + } + } + } + throw new IllegalArgumentException(target.getClass().getSimpleName() + " has no " + setter); + } + + private static Map> generate() throws Exception { + final DefaultScopeDefine.Listener scopes = new DefaultScopeDefine.Listener(); + for (final String source : SOURCES) { + try { + scopes.notify(Class.forName(SOURCE_PACKAGE + source)); + } catch (Exception alreadyRegistered) { + // Another IT in this JVM registered it first. + } + } + final SourceDecoratorManager decorators = new SourceDecoratorManager(); + for (final String decorator : DECORATORS) { + decorators.addIfAsSourceDecorator(Class.forName(SOURCE_PACKAGE + decorator)); + } + + final OALDefine define = new OALDefine(CORE_OAL, SOURCE_PACKAGE, "") { + }; + final List definitions; + try (Reader reader = new FileReader(coreOAL())) { + definitions = OALScriptParserV2.parse(reader, CORE_OAL).getMetrics(); + } + final MetricDefinitionEnricher enricher = new MetricDefinitionEnricher(SOURCE_PACKAGE, METRICS_PACKAGE); + final List models = new ArrayList<>(); + for (final MetricDefinition definition : definitions) { + models.add(enricher.enrich(definition)); + } + final OALClassGeneratorV2 generator = new OALClassGeneratorV2(define, new ClassPool(true)); + generator.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + final List metricsClasses = new ArrayList<>(); + generator.generateClassAtRuntime(models, new ArrayList<>(), metricsClasses, new ArrayList<>()); + + final Map> classes = new HashMap<>(); + for (final Class metricsClass : metricsClasses) { + classes.put(metricsClass.getAnnotation(Stream.class).name(), metricsClass.asSubclass(Metrics.class)); + } + return classes; + } + + private static File coreOAL() { + for (final String resources : STARTER_RESOURCES) { + final File file = new File(resources + CORE_OAL); + if (file.isFile()) { + return file; + } + } + throw new IllegalStateException( + "Cannot find the shipped " + CORE_OAL + " from " + new File("").getAbsolutePath()); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch1IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch1IT.java new file mode 100644 index 000000000000..68d53ca777a3 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch1IT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport.Mode; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.otlp.OTLPTraceQueryEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin.ZipkinQueryEsDAOITBase; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * The storage ITs on OpenSearch 1, in OAP's default storage mode, on one server for every nested IT. + */ +@Tag("slow") +@Tag("elastic-it") +@Testcontainers +public class OpenSearch1IT { + @Container + private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("1.3.20"); + private static final Mode MODE = Mode.DEFAULT; + + @Nested + class AIAgentConversations extends AIAgentConversationQueryITBase { + AIAgentConversations() { + super(SERVER, MODE); + } + } + + @Nested + class Alarms extends AlarmQueryITBase { + Alarms() { + super(SERVER, MODE); + } + } + + @Nested + class BatchProcess extends BatchProcessEsDAOITBase { + BatchProcess() { + super(SERVER, MODE); + } + } + + @Nested + class BrowserLogs extends BrowserLogQueryITBase { + BrowserLogs() { + super(SERVER, MODE); + } + } + + @Nested + class EBPFProfiling extends EBPFProfilingQueryITBase { + EBPFProfiling() { + super(SERVER, MODE); + } + } + + @Nested + class Events extends EventQueryITBase { + Events() { + super(SERVER, MODE); + } + } + + @Nested + class GenAIEvaluations extends GenAIEvaluationQueryITBase { + GenAIEvaluations() { + super(SERVER, MODE); + } + } + + @Nested + class Hierarchy extends HierarchyQueryITBase { + Hierarchy() { + super(SERVER, MODE); + } + } + + @Nested + class HistoryDelete extends HistoryDeleteEsDAOITBase { + HistoryDelete() { + super(SERVER, MODE); + } + } + + @Nested + class Logs extends LogQueryITBase { + Logs() { + super(SERVER, MODE); + } + } + + @Nested + class ManagementData extends ManagementDataITBase { + ManagementData() { + super(SERVER, MODE); + } + } + + @Nested + class Metadata extends MetadataQueryITBase { + Metadata() { + super(SERVER, MODE); + } + } + + @Nested + class Metrics extends MetricsQueryITBase { + Metrics() { + super(SERVER, MODE); + } + } + + @Nested + class MetricsPersistence extends MetricsPersistenceITBase { + MetricsPersistence() { + super(SERVER, MODE); + } + } + + @Nested + class OTLPTraces extends OTLPTraceQueryEsDAOITBase { + OTLPTraces() { + super(SERVER, MODE); + } + } + + @Nested + class ProfileThreadSnapshots extends ProfileThreadSnapshotQueryITBase { + ProfileThreadSnapshots() { + super(SERVER, MODE); + } + } + + @Nested + class ProfilingTasks extends ProfilingTaskQueryITBase { + ProfilingTasks() { + super(SERVER, MODE); + } + } + + @Nested + class SpanAttachedEvents extends SpanAttachedEventQueryITBase { + SpanAttachedEvents() { + super(SERVER, MODE); + } + } + + @Nested + class TagAutocomplete extends TagAutocompleteQueryITBase { + TagAutocomplete() { + super(SERVER, MODE); + } + } + + @Nested + class Topology extends TopologyQueryITBase { + Topology() { + super(SERVER, MODE); + } + } + + @Nested + class Traces extends TraceQueryEsDAOITBase { + Traces() { + super(SERVER, MODE); + } + } + + @Nested + class Zipkin extends ZipkinQueryEsDAOITBase { + Zipkin() { + super(SERVER, MODE); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch2IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch2IT.java new file mode 100644 index 000000000000..70a17cfff34f --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch2IT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport.Mode; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.otlp.OTLPTraceQueryEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin.ZipkinQueryEsDAOITBase; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * The storage ITs on OpenSearch 2, in OAP's default storage mode, on one server for every nested IT. + */ +@Tag("slow") +@Tag("elastic-it") +@Testcontainers +public class OpenSearch2IT { + @Container + private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("2.8.0"); + private static final Mode MODE = Mode.DEFAULT; + + @Nested + class AIAgentConversations extends AIAgentConversationQueryITBase { + AIAgentConversations() { + super(SERVER, MODE); + } + } + + @Nested + class Alarms extends AlarmQueryITBase { + Alarms() { + super(SERVER, MODE); + } + } + + @Nested + class BatchProcess extends BatchProcessEsDAOITBase { + BatchProcess() { + super(SERVER, MODE); + } + } + + @Nested + class BrowserLogs extends BrowserLogQueryITBase { + BrowserLogs() { + super(SERVER, MODE); + } + } + + @Nested + class EBPFProfiling extends EBPFProfilingQueryITBase { + EBPFProfiling() { + super(SERVER, MODE); + } + } + + @Nested + class Events extends EventQueryITBase { + Events() { + super(SERVER, MODE); + } + } + + @Nested + class GenAIEvaluations extends GenAIEvaluationQueryITBase { + GenAIEvaluations() { + super(SERVER, MODE); + } + } + + @Nested + class Hierarchy extends HierarchyQueryITBase { + Hierarchy() { + super(SERVER, MODE); + } + } + + @Nested + class HistoryDelete extends HistoryDeleteEsDAOITBase { + HistoryDelete() { + super(SERVER, MODE); + } + } + + @Nested + class Logs extends LogQueryITBase { + Logs() { + super(SERVER, MODE); + } + } + + @Nested + class ManagementData extends ManagementDataITBase { + ManagementData() { + super(SERVER, MODE); + } + } + + @Nested + class Metadata extends MetadataQueryITBase { + Metadata() { + super(SERVER, MODE); + } + } + + @Nested + class Metrics extends MetricsQueryITBase { + Metrics() { + super(SERVER, MODE); + } + } + + @Nested + class MetricsPersistence extends MetricsPersistenceITBase { + MetricsPersistence() { + super(SERVER, MODE); + } + } + + @Nested + class OTLPTraces extends OTLPTraceQueryEsDAOITBase { + OTLPTraces() { + super(SERVER, MODE); + } + } + + @Nested + class ProfileThreadSnapshots extends ProfileThreadSnapshotQueryITBase { + ProfileThreadSnapshots() { + super(SERVER, MODE); + } + } + + @Nested + class ProfilingTasks extends ProfilingTaskQueryITBase { + ProfilingTasks() { + super(SERVER, MODE); + } + } + + @Nested + class SpanAttachedEvents extends SpanAttachedEventQueryITBase { + SpanAttachedEvents() { + super(SERVER, MODE); + } + } + + @Nested + class TagAutocomplete extends TagAutocompleteQueryITBase { + TagAutocomplete() { + super(SERVER, MODE); + } + } + + @Nested + class Topology extends TopologyQueryITBase { + Topology() { + super(SERVER, MODE); + } + } + + @Nested + class Traces extends TraceQueryEsDAOITBase { + Traces() { + super(SERVER, MODE); + } + } + + @Nested + class Zipkin extends ZipkinQueryEsDAOITBase { + Zipkin() { + super(SERVER, MODE); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch3IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch3IT.java new file mode 100644 index 000000000000..dd06c8baf888 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch3IT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport.Mode; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.otlp.OTLPTraceQueryEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin.ZipkinQueryEsDAOITBase; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * The storage ITs on OpenSearch 3, in OAP's default storage mode, on one server for every nested IT. + */ +@Tag("slow") +@Tag("elastic-it") +@Testcontainers +public class OpenSearch3IT { + @Container + private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("3.0.0"); + private static final Mode MODE = Mode.DEFAULT; + + @Nested + class AIAgentConversations extends AIAgentConversationQueryITBase { + AIAgentConversations() { + super(SERVER, MODE); + } + } + + @Nested + class Alarms extends AlarmQueryITBase { + Alarms() { + super(SERVER, MODE); + } + } + + @Nested + class BatchProcess extends BatchProcessEsDAOITBase { + BatchProcess() { + super(SERVER, MODE); + } + } + + @Nested + class BrowserLogs extends BrowserLogQueryITBase { + BrowserLogs() { + super(SERVER, MODE); + } + } + + @Nested + class EBPFProfiling extends EBPFProfilingQueryITBase { + EBPFProfiling() { + super(SERVER, MODE); + } + } + + @Nested + class Events extends EventQueryITBase { + Events() { + super(SERVER, MODE); + } + } + + @Nested + class GenAIEvaluations extends GenAIEvaluationQueryITBase { + GenAIEvaluations() { + super(SERVER, MODE); + } + } + + @Nested + class Hierarchy extends HierarchyQueryITBase { + Hierarchy() { + super(SERVER, MODE); + } + } + + @Nested + class HistoryDelete extends HistoryDeleteEsDAOITBase { + HistoryDelete() { + super(SERVER, MODE); + } + } + + @Nested + class Logs extends LogQueryITBase { + Logs() { + super(SERVER, MODE); + } + } + + @Nested + class ManagementData extends ManagementDataITBase { + ManagementData() { + super(SERVER, MODE); + } + } + + @Nested + class Metadata extends MetadataQueryITBase { + Metadata() { + super(SERVER, MODE); + } + } + + @Nested + class Metrics extends MetricsQueryITBase { + Metrics() { + super(SERVER, MODE); + } + } + + @Nested + class MetricsPersistence extends MetricsPersistenceITBase { + MetricsPersistence() { + super(SERVER, MODE); + } + } + + @Nested + class OTLPTraces extends OTLPTraceQueryEsDAOITBase { + OTLPTraces() { + super(SERVER, MODE); + } + } + + @Nested + class ProfileThreadSnapshots extends ProfileThreadSnapshotQueryITBase { + ProfileThreadSnapshots() { + super(SERVER, MODE); + } + } + + @Nested + class ProfilingTasks extends ProfilingTaskQueryITBase { + ProfilingTasks() { + super(SERVER, MODE); + } + } + + @Nested + class SpanAttachedEvents extends SpanAttachedEventQueryITBase { + SpanAttachedEvents() { + super(SERVER, MODE); + } + } + + @Nested + class TagAutocomplete extends TagAutocompleteQueryITBase { + TagAutocomplete() { + super(SERVER, MODE); + } + } + + @Nested + class Topology extends TopologyQueryITBase { + Topology() { + super(SERVER, MODE); + } + } + + @Nested + class Traces extends TraceQueryEsDAOITBase { + Traces() { + super(SERVER, MODE); + } + } + + @Nested + class Zipkin extends ZipkinQueryEsDAOITBase { + Zipkin() { + super(SERVER, MODE); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch3ShardedIT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch3ShardedIT.java new file mode 100644 index 000000000000..179ea65b68f5 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/OpenSearch3ShardedIT.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport.Mode; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.BatchProcessEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.HistoryDeleteEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.otlp.OTLPTraceQueryEsDAOITBase; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin.ZipkinQueryEsDAOITBase; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Tag; +import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +/** + * The storage ITs on OpenSearch 3, with logic sharding and custom routing, on one server for every nested IT. + */ +@Tag("slow") +@Tag("elastic-it") +@Testcontainers +public class OpenSearch3ShardedIT { + @Container + private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("3.0.0"); + private static final Mode MODE = Mode.LOGIC_SHARDING_AND_CUSTOM_ROUTING; + + @Nested + class AIAgentConversations extends AIAgentConversationQueryITBase { + AIAgentConversations() { + super(SERVER, MODE); + } + } + + @Nested + class Alarms extends AlarmQueryITBase { + Alarms() { + super(SERVER, MODE); + } + } + + @Nested + class BatchProcess extends BatchProcessEsDAOITBase { + BatchProcess() { + super(SERVER, MODE); + } + } + + @Nested + class BrowserLogs extends BrowserLogQueryITBase { + BrowserLogs() { + super(SERVER, MODE); + } + } + + @Nested + class EBPFProfiling extends EBPFProfilingQueryITBase { + EBPFProfiling() { + super(SERVER, MODE); + } + } + + @Nested + class Events extends EventQueryITBase { + Events() { + super(SERVER, MODE); + } + } + + @Nested + class GenAIEvaluations extends GenAIEvaluationQueryITBase { + GenAIEvaluations() { + super(SERVER, MODE); + } + } + + @Nested + class Hierarchy extends HierarchyQueryITBase { + Hierarchy() { + super(SERVER, MODE); + } + } + + @Nested + class HistoryDelete extends HistoryDeleteEsDAOITBase { + HistoryDelete() { + super(SERVER, MODE); + } + } + + @Nested + class Logs extends LogQueryITBase { + Logs() { + super(SERVER, MODE); + } + } + + @Nested + class ManagementData extends ManagementDataITBase { + ManagementData() { + super(SERVER, MODE); + } + } + + @Nested + class Metadata extends MetadataQueryITBase { + Metadata() { + super(SERVER, MODE); + } + } + + @Nested + class Metrics extends MetricsQueryITBase { + Metrics() { + super(SERVER, MODE); + } + } + + @Nested + class MetricsPersistence extends MetricsPersistenceITBase { + MetricsPersistence() { + super(SERVER, MODE); + } + } + + @Nested + class OTLPTraces extends OTLPTraceQueryEsDAOITBase { + OTLPTraces() { + super(SERVER, MODE); + } + } + + @Nested + class ProfileThreadSnapshots extends ProfileThreadSnapshotQueryITBase { + ProfileThreadSnapshots() { + super(SERVER, MODE); + } + } + + @Nested + class ProfilingTasks extends ProfilingTaskQueryITBase { + ProfilingTasks() { + super(SERVER, MODE); + } + } + + @Nested + class SpanAttachedEvents extends SpanAttachedEventQueryITBase { + SpanAttachedEvents() { + super(SERVER, MODE); + } + } + + @Nested + class TagAutocomplete extends TagAutocompleteQueryITBase { + TagAutocomplete() { + super(SERVER, MODE); + } + } + + @Nested + class Topology extends TopologyQueryITBase { + Topology() { + super(SERVER, MODE); + } + } + + @Nested + class Traces extends TraceQueryEsDAOITBase { + Traces() { + super(SERVER, MODE); + } + } + + @Nested + class Zipkin extends ZipkinQueryEsDAOITBase { + Zipkin() { + super(SERVER, MODE); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfileThreadSnapshotQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfileThreadSnapshotQueryITBase.java new file mode 100644 index 000000000000..2d6494b8b104 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfileThreadSnapshotQueryITBase.java @@ -0,0 +1,216 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.trace.ProfileLanguageType; +import org.apache.skywalking.oap.server.core.profiling.trace.ProfileThreadSnapshotRecord; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads trace-profiling thread snapshots the way the profile analysis does on Elasticsearch and OpenSearch: a task's + * profiled segments (the snapshots of sequence 0), latest dump first; a segment's sequence range in a dump-time + * window, both ends included, -1 when there is none; and the snapshots of {@code [minSequence, maxSequence)}, which + * the analysis reads in adjacent batches that meet at their ends. Each snapshot is read with its language, so a Go + * profile is told from a Java one. A nested class of a server's IT names the server and the storage mode. + * + *

The Java segment is dumped on both sides of a midnight, so its snapshots lie in two daily indices. A segment of + * another task is dumped within the same windows, with sequences both inside and beyond the Java segment's. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class ProfileThreadSnapshotQueryITBase { + /** A fixed local midnight, so the daily indices the fixture is written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 3, 10, 0, 0).getMillis(); + private static final String JAVA_TASK = + (MIDNIGHT - 600_000L) + Const.ID_CONNECTOR + IDManager.ServiceID.buildId("checkout", true); + private static final String GO_TASK = + (MIDNIGHT - 300_000L) + Const.ID_CONNECTOR + IDManager.ServiceID.buildId("payment", true); + private static final String OTHER_TASK = + (MIDNIGHT - 900_000L) + Const.ID_CONNECTOR + IDManager.ServiceID.buildId("inventory", true); + private static final String JAVA_SEGMENT = "segment-java"; + /** The Java agent dumps the thread of a profiled segment every dump period, the first three before midnight. */ + private static final int JAVA_SNAPSHOTS = 6; + private static final long DUMP_PERIOD = 10L; + /** The Go agent reports one snapshot per segment, at the time of its profile. */ + private static final String GO_SEGMENT_YESTERDAY = "segment-go-1"; + private static final String GO_SEGMENT_TODAY = "segment-go-2"; + private static final String OTHER_SEGMENT = "segment-other"; + private static final int OTHER_SNAPSHOTS = 12; + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private ProfileThreadSnapshotQueryEsDAO dao; + + ProfileThreadSnapshotQueryITBase(final ElasticsearchContainer server, + final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeSnapshots() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final Model model = support.installRecord(ProfileThreadSnapshotRecord.class); + final List snapshots = new ArrayList<>(); + for (int sequence = 0; sequence < JAVA_SNAPSHOTS; sequence++) { + snapshots.add(javaSnapshot(sequence)); + } + snapshots.add(goSnapshot(GO_SEGMENT_YESTERDAY, MIDNIGHT - 5_000L)); + snapshots.add(goSnapshot(GO_SEGMENT_TODAY, MIDNIGHT + 5_000L)); + for (int sequence = 0; sequence < OTHER_SNAPSHOTS; sequence++) { + snapshots.add(snapshot(OTHER_TASK, OTHER_SEGMENT, sequence, dumpTime(0) + sequence * DUMP_PERIOD / 2, + ProfileLanguageType.JAVA)); + } + support.writeRecords(model, new ProfileThreadSnapshotRecord.Builder(), snapshots); + dao = new ProfileThreadSnapshotQueryEsDAO( + support.getClient(), support.getConfig().getProfileTaskQueryMaxSize()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheProfiledSegmentsOfATaskLatestDumpFirst() throws IOException { + assertEquals(describe(Arrays.asList(goSnapshot(GO_SEGMENT_TODAY, MIDNIGHT + 5_000L), + goSnapshot(GO_SEGMENT_YESTERDAY, MIDNIGHT - 5_000L))), + describe(dao.queryRecords(GO_TASK))); + assertEquals(describe(Collections.singletonList(javaSnapshot(0))), describe(dao.queryRecords(JAVA_TASK))); + assertEquals(Collections.emptyList(), dao.queryRecords("task-missing")); + } + + /** + * The analysis of a span reads the sequences dumped within the span's time, which here crosses midnight. + */ + @Test + public void findsTheSequenceRangeOfADumpTimeWindow() throws IOException { + assertEquals(Arrays.asList(1, 4), sequenceRange(JAVA_SEGMENT, dumpTime(1), dumpTime(4))); + assertEquals(Arrays.asList(2, 3), sequenceRange(JAVA_SEGMENT, dumpTime(1) + 1, dumpTime(4) - 1)); + assertEquals(Arrays.asList(4, 5), sequenceRange(JAVA_SEGMENT, dumpTime(4), dumpTime(JAVA_SNAPSHOTS))); + } + + /** + * The Go fallback of the analysis reads the whole sequence range of a segment, whatever its dump times. + */ + @Test + public void findsTheWholeSequenceRangeOfASegment() throws IOException { + assertEquals(Arrays.asList(0, JAVA_SNAPSHOTS - 1), sequenceRange(JAVA_SEGMENT, 0L, Long.MAX_VALUE)); + assertEquals(Arrays.asList(0, 0), sequenceRange(GO_SEGMENT_TODAY, 0L, Long.MAX_VALUE)); + } + + @Test + public void findsNoSequenceInAnEmptyWindow() throws IOException { + final long afterLastDump = dumpTime(JAVA_SNAPSHOTS - 1) + 1; + assertEquals(Arrays.asList(-1, -1), sequenceRange(JAVA_SEGMENT, afterLastDump, afterLastDump + 1_000L)); + assertEquals(Arrays.asList(-1, -1), sequenceRange("segment-missing", 0L, Long.MAX_VALUE)); + } + + @Test + public void readsAdjacentSequenceBatchesWithoutOverlap() throws IOException { + assertEquals(describe(Arrays.asList(javaSnapshot(0), javaSnapshot(1), javaSnapshot(2))), + describeBySequence(dao.queryRecords(JAVA_SEGMENT, 0, 3))); + assertEquals(describe(Arrays.asList(javaSnapshot(3), javaSnapshot(4), javaSnapshot(5))), + describeBySequence(dao.queryRecords(JAVA_SEGMENT, 3, 6))); + assertEquals(Collections.emptyList(), dao.queryRecords(JAVA_SEGMENT, JAVA_SNAPSHOTS, JAVA_SNAPSHOTS + 3)); + } + + /** + * The analysis falls back to the Go profile of a segment, which it keeps only when the snapshot reads as Go. + */ + @Test + public void readsTheGoSnapshotOfASegmentWhole() throws IOException { + assertEquals(describe(Collections.singletonList(goSnapshot(GO_SEGMENT_TODAY, MIDNIGHT + 5_000L))), + describe(dao.queryRecords(GO_SEGMENT_TODAY, 0, 1))); + } + + private List sequenceRange(final String segmentId, final long start, final long end) { + return Arrays.asList(dao.queryMinSequence(segmentId, start, end), dao.queryMaxSequence(segmentId, start, end)); + } + + private static List describe(final List snapshots) { + return snapshots.stream() + .map(snapshot -> String.join( + "|", snapshot.getTaskId(), snapshot.getSegmentId(), + String.valueOf(snapshot.getSequence()), String.valueOf(snapshot.getDumpTime()), + new String(snapshot.getStackBinary(), StandardCharsets.UTF_8), + snapshot.getLanguage().name(), String.valueOf(snapshot.getTimeBucket()))) + .collect(Collectors.toList()); + } + + /** + * A batch of sequences has no promised order. + */ + private static List describeBySequence(final List snapshots) { + return describe(snapshots.stream() + .sorted(Comparator.comparingInt(ProfileThreadSnapshotRecord::getSequence)) + .collect(Collectors.toList())); + } + + private static long dumpTime(final int sequence) { + return MIDNIGHT + (sequence - JAVA_SNAPSHOTS / 2) * DUMP_PERIOD; + } + + private static ProfileThreadSnapshotRecord javaSnapshot(final int sequence) { + return snapshot(JAVA_TASK, JAVA_SEGMENT, sequence, dumpTime(sequence), ProfileLanguageType.JAVA); + } + + private static ProfileThreadSnapshotRecord goSnapshot(final String segmentId, final long dumpTime) { + return snapshot(GO_TASK, segmentId, 0, dumpTime, ProfileLanguageType.GO); + } + + /** + * A snapshot as the profile receiver writes it, the stack being the dumped thread stack of a Java agent or the + * segment's samples of a Go agent's pprof. + */ + private static ProfileThreadSnapshotRecord snapshot(final String taskId, final String segmentId, final int sequence, + final long dumpTime, final ProfileLanguageType language) { + final ProfileThreadSnapshotRecord snapshot = new ProfileThreadSnapshotRecord(); + snapshot.setTaskId(taskId); + snapshot.setSegmentId(segmentId); + snapshot.setDumpTime(dumpTime); + snapshot.setSequence(sequence); + snapshot.setStackBinary(("stack " + segmentId + " " + sequence).getBytes(StandardCharsets.UTF_8)); + snapshot.setLanguage(language); + snapshot.setTimeBucket(TimeBucket.getRecordTimeBucket(dumpTime)); + return snapshot; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfilingTaskQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfilingTaskQueryITBase.java new file mode 100644 index 000000000000..94d80ccfa1ba --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/ProfilingTaskQueryITBase.java @@ -0,0 +1,962 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import com.google.common.hash.Hashing; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.ToLongFunction; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.AsyncProfilerMutationService; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.AsyncProfilerQueryService; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskLogRecord; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.JFRProfilingDataRecord; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskLogRecord; +import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskRecord; +import org.apache.skywalking.oap.server.core.query.AsyncProfilerTaskLog; +import org.apache.skywalking.oap.server.core.query.PprofTaskLog; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerEventType; +import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; +import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTaskCreationType; +import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTaskLogOperationType; +import org.apache.skywalking.oap.server.core.query.type.PprofEventType; +import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationType; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; +import org.apache.skywalking.oap.server.core.query.type.ProfileTask; +import org.apache.skywalking.oap.server.core.query.type.ProfileTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.library.jfr.type.JFREventType; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.StorageModuleElasticsearchConfig; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.IndexController; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Reads trace-profiling, async-profiler and pprof tasks, their logs and the async-profiler (JFR) and pprof data the + * way the profiling query services, the mutation services and the agent task caches do on Elasticsearch and + * OpenSearch. OAP writes a task at once through its none-stream DAO, into the index of the task's time bucket: a + * trace-profiling task into the day it starts, an async-profiler or pprof task into the day it is created. Logs and + * data are records, written into the index of the day they are reported. A nested class of a server's IT names the + * server and the storage mode. + * + *

The fixture lies on both sides of a midnight, so every read spans two daily indices. Every filter has a near miss: + * a task of another service, of another endpoint, created or starting a second or a minute outside a time range, a + * log of another task, data of another task, instance or event. In the default mode the task, log and data models + * share one physical index, so a read missing its model filter would also see the other models' documents. A task's + * data is more than a search returns without a size, and some of its uploads are too large to share a response. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class ProfilingTaskQueryITBase { + /** A fixed local midnight, so the daily indices the fixture is written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 3, 10, 0, 0).getMillis(); + private static final String SERVICE_ID = IDManager.ServiceID.buildId("checkout", true); + private static final String OTHER_SERVICE_ID = IDManager.ServiceID.buildId("payment", true); + private static final String INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-1"); + private static final String OTHER_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-2"); + /** A service with two async-profiler and two pprof tasks created in the same minute, half a minute apart. */ + private static final String SAME_MINUTE_SERVICE_ID = IDManager.ServiceID.buildId("inventory", true); + private static final int SAME_MINUTE = 5; + /** A service whose tasks the creation tests write as running now. */ + private static final String RUNNING_SERVICE_ID = IDManager.ServiceID.buildId("shipping", true); + private static final int TRACE_PROFILING_MINUTES = 10; + /** When each async-profiler and pprof task of the service was created, in minutes from midnight. */ + private static final int[] CREATED_MINUTES = {-30, -20, -10, 10, 20}; + /** When the other service's async-profiler and pprof task was created, in minutes from midnight. */ + private static final int OTHER_SERVICE_CREATED_MINUTE = -15; + /** The task whose logs and data are read: created ten minutes before midnight, reported on both sides of it. */ + private static final int READ_TASK_MINUTE = -10; + /** Uploads of the read task by its first instance, more than a search returns without a size. */ + private static final int UPLOADS = 12; + /** As {@code application.yml} sets the async-profiler and pprof task query sizes. */ + private static final int TASK_QUERY_MAX_SIZE = 200; + /** The task with large uploads: three of them, each of which fits a response alone but not with the others. */ + private static final int LARGE_UPLOAD_TASK_MINUTE = 10; + private static final int LARGE_UPLOADS = 3; + /** Four MiB of frame tree, over five MiB as Base64: two make a response larger than Armeria's 10 MiB default. */ + private static final int LARGE_UPLOAD_BYTES = 4 * 1024 * 1024; + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + + private final ProfileTaskRecord startedYesterday = + traceProfilingTask(SERVICE_ID, "/pay", MIDNIGHT - minutes(10), MIDNIGHT - minutes(10)); + private final ProfileTaskRecord startedToday = + traceProfilingTask(SERVICE_ID, "/checkout", MIDNIGHT + minutes(1), MIDNIGHT + minutes(1)); + /** Created yesterday evening to start today at 10:00. */ + private final ProfileTaskRecord scheduledForToday = + traceProfilingTask(SERVICE_ID, "/checkout", MIDNIGHT - minutes(120), MIDNIGHT + minutes(600)); + private final ProfileTaskRecord otherServiceTask = + traceProfilingTask(OTHER_SERVICE_ID, "/checkout", MIDNIGHT + minutes(2), MIDNIGHT + minutes(2)); + private final List traceProfilingLogs = new ArrayList<>(); + private final List asyncProfilerLogs = new ArrayList<>(); + private final List pprofLogs = new ArrayList<>(); + private final List jfrData = new ArrayList<>(); + private final List pprofData = new ArrayList<>(); + private final List largeJfrData = new ArrayList<>(); + private final List largePprofData = new ArrayList<>(); + private Model traceProfilingModel; + private Model asyncProfilerModel; + private Model pprofModel; + private Model traceProfilingLogModel; + private Model asyncProfilerLogModel; + private Model pprofLogModel; + private Model jfrDataModel; + private Model pprofDataModel; + + private ProfileTaskQueryEsDAO traceProfilingDAO; + private ProfileTaskLogEsDAO traceProfilingLogDAO; + private AsyncProfilerTaskQueryEsDAO asyncProfilerDAO; + private AsyncProfilerTaskLogQueryEsDAO asyncProfilerLogDAO; + private JFRDataQueryEsDAO jfrDataDAO; + private PprofTaskQueryEsDAO pprofDAO; + private PprofTaskLogQueryEsDAO pprofLogDAO; + private PprofDataQueryEsDAO pprofDataDAO; + + ProfilingTaskQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeProfiling() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass(), config -> { + config.setAsyncProfilerTaskQueryMaxSize(TASK_QUERY_MAX_SIZE); + config.setPprofTaskQueryMaxSize(TASK_QUERY_MAX_SIZE); + }); + // As at boot, every model is installed before anything is written: in the default mode they share one + // index template, and an index created before a model is installed lacks the model's columns. + traceProfilingModel = support.installNoneStream(ProfileTaskRecord.class); + asyncProfilerModel = support.installNoneStream(AsyncProfilerTaskRecord.class); + pprofModel = support.installNoneStream(PprofTaskRecord.class); + traceProfilingLogModel = support.installRecord(ProfileTaskLogRecord.class); + asyncProfilerLogModel = support.installRecord(AsyncProfilerTaskLogRecord.class); + pprofLogModel = support.installRecord(PprofTaskLogRecord.class); + jfrDataModel = support.installRecord(JFRProfilingDataRecord.class); + pprofDataModel = support.installRecord(PprofProfilingDataRecord.class); + writeTasks(); + writeLogs(); + writeData(); + + final ElasticSearchClient client = support.getClient(); + final StorageModuleElasticsearchConfig config = support.getConfig(); + traceProfilingDAO = new ProfileTaskQueryEsDAO(client, config.getProfileTaskQueryMaxSize()); + traceProfilingLogDAO = new ProfileTaskLogEsDAO(client, config.getProfileTaskQueryMaxSize()); + asyncProfilerDAO = new AsyncProfilerTaskQueryEsDAO(client, config.getAsyncProfilerTaskQueryMaxSize()); + asyncProfilerLogDAO = new AsyncProfilerTaskLogQueryEsDAO(client, config.getAsyncProfilerTaskQueryMaxSize()); + jfrDataDAO = new JFRDataQueryEsDAO(client); + pprofDAO = new PprofTaskQueryEsDAO(client, config.getPprofTaskQueryMaxSize()); + pprofLogDAO = new PprofTaskLogQueryEsDAO(client, config.getPprofTaskQueryMaxSize()); + pprofDataDAO = new PprofDataQueryEsDAO(client); + } + + private void writeTasks() throws Exception { + for (final ProfileTaskRecord task : Arrays.asList( + startedYesterday, startedToday, scheduledForToday, otherServiceTask)) { + support.writeNoneStream(traceProfilingModel, new ProfileTaskRecord.Builder(), task); + } + for (final int minute : CREATED_MINUTES) { + support.writeNoneStream( + asyncProfilerModel, new AsyncProfilerTaskRecord.Builder(), asyncProfilerTask(SERVICE_ID, minute)); + support.writeNoneStream(pprofModel, new PprofTaskRecord.Builder(), pprofTask(SERVICE_ID, minute)); + } + support.writeNoneStream(asyncProfilerModel, new AsyncProfilerTaskRecord.Builder(), + asyncProfilerTask(OTHER_SERVICE_ID, OTHER_SERVICE_CREATED_MINUTE)); + support.writeNoneStream( + pprofModel, new PprofTaskRecord.Builder(), pprofTask(OTHER_SERVICE_ID, OTHER_SERVICE_CREATED_MINUTE)); + for (final long createTime : sameMinuteCreateTimes()) { + support.writeNoneStream(asyncProfilerModel, new AsyncProfilerTaskRecord.Builder(), + asyncProfilerTask(SAME_MINUTE_SERVICE_ID, createTime)); + support.writeNoneStream( + pprofModel, new PprofTaskRecord.Builder(), pprofTask(SAME_MINUTE_SERVICE_ID, createTime)); + } + } + + private static long[] sameMinuteCreateTimes() { + return new long[] {createTime(SAME_MINUTE) + 10_000L, createTime(SAME_MINUTE) + 40_000L}; + } + + /** + * The task started yesterday finishes after midnight; the task started today is notified to two instances. + * The async-profiler and pprof task read has a log on each side of midnight, and the tasks before and after it + * have logs of the same instance. + */ + private void writeLogs() throws Exception { + traceProfilingLogs.add(traceProfilingLog(startedYesterday, INSTANCE_ID, ProfileTaskLogOperationType.NOTIFIED, + MIDNIGHT - minutes(9))); + traceProfilingLogs.add(traceProfilingLog(startedYesterday, INSTANCE_ID, + ProfileTaskLogOperationType.EXECUTION_FINISHED, + MIDNIGHT + minutes(1))); + traceProfilingLogs.add(traceProfilingLog(startedToday, INSTANCE_ID, ProfileTaskLogOperationType.NOTIFIED, + MIDNIGHT + minutes(1) + 10_000L)); + traceProfilingLogs.add(traceProfilingLog(startedToday, OTHER_INSTANCE_ID, ProfileTaskLogOperationType.NOTIFIED, + MIDNIGHT + minutes(1) + 20_000L)); + support.writeRecords(traceProfilingLogModel, new ProfileTaskLogRecord.Builder(), traceProfilingLogs); + + // Each log: the minute its task was created, the minute it is reported, its operation type. + for (final int[] log : new int[][] { + {READ_TASK_MINUTE, -9, 1}, {READ_TASK_MINUTE, 1, 2}, {-20, -19, 1}, {-20, -14, 4}, {10, 11, 1}}) { + asyncProfilerLogs.add(asyncProfilerLog(taskId(SERVICE_ID, log[0]), MIDNIGHT + minutes(log[1]), log[2])); + pprofLogs.add(pprofLog(taskId(SERVICE_ID, log[0]), MIDNIGHT + minutes(log[1]), log[2])); + } + support.writeRecords(asyncProfilerLogModel, new AsyncProfilerTaskLogRecord.Builder(), asyncProfilerLogs); + support.writeRecords(pprofLogModel, new PprofTaskLogRecord.Builder(), pprofLogs); + } + + /** + * Uploads every second from six seconds before midnight. The read task also has uploads by its other instance, + * uploads of another event, and the task before it uploads of the same instance and event. + */ + private void writeData() throws Exception { + final String readTask = taskId(SERVICE_ID, READ_TASK_MINUTE); + final String otherTask = taskId(SERVICE_ID, -20); + final String cpu = JFREventType.EXECUTION_SAMPLE.name(); + final String lock = JFREventType.LOCK.name(); + for (int i = 0; i < UPLOADS; i++) { + jfrData.add(jfr(readTask, INSTANCE_ID, cpu, i)); + pprofData.add(pprof(readTask, INSTANCE_ID, i)); + } + for (int i = 0; i < 3; i++) { + jfrData.add(jfr(readTask, OTHER_INSTANCE_ID, cpu, i)); + pprofData.add(pprof(readTask, OTHER_INSTANCE_ID, i)); + } + for (int i = 0; i < 2; i++) { + jfrData.add(jfr(readTask, INSTANCE_ID, lock, i)); + jfrData.add(jfr(otherTask, INSTANCE_ID, cpu, i)); + pprofData.add(pprof(otherTask, INSTANCE_ID, i)); + } + support.writeRecords(jfrDataModel, new JFRProfilingDataRecord.Builder(), jfrData); + support.writeRecords(pprofDataModel, new PprofProfilingDataRecord.Builder(), pprofData); + + final String largeTask = taskId(SERVICE_ID, LARGE_UPLOAD_TASK_MINUTE); + final Random random = new Random(LARGE_UPLOAD_BYTES); + for (int i = 0; i < LARGE_UPLOADS; i++) { + final JFRProfilingDataRecord jfrUpload = jfr(largeTask, INSTANCE_ID, cpu, i); + jfrUpload.setDataBinary(randomBytes(random)); + largeJfrData.add(jfrUpload); + final PprofProfilingDataRecord pprofUpload = pprof(largeTask, INSTANCE_ID, i); + pprofUpload.setDataBinary(randomBytes(random)); + largePprofData.add(pprofUpload); + } + support.writeRecords(jfrDataModel, new JFRProfilingDataRecord.Builder(), largeJfrData); + support.writeRecords(pprofDataModel, new PprofProfilingDataRecord.Builder(), largePprofData); + } + + private static byte[] randomBytes(final Random random) { + final byte[] bytes = new byte[LARGE_UPLOAD_BYTES]; + random.nextBytes(bytes); + return bytes; + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void listsTheTraceProfilingTasksOfAServiceLatestStartFirst() throws IOException { + assertEquals(describe(scheduledForToday, startedToday, startedYesterday), + describe(traceProfilingDAO.getTaskList(SERVICE_ID, null, null, null, null))); + } + + @Test + public void listsTheTraceProfilingTasksOfAnEndpoint() throws IOException { + assertEquals(describe(scheduledForToday, startedToday), + describe(traceProfilingDAO.getTaskList(SERVICE_ID, "/checkout", null, null, null))); + } + + /** + * The agent task cache reads the tasks of every service starting within a window of minute buckets, both ends + * included. + */ + @Test + public void readsTheTraceProfilingTasksStartingWithinAWindow() throws IOException { + final long first = TimeBucket.getMinuteTimeBucket(startedYesterday.getStartTime()); + final long last = TimeBucket.getMinuteTimeBucket(startedToday.getStartTime()); + assertEquals(describe(startedToday, startedYesterday), + describe(traceProfilingDAO.getTaskList(null, null, first, last, null))); + assertEquals(describe(otherServiceTask, startedToday, startedYesterday), + describe(traceProfilingDAO.getTaskList( + null, null, first, TimeBucket.getMinuteTimeBucket(otherServiceTask.getStartTime()), null))); + assertEquals(Collections.emptyList(), describe(traceProfilingDAO.getTaskList( + null, null, TimeBucket.getMinuteTimeBucket(startedYesterday.getStartTime() + minutes(1)), + TimeBucket.getMinuteTimeBucket(startedToday.getStartTime() - minutes(1)), null))); + } + + /** + * Creating a task checks the latest-starting task of the service that starts before the new one ends for an + * overlap. + */ + @Test + public void readsTheLatestStartingTraceProfilingTaskBeforeAnEnd() throws IOException { + final long newTaskEnd = TimeBucket.getMinuteTimeBucket( + scheduledForToday.getStartTime() + minutes(5 + TRACE_PROFILING_MINUTES)); + assertEquals(describe(scheduledForToday), + describe(traceProfilingDAO.getTaskList(SERVICE_ID, null, null, newTaskEnd, 1))); + assertEquals(describe(startedToday), describe(traceProfilingDAO.getTaskList( + SERVICE_ID, null, null, TimeBucket.getMinuteTimeBucket(otherServiceTask.getStartTime()), 1))); + } + + @Test + public void readsATraceProfilingTaskById() throws IOException { + assertEquals(describe(scheduledForToday), + describe(Collections.singletonList(traceProfilingDAO.getById(scheduledForToday.getTaskId())))); + assertEquals(describe(startedYesterday), + describe(Collections.singletonList(traceProfilingDAO.getById(startedYesterday.getTaskId())))); + assertNull(traceProfilingDAO.getById(taskId(SERVICE_ID, 0))); + assertNull(traceProfilingDAO.getById("")); + } + + /** + * The trace-profiling service reads the logs of every task and matches them to the tasks it lists. + */ + @Test + public void readsTheLogsOfEveryTraceProfilingTaskNewestFirst() throws IOException { + final List expected = traceProfilingLogs.stream() + .sorted(Comparator.comparingLong(ProfileTaskLogRecord::getOperationTime).reversed()) + .map(log -> String.join( + "|", IndexController.INSTANCE.generateDocId(traceProfilingLogModel, log.id().build()), + log.getTaskId(), log.getInstanceId(), ProfileTaskLogOperationType.parse(log.getOperationType()).name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList()); + assertEquals(expected, traceProfilingLogDAO.getTaskLogList().stream() + .map(log -> String.join("|", log.getId(), log.getTaskId(), log.getInstanceId(), + log.getOperationType().name(), String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList())); + } + + @Test + public void listsTheAsyncProfilerTasksOfAServiceNewestFirst() throws IOException { + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, 20, 10, -10, -20, -30), + describeAsyncProfilerTasks(asyncProfilerDAO.getTaskList(SERVICE_ID, null, null, null))); + } + + /** + * The task list of a duration bounds the tasks' second buckets, both ends included, and cuts the newest of both + * days to the limit. + */ + @Test + public void listsTheAsyncProfilerTasksCreatedWithinADuration() throws IOException { + final long first = secondBucket(-20, 0); + final long last = secondBucket(10, 0); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, 10, -10, -20), + describeAsyncProfilerTasks(asyncProfilerDAO.getTaskList(SERVICE_ID, first, last, null))); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, 10, -10), + describeAsyncProfilerTasks(asyncProfilerDAO.getTaskList(SERVICE_ID, first, last, 2))); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, -10), describeAsyncProfilerTasks( + asyncProfilerDAO.getTaskList(SERVICE_ID, secondBucket(-20, 1), secondBucket(10, -1), null))); + } + + /** + * The agent task cache reads the tasks of every service within its window; creating a task checks the newest + * task of the service created up to now. + */ + @Test + public void readsTheAsyncProfilerTasksAsTheCacheAndTheTaskCreationDo() throws IOException { + final List cached = new ArrayList<>(describeAsyncProfilerTasks(SERVICE_ID, -10)); + cached.addAll(describeAsyncProfilerTasks(OTHER_SERVICE_ID, OTHER_SERVICE_CREATED_MINUTE)); + cached.addAll(describeAsyncProfilerTasks(SERVICE_ID, -20)); + assertEquals(cached, describeAsyncProfilerTasks( + asyncProfilerDAO.getTaskList(null, secondBucket(-20, 0), secondBucket(-10, 0), null))); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, -10), describeAsyncProfilerTasks( + asyncProfilerDAO.getTaskList(SERVICE_ID, null, secondBucket(10, -1), 1))); + } + + @Test + public void readsAnAsyncProfilerTaskById() throws IOException { + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, -30), describeAsyncProfilerTasks( + Collections.singletonList(asyncProfilerDAO.getById(taskId(SERVICE_ID, -30))))); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, 20), describeAsyncProfilerTasks( + Collections.singletonList(asyncProfilerDAO.getById(taskId(SERVICE_ID, 20))))); + assertNull(asyncProfilerDAO.getById(taskId(SERVICE_ID, 0))); + } + + @Test + public void readsTheLogsOfOneAsyncProfilerTaskNewestFirst() throws IOException { + final String task = taskId(SERVICE_ID, READ_TASK_MINUTE); + final List expected = asyncProfilerLogs.stream() + .filter(log -> log.getTaskId().equals(task)) + .sorted(Comparator.comparingLong(AsyncProfilerTaskLogRecord::getOperationTime).reversed()) + .map(log -> String.join("|", log.getTaskId(), log.getInstanceId(), + AsyncProfilerTaskLogOperationType.parse(log.getOperationType()).name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList()); + assertEquals(2, expected.size()); + assertEquals(expected, describeAsyncProfilerLogs(asyncProfilerLogDAO.getTaskLogList(task))); + assertEquals(Collections.emptyList(), asyncProfilerLogDAO.getTaskLogList(taskId(SERVICE_ID, 0))); + } + + @Test + public void listsThePprofTasksOfAServiceNewestFirst() throws IOException { + assertEquals(describePprofTasks(SERVICE_ID, 20, 10, -10, -20, -30), + describePprofTasks(pprofDAO.getTaskList(SERVICE_ID, null, null, null))); + } + + @Test + public void listsThePprofTasksCreatedWithinADuration() throws IOException { + final long first = secondBucket(-20, 0); + final long last = secondBucket(10, 0); + assertEquals(describePprofTasks(SERVICE_ID, 10, -10, -20), + describePprofTasks(pprofDAO.getTaskList(SERVICE_ID, first, last, null))); + assertEquals(describePprofTasks(SERVICE_ID, 10, -10), + describePprofTasks(pprofDAO.getTaskList(SERVICE_ID, first, last, 2))); + assertEquals(describePprofTasks(SERVICE_ID, -10), describePprofTasks( + pprofDAO.getTaskList(SERVICE_ID, secondBucket(-20, 1), secondBucket(10, -1), null))); + } + + /** + * Creating a pprof task checks every task of the service created up to now, not only the newest. + */ + @Test + public void readsThePprofTasksAsTheCacheAndTheTaskCreationDo() throws IOException { + final List cached = new ArrayList<>(describePprofTasks(SERVICE_ID, -10)); + cached.addAll(describePprofTasks(OTHER_SERVICE_ID, OTHER_SERVICE_CREATED_MINUTE)); + cached.addAll(describePprofTasks(SERVICE_ID, -20)); + assertEquals(cached, describePprofTasks( + pprofDAO.getTaskList(null, secondBucket(-20, 0), secondBucket(-10, 0), null))); + assertEquals(describePprofTasks(SERVICE_ID, -10, -20, -30), describePprofTasks( + pprofDAO.getTaskList(SERVICE_ID, null, secondBucket(10, -1), null))); + } + + @Test + public void readsAPprofTaskById() throws IOException { + assertEquals(describePprofTasks(SERVICE_ID, -30), + describePprofTasks(Collections.singletonList(pprofDAO.getById(taskId(SERVICE_ID, -30))))); + assertEquals(describePprofTasks(SERVICE_ID, 20), + describePprofTasks(Collections.singletonList(pprofDAO.getById(taskId(SERVICE_ID, 20))))); + assertNull(pprofDAO.getById(taskId(SERVICE_ID, 0))); + } + + @Test + public void readsTheLogsOfOnePprofTaskNewestFirst() throws IOException { + final String task = taskId(SERVICE_ID, READ_TASK_MINUTE); + final List expected = pprofLogs.stream() + .filter(log -> log.getTaskId().equals(task)) + .sorted(Comparator.comparingLong(PprofTaskLogRecord::getOperationTime).reversed()) + .map(log -> String.join("|", log.getTaskId(), log.getInstanceId(), + PprofTaskLogOperationType.parse(log.getOperationType()).name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList()); + assertEquals(2, expected.size()); + assertEquals(expected, describePprofLogs(pprofLogDAO.getTaskLogList(task))); + assertEquals(Collections.emptyList(), pprofLogDAO.getTaskLogList(taskId(SERVICE_ID, 0))); + assertEquals(Collections.emptyList(), pprofLogDAO.getTaskLogList("")); + } + + @Test + public void readsEveryJFRDataOfAnInstanceAndEvent() throws IOException { + final String task = taskId(SERVICE_ID, READ_TASK_MINUTE); + final String cpu = JFREventType.EXECUTION_SAMPLE.name(); + assertEquals(describeJFR(jfrData, task, Collections.singletonList(INSTANCE_ID), cpu), + describeJFR(jfrDataDAO.getByTaskIdAndInstancesAndEvent( + task, Collections.singletonList(INSTANCE_ID), cpu))); + } + + /** + * No instance listed reads the uploads of every instance of the task. + */ + @Test + public void readsTheJFRDataOfTheListedInstances() throws IOException { + final String task = taskId(SERVICE_ID, READ_TASK_MINUTE); + final String cpu = JFREventType.EXECUTION_SAMPLE.name(); + final List instances = Arrays.asList(INSTANCE_ID, OTHER_INSTANCE_ID); + assertEquals(describeJFR(jfrData, task, instances, cpu), + describeJFR(jfrDataDAO.getByTaskIdAndInstancesAndEvent(task, instances, cpu))); + assertEquals(describeJFR(jfrData, task, instances, cpu), + describeJFR(jfrDataDAO.getByTaskIdAndInstancesAndEvent(task, Collections.emptyList(), cpu))); + final String lock = JFREventType.LOCK.name(); + assertEquals(describeJFR(jfrData, task, instances, lock), + describeJFR(jfrDataDAO.getByTaskIdAndInstancesAndEvent(task, instances, lock))); + assertEquals(Collections.emptyList(), jfrDataDAO.getByTaskIdAndInstancesAndEvent(task, instances, "")); + } + + @Test + public void readsEveryPprofDataOfTheListedInstances() throws IOException { + final String task = taskId(SERVICE_ID, READ_TASK_MINUTE); + final List instance = Collections.singletonList(INSTANCE_ID); + assertEquals(describePprof(pprofData, task, instance), + describePprof(pprofDataDAO.getByTaskIdAndInstances(task, instance))); + final List instances = Arrays.asList(INSTANCE_ID, OTHER_INSTANCE_ID); + assertEquals(describePprof(pprofData, task, instances), + describePprof(pprofDataDAO.getByTaskIdAndInstances(task, instances))); + assertEquals(Collections.emptyList(), pprofDataDAO.getByTaskIdAndInstances("", instances)); + } + + /** + * No instance listed reads the uploads of every instance of the task, as on the other storages and as the JFR + * read does. + */ + @Test + public void readsThePprofDataOfEveryInstanceWhenNoneIsListed() throws IOException { + final String task = taskId(SERVICE_ID, READ_TASK_MINUTE); + assertEquals(describePprof(pprofData, task, Arrays.asList(INSTANCE_ID, OTHER_INSTANCE_ID)), + describePprof(pprofDataDAO.getByTaskIdAndInstances(task, Collections.emptyList()))); + } + + /** + * An upload is a frame tree of up to megabytes, which Elasticsearch returns as Base64 in the hit's source, and the + * client refuses a response larger than Armeria's 10 MiB default. These uploads fit a response one at a time, but + * not together. + */ + @Test + public void readsUploadsTooLargeToShareAResponse() throws IOException { + final String task = taskId(SERVICE_ID, LARGE_UPLOAD_TASK_MINUTE); + final List instance = Collections.singletonList(INSTANCE_ID); + final String cpu = JFREventType.EXECUTION_SAMPLE.name(); + assertEquals(fingerprints(largeJfrData, JFRProfilingDataRecord::getUploadTime, + JFRProfilingDataRecord::getDataBinary), + fingerprints(jfrDataDAO.getByTaskIdAndInstancesAndEvent(task, instance, cpu), + JFRProfilingDataRecord::getUploadTime, JFRProfilingDataRecord::getDataBinary)); + assertEquals(fingerprints(largePprofData, PprofProfilingDataRecord::getUploadTime, + PprofProfilingDataRecord::getDataBinary), + fingerprints(pprofDataDAO.getByTaskIdAndInstances(task, instance), + PprofProfilingDataRecord::getUploadTime, PprofProfilingDataRecord::getDataBinary)); + } + + /** + * The GraphQL task lists accept a duration of seconds, and the query services bound the tasks by its seconds: of + * two tasks created in the same minute, a duration ending between them, with a limit of one, lists the first. + */ + @Test + public void listsTheTasksOfADurationOfSecondsExactly() throws IOException { + final ModuleManager moduleManager = storageServices(); + final AsyncProfilerQueryService async = new AsyncProfilerQueryService(moduleManager); + final PprofQueryService pprof = new PprofQueryService(moduleManager); + final long[] created = sameMinuteCreateTimes(); + final long minuteStart = createTime(SAME_MINUTE); + final List first = Collections.singletonList(taskId(SAME_MINUTE_SERVICE_ID, created[0])); + final List both = Arrays.asList( + taskId(SAME_MINUTE_SERVICE_ID, created[1]), taskId(SAME_MINUTE_SERVICE_ID, created[0])); + final Duration endingBetween = secondsDuration(minuteStart, minuteStart + 30_000L); + assertEquals(first, asyncProfilerIds(async.queryTask(SAME_MINUTE_SERVICE_ID, endingBetween, 1))); + assertEquals(first, pprofIds(pprof.queryTask(SAME_MINUTE_SERVICE_ID, endingBetween, 1))); + final Duration onBoth = secondsDuration(created[0], created[1]); + assertEquals(both, asyncProfilerIds(async.queryTask(SAME_MINUTE_SERVICE_ID, onBoth, null))); + assertEquals(both, pprofIds(pprof.queryTask(SAME_MINUTE_SERVICE_ID, onBoth, null))); + final Duration between = secondsDuration(created[0] + 1_000L, created[1] - 1_000L); + assertEquals(Collections.emptyList(), async.queryTask(SAME_MINUTE_SERVICE_ID, between, null)); + assertEquals(Collections.emptyList(), pprof.queryTask(SAME_MINUTE_SERVICE_ID, between, null)); + } + + /** + * A task created ten seconds ago with the second bucket the mutation services write, as earlier versions wrote it + * too, runs: creating another task of the service is refused, as the check finds the running task by that bucket. + */ + @Test + public void refusesASecondTaskOfAServiceWhileTheFirstRuns() throws Exception { + final long createTime = System.currentTimeMillis() - 10_000L; + support.writeNoneStream(asyncProfilerModel, new AsyncProfilerTaskRecord.Builder(), + asyncProfilerTask(RUNNING_SERVICE_ID, createTime)); + support.writeNoneStream(pprofModel, new PprofTaskRecord.Builder(), pprofTask(RUNNING_SERVICE_ID, createTime)); + final ModuleManager moduleManager = storageServices(); + assertEquals(AsyncProfilerTaskCreationType.ALREADY_PROFILING_ERROR, + new AsyncProfilerMutationService(moduleManager).createTask( + RUNNING_SERVICE_ID, Collections.singletonList(INSTANCE_ID), 60, + Collections.singletonList(AsyncProfilerEventType.CPU), "").getCode()); + assertEquals(PprofTaskCreationType.ALREADY_PROFILING_ERROR, + new PprofMutationService(moduleManager).createTask( + RUNNING_SERVICE_ID, Collections.singletonList(INSTANCE_ID), 5, PprofEventType.CPU, 10 + ).getCode()); + } + + /** + * Every index of the task and log tables is a daily index, named after the day of its documents' time buckets, + * which is what the TTL reads to delete the indices of past days. A task lies in the index of its time bucket's + * day, whatever the bucket's precision: an async-profiler or pprof task carries a second bucket on a + * minute-downsampled model. + */ + @Test + public void writesTheTasksAndTheirLogsIntoDailyIndices() { + final String yesterday = String.valueOf(TimeBucket.getTimeBucket(MIDNIGHT - 1, DownSampling.Day)); + final String today = String.valueOf(TimeBucket.getTimeBucket(MIDNIGHT, DownSampling.Day)); + for (final Model model : Arrays.asList(traceProfilingModel, asyncProfilerModel, pprofModel, + traceProfilingLogModel, asyncProfilerLogModel, pprofLogModel)) { + final List days = support.getClient() + .retrievalIndexByAliases(IndexController.INSTANCE.getTableName(model)) + .stream() + .map(index -> index.substring(index.lastIndexOf('-') + 1)) + .sorted() + .collect(Collectors.toList()); + assertEquals(Collections.emptyList(), + days.stream().filter(day -> !day.matches("\\d{8}")).collect(Collectors.toList()), + model.getName()); + assertTrue(days.containsAll(Arrays.asList(yesterday, today)), model.getName() + " " + days); + } + } + + private static long minutes(final int minutes) { + return TimeUnit.MINUTES.toMillis(minutes); + } + + private static long createTime(final int minute) { + return MIDNIGHT + minutes(minute); + } + + private static long secondBucket(final int minute, final int seconds) { + return TimeBucket.getRecordTimeBucket(createTime(minute) + TimeUnit.SECONDS.toMillis(seconds)); + } + + /** + * The id the mutation services give a task: its creation time and its service. + */ + private static String taskId(final String serviceId, final int minute) { + return taskId(serviceId, createTime(minute)); + } + + private static String taskId(final String serviceId, final long createTime) { + return createTime + Const.ID_CONNECTOR + serviceId; + } + + /** + * A duration of the second step, as the GraphQL task lists take one. + */ + private static Duration secondsDuration(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.SECOND); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmmss")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmmss")); + return duration; + } + + private static List asyncProfilerIds(final List tasks) { + return tasks.stream().map(AsyncProfilerTask::getId).collect(Collectors.toList()); + } + + private static List pprofIds(final List tasks) { + return tasks.stream().map(PprofTask::getId).collect(Collectors.toList()); + } + + /** + * Hands the profiling services this plugin's task DAOs, as the storage module would. + */ + private ModuleManager storageServices() { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(moduleManager.find(StorageModule.NAME)).thenReturn(provider); + when(provider.provider()).thenReturn(services); + when(services.getService(IAsyncProfilerTaskQueryDAO.class)).thenReturn(asyncProfilerDAO); + when(services.getService(IPprofTaskQueryDAO.class)).thenReturn(pprofDAO); + return moduleManager; + } + + private static List describe(final ProfileTaskRecord... tasks) { + return Arrays.stream(tasks) + .map(task -> String.join( + "|", task.getTaskId(), task.getServiceId(), task.getEndpointName(), + String.valueOf(task.getStartTime()), String.valueOf(task.getCreateTime()), + String.valueOf(task.getDuration()), String.valueOf(task.getMinDurationThreshold()), + String.valueOf(task.getDumpPeriod()), String.valueOf(task.getMaxSamplingCount()))) + .collect(Collectors.toList()); + } + + private static List describe(final List tasks) { + return tasks.stream() + .map(task -> String.join( + "|", task.getId(), task.getServiceId(), task.getEndpointName(), + String.valueOf(task.getStartTime()), String.valueOf(task.getCreateTime()), + String.valueOf(task.getDuration()), String.valueOf(task.getMinDurationThreshold()), + String.valueOf(task.getDumpPeriod()), String.valueOf(task.getMaxSamplingCount()))) + .collect(Collectors.toList()); + } + + private static List describeAsyncProfilerTasks(final String serviceId, final int... minutes) { + return describeAsyncProfilerTasks(Arrays.stream(minutes).mapToObj(minute -> { + final AsyncProfilerTaskRecord record = asyncProfilerTask(serviceId, minute); + return AsyncProfilerTask.builder() + .id(record.getTaskId()) + .serviceId(record.getServiceId()) + .serviceInstanceIds(Collections.singletonList(INSTANCE_ID)) + .createTime(record.getCreateTime()) + .events(Arrays.asList(AsyncProfilerEventType.CPU, AsyncProfilerEventType.LOCK)) + .duration(record.getDuration()) + .execArgs(record.getExecArgs()) + .build(); + }).collect(Collectors.toList())); + } + + private static List describeAsyncProfilerTasks(final List tasks) { + return tasks.stream() + .map(task -> String.join( + "|", task.getId(), task.getServiceId(), String.valueOf(task.getServiceInstanceIds()), + String.valueOf(task.getCreateTime()), String.valueOf(task.getEvents()), + String.valueOf(task.getDuration()), task.getExecArgs())) + .collect(Collectors.toList()); + } + + private static List describePprofTasks(final String serviceId, final int... minutes) { + return describePprofTasks(Arrays.stream(minutes).mapToObj(minute -> { + final PprofTaskRecord record = pprofTask(serviceId, minute); + return PprofTask.builder() + .id(record.getTaskId()) + .serviceId(record.getServiceId()) + .serviceInstanceIds(Collections.singletonList(INSTANCE_ID)) + .createTime(record.getCreateTime()) + .events(PprofEventType.CPU) + .duration(record.getDuration()) + .dumpPeriod(record.getDumpPeriod()) + .build(); + }).collect(Collectors.toList())); + } + + private static List describePprofTasks(final List tasks) { + return tasks.stream() + .map(task -> String.join( + "|", task.getId(), task.getServiceId(), String.valueOf(task.getServiceInstanceIds()), + String.valueOf(task.getCreateTime()), String.valueOf(task.getEvents()), + String.valueOf(task.getDuration()), String.valueOf(task.getDumpPeriod()))) + .collect(Collectors.toList()); + } + + /** + * A task log's id is the id of its task on every storage. + */ + private static List describeAsyncProfilerLogs(final List logs) { + return logs.stream() + .map(log -> String.join("|", log.getId(), log.getInstanceId(), log.getOperationType().name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList()); + } + + private static List describePprofLogs(final List logs) { + return logs.stream() + .map(log -> String.join("|", log.getId(), log.getInstanceId(), log.getOperationType().name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList()); + } + + /** + * The data of a task in no promised order, sorted by instance and upload time. + */ + private static List describeJFR(final List data) { + return data.stream() + .map(row -> String.join("|", row.getInstanceId(), String.valueOf(row.getUploadTime()), + row.getTaskId(), row.getEventType(), String.valueOf(row.getTimeBucket()), + new String(row.getDataBinary(), StandardCharsets.UTF_8))) + .sorted() + .collect(Collectors.toList()); + } + + private static List describeJFR(final List data, final String taskId, + final List instanceIds, final String eventType) { + return describeJFR(data.stream() + .filter(row -> row.getTaskId().equals(taskId) + && instanceIds.contains(row.getInstanceId()) + && row.getEventType().equals(eventType)) + .collect(Collectors.toList())); + } + + private static List describePprof(final List data) { + return data.stream() + .map(row -> String.join("|", row.getInstanceId(), String.valueOf(row.getUploadTime()), + row.getTaskId(), String.valueOf(row.getTimeBucket()), + new String(row.getDataBinary(), StandardCharsets.UTF_8))) + .sorted() + .collect(Collectors.toList()); + } + + /** + * Uploads by upload time, each with the length and the SHA-256 of its data, which is too large to compare as text. + */ + private static List fingerprints(final List uploads, final ToLongFunction uploadTime, + final Function data) { + return uploads.stream() + .sorted(Comparator.comparingLong(uploadTime)) + .map(upload -> uploadTime.applyAsLong(upload) + "|" + data.apply(upload).length + "|" + + Hashing.sha256().hashBytes(data.apply(upload))) + .collect(Collectors.toList()); + } + + private static List describePprof(final List data, final String taskId, + final List instanceIds) { + return describePprof(data.stream() + .filter(row -> row.getTaskId().equals(taskId) + && instanceIds.contains(row.getInstanceId())) + .collect(Collectors.toList())); + } + + /** + * A task as {@code ProfileTaskMutationService} writes it: its time bucket is the minute it starts. + */ + private static ProfileTaskRecord traceProfilingTask(final String serviceId, final String endpointName, + final long createTime, final long startTime) { + final ProfileTaskRecord task = new ProfileTaskRecord(); + task.setTaskId(createTime + Const.ID_CONNECTOR + serviceId); + task.setServiceId(serviceId); + task.setEndpointName(endpointName); + task.setStartTime(startTime); + task.setDuration(TRACE_PROFILING_MINUTES); + task.setMinDurationThreshold(5); + task.setDumpPeriod(10); + task.setCreateTime(createTime); + task.setMaxSamplingCount(9); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(startTime)); + return task; + } + + /** + * A task as {@code AsyncProfilerMutationService} writes it: its time bucket is the second it is created. + */ + private static AsyncProfilerTaskRecord asyncProfilerTask(final String serviceId, final int minute) { + return asyncProfilerTask(serviceId, createTime(minute)); + } + + private static AsyncProfilerTaskRecord asyncProfilerTask(final String serviceId, final long createTime) { + final AsyncProfilerTaskRecord task = new AsyncProfilerTaskRecord(); + task.setTaskId(taskId(serviceId, createTime)); + task.setServiceId(serviceId); + task.setServiceInstanceIdsFromList(Collections.singletonList(INSTANCE_ID)); + task.setDuration((int) TimeUnit.MINUTES.toSeconds(5)); + task.setEventsFromList(Arrays.asList(AsyncProfilerEventType.CPU.name(), AsyncProfilerEventType.LOCK.name())); + task.setExecArgs("interval=10ms"); + task.setCreateTime(createTime); + task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); + return task; + } + + /** + * A task as {@code PprofMutationService} writes it: its time bucket is the second it is created. + */ + private static PprofTaskRecord pprofTask(final String serviceId, final int minute) { + return pprofTask(serviceId, createTime(minute)); + } + + private static PprofTaskRecord pprofTask(final String serviceId, final long createTime) { + final PprofTaskRecord task = new PprofTaskRecord(); + task.setTaskId(taskId(serviceId, createTime)); + task.setServiceId(serviceId); + task.setServiceInstanceIdsFromList(Collections.singletonList(INSTANCE_ID)); + task.setDuration(5); + task.setEvents(PprofEventType.CPU.name()); + task.setDumpPeriod(10); + task.setCreateTime(createTime); + task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); + return task; + } + + private static ProfileTaskLogRecord traceProfilingLog(final ProfileTaskRecord task, final String instanceId, + final ProfileTaskLogOperationType operationType, + final long operationTime) { + final ProfileTaskLogRecord log = new ProfileTaskLogRecord(); + log.setTaskId(task.getTaskId()); + log.setInstanceId(instanceId); + log.setOperationType(operationType.getCode()); + log.setOperationTime(operationTime); + log.setTimestamp(operationTime); + log.setTimeBucket(TimeBucket.getRecordTimeBucket(operationTime)); + return log; + } + + private static AsyncProfilerTaskLogRecord asyncProfilerLog(final String taskId, final long operationTime, + final int operationType) { + final AsyncProfilerTaskLogRecord log = new AsyncProfilerTaskLogRecord(); + log.setTaskId(taskId); + log.setInstanceId(INSTANCE_ID); + log.setOperationType(operationType); + log.setOperationTime(operationTime); + log.setTimestamp(operationTime); + log.setTimeBucket(TimeBucket.getRecordTimeBucket(operationTime)); + return log; + } + + private static PprofTaskLogRecord pprofLog(final String taskId, final long operationTime, + final int operationType) { + final PprofTaskLogRecord log = new PprofTaskLogRecord(); + log.setTaskId(taskId); + log.setInstanceId(INSTANCE_ID); + log.setOperationType(operationType); + log.setOperationTime(operationTime); + log.setTimestamp(operationTime); + log.setTimeBucket(TimeBucket.getRecordTimeBucket(operationTime)); + return log; + } + + private static long uploadTime(final int upload) { + return MIDNIGHT + TimeUnit.SECONDS.toMillis(upload - UPLOADS / 2); + } + + /** + * An upload as {@code JFRProfilingDataDispatcher} writes it, the data being the frame tree of one event. + */ + private static JFRProfilingDataRecord jfr(final String taskId, final String instanceId, final String eventType, + final int upload) { + final JFRProfilingDataRecord data = new JFRProfilingDataRecord(); + data.setTaskId(taskId); + data.setInstanceId(instanceId); + data.setEventType(eventType); + data.setUploadTime(uploadTime(upload)); + data.setDataBinary(("{\"frame\":\"" + eventType + " " + upload + "\"}").getBytes(StandardCharsets.UTF_8)); + data.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime(upload))); + return data; + } + + /** + * An upload as {@code PprofProfilingDataDispatcher} writes it. + */ + private static PprofProfilingDataRecord pprof(final String taskId, final String instanceId, final int upload) { + final PprofProfilingDataRecord data = new PprofProfilingDataRecord(); + data.setTaskId(taskId); + data.setInstanceId(instanceId); + data.setUploadTime(uploadTime(upload)); + data.setDataBinary(("{\"frame\":\"pprof " + upload + "\"}").getBytes(StandardCharsets.UTF_8)); + data.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime(upload))); + return data; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch8IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch8IT.java deleted file mode 100644 index 26120e0b8dd6..000000000000 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch8IT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; - -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Tag("slow") -@Tag("elastic-it") -@Testcontainers -public class SpanAttachedEventQueryElasticsearch8IT extends SpanAttachedEventQueryITBase { - @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("8.18.8"); - - public SpanAttachedEventQueryElasticsearch8IT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.DEFAULT); - } -} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch9IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch9IT.java deleted file mode 100644 index b828e8459aee..000000000000 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch9IT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; - -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Tag("slow") -@Tag("elastic-it") -@Testcontainers -public class SpanAttachedEventQueryElasticsearch9IT extends SpanAttachedEventQueryITBase { - @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("9.3.0"); - - public SpanAttachedEventQueryElasticsearch9IT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.DEFAULT); - } -} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch9ShardedIT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch9ShardedIT.java deleted file mode 100644 index 9ef8d8b65944..000000000000 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryElasticsearch9ShardedIT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; - -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Tag("slow") -@Tag("elastic-it") -@Testcontainers -public class SpanAttachedEventQueryElasticsearch9ShardedIT extends SpanAttachedEventQueryITBase { - @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("9.3.0"); - - public SpanAttachedEventQueryElasticsearch9ShardedIT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.LOGIC_SHARDING_AND_CUSTOM_ROUTING); - } -} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryITBase.java index bbbdbb397e69..c6aadfaeab35 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryITBase.java @@ -23,14 +23,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; import org.apache.skywalking.apm.network.language.agent.v3.SpanAttachedEvent; import org.apache.skywalking.oap.query.zipkin.ZipkinQueryService; import org.apache.skywalking.oap.server.core.CoreModule; @@ -39,7 +36,6 @@ import org.apache.skywalking.oap.server.core.analysis.manual.spanattach.SpanAttachedEventRecord; import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; import org.apache.skywalking.oap.server.core.query.TraceQueryService; -import org.apache.skywalking.oap.server.core.storage.StorageData; import org.apache.skywalking.oap.server.core.storage.StorageModule; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.core.storage.query.ISpanAttachedEventQueryDAO; @@ -58,23 +54,21 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.testcontainers.elasticsearch.ElasticsearchContainer; -import zipkin2.Span; import zipkin2.storage.QueryRequest; import static org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AttachedEventFixture.DURATION; import static org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AttachedEventFixture.MIDNIGHT; import static org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.AttachedEventFixture.stored; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.testcontainers.shaded.org.awaitility.Awaitility.await; /** * Reads span attached events the way the trace and the Zipkin queries do on Elasticsearch and OpenSearch: the V1 * branch reads the spans from the trace DAOs and the events from {@link SpanAttachedEventEsDAO}, and the query - * services attach each event to the span it belongs to. A subclass names the server and the storage mode. + * services attach each event to the span it belongs to. A nested class of a server's IT names the server and the + * storage mode. * *

The events are rover's network profiling, see {@link NativeTraceFixture} and {@link ZipkinTraceFixture}. Every * trace has more events than a page of the events read, events of traces not asked for name the same spans, events @@ -109,7 +103,7 @@ abstract class SpanAttachedEventQueryITBase { @BeforeAll public void writeTraces() throws Exception { support = ElasticSearchQueryITSupport.connect( - server, mode, config -> config.setProfileDataQueryBatchSize(EVENTS_PER_PAGE)); + server, mode, getClass(), config -> config.setProfileDataQueryBatchSize(EVENTS_PER_PAGE)); final Model segments = support.installRecord(SegmentRecord.class); final Model zipkinSpans = support.installRecord(ZipkinSpanRecord.class); final Model skyWalkingEventModel = support.installRecord(SWSpanAttachedEventRecord.class); @@ -138,10 +132,6 @@ public void writeTraces() throws Exception { final ModuleManager moduleManager = moduleManager(); traceQueryService = new TraceQueryService(moduleManager); zipkinQueryService = new ZipkinQueryService(moduleManager); - // A write is searchable after its index's next refresh, every 30 s as OAP configures the indices. - await().atMost(90, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .untilAsserted(this::assertEverythingSearchable); } @AfterAll @@ -252,29 +242,6 @@ public void attachesEachEventToItsSpanOfTheZipkinTracesReadByIds() throws IOExce ZipkinTraceFixture.canonical(zipkinQueryService.getTracesByIds(ids, null))); } - /** - * The events are waited for by their storage ids, which two events can share. - */ - private void assertEverythingSearchable() throws IOException { - for (final SegmentObject segment : NativeTraceFixture.SEGMENTS) { - assertTrue(traceDAO.queryByTraceId(segment.getTraceId(), null).stream() - .anyMatch(record -> record.getSegmentId().equals(segment.getTraceSegmentId())), - segment.getTraceSegmentId()); - } - for (final Span span : ZipkinTraceFixture.SPANS) { - assertTrue(zipkinDAO.getTrace(span.traceId(), null).stream().anyMatch(read -> read.id().equals(span.id())), - span.id()); - } - final List skyWalkingTraces = skyWalkingEvents.stream().map(SWSpanAttachedEventRecord::getRelatedTraceId) - .distinct().collect(Collectors.toList()); - assertTrue(idsOf(eventDAO.querySWSpanAttachedEvents(skyWalkingTraces, null)) - .containsAll(idsOf(skyWalkingEvents)), "SkyWalking events"); - final List zipkinTraces = zipkinEvents.stream().map(SpanAttachedEventRecord::getRelatedTraceId) - .distinct().collect(Collectors.toList()); - assertTrue(idsOf(eventDAO.queryZKSpanAttachedEvents(zipkinTraces, null)) - .containsAll(idsOf(zipkinEvents)), "Zipkin events"); - } - /** * Hands the query services this plugin's DAOs, as the storage module would. */ @@ -315,12 +282,4 @@ private static List> byTraceAndSpan(final List (String) columns.get(SpanAttachedEventRecord.TRACE_SPAN_ID))); return events; } - - private static Set idsOf(final List records) { - final Set ids = new HashSet<>(); - for (final StorageData record : records) { - ids.add(record.id().build()); - } - return ids; - } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch2IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch2IT.java deleted file mode 100644 index 5812ac6a107b..000000000000 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch2IT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; - -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Tag("slow") -@Tag("elastic-it") -@Testcontainers -public class SpanAttachedEventQueryOpenSearch2IT extends SpanAttachedEventQueryITBase { - @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("2.8.0"); - - public SpanAttachedEventQueryOpenSearch2IT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.DEFAULT); - } -} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch3IT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch3IT.java deleted file mode 100644 index 980152a062a2..000000000000 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch3IT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; - -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Tag("slow") -@Tag("elastic-it") -@Testcontainers -public class SpanAttachedEventQueryOpenSearch3IT extends SpanAttachedEventQueryITBase { - @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("3.0.0"); - - public SpanAttachedEventQueryOpenSearch3IT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.DEFAULT); - } -} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutocompleteQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutocompleteQueryITBase.java new file mode 100644 index 000000000000..313c46d1e14b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TagAutocompleteQueryITBase.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +/** + * Reads tag autocompletion the way the tag autocomplete query service does on Elasticsearch and OpenSearch, through + * {@link TagAutoCompleteQueryDAO}, with indices rolled by three days, so an index holds the tags of days outside a + * window. A nested class of a server's IT names the server and the storage mode. + * + *

A tag is stored once for each day it is seen, under the first minute of that day. A window within one day reads + * the tags of that day only, not those of the two other days its index holds, and so does a window of the whole day, + * whose end instant opens the next day. A value seen on each of many days is + * stored once for each of them, so the values of a long window are read page by page until the limit is reached, + * and no further page is read once the first holds as many values as the limit. A limit of 0 reads nothing. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class TagAutocompleteQueryITBase { + /** The middle day of a group of three counted from 2000-01-01, so its index holds the day before and after. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final int DAY_STEP = 3; + private static final long DAY = 24 * 60 * 60_000L; + /** Ten to eleven in the morning of the middle day. */ + private static final Duration WITHIN_ONE_DAY = duration( + MIDNIGHT + 10 * 60 * 60_000L, MIDNIGHT + 11 * 60 * 60_000L, Step.MINUTE); + private static final Duration THE_WHOLE_DAY = duration(MIDNIGHT, MIDNIGHT, Step.DAY); + /** Days on which one route each is seen, ending ten days before the middle day. */ + private static final int ROUTE_DAYS = 30; + private static final long FIRST_ROUTE_DAY = MIDNIGHT - (ROUTE_DAYS + 10) * DAY; + private static final Duration EVERY_ROUTE_DAY = duration( + FIRST_ROUTE_DAY, FIRST_ROUTE_DAY + (ROUTE_DAYS - 1) * DAY, Step.DAY); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private ElasticSearchClient client; + private TagAutoCompleteQueryDAO dao; + + TagAutocompleteQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeTags() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass(), config -> config.setDayStep(DAY_STEP)); + final Model model = support.installMetrics(TagAutocompleteData.class); + final List tags = new ArrayList<>(Arrays.asList( + tag("http.method", "GET", MIDNIGHT), + tag("db.type", "MySQL", MIDNIGHT), + tag("http.method", "POST", MIDNIGHT - DAY), + tag("http.method", "PUT", MIDNIGHT + DAY), + tag("outside.only", "x", MIDNIGHT - DAY))); + for (int day = 0; day < ROUTE_DAYS; day++) { + tags.add(tag("http.route", route(day), FIRST_ROUTE_DAY + day * DAY)); + } + support.writeMetrics(model, new TagAutocompleteData.Builder(), tags); + client = spy(support.getClient()); + dao = new TagAutoCompleteQueryDAO(client); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheTagsOfTheDaysOfTheWindowOnly() throws Exception { + for (final Duration window : Arrays.asList(WITHIN_ONE_DAY, THE_WHOLE_DAY)) { + assertEquals(new TreeSet<>(Arrays.asList("db.type", "http.method")), + new TreeSet<>(dao.queryTagAutocompleteKeys(TagType.TRACE, 10, window))); + assertEquals(Collections.singleton("GET"), + dao.queryTagAutocompleteValues(TagType.TRACE, "http.method", 10, window)); + } + } + + /** + * Every route is a value of its own, so the first page of the read already holds as many values as the limit. + */ + @Test + public void readsNoFurtherPageOnceTheLimitIsReached() throws Exception { + final Set routes = new TreeSet<>(); + for (int day = 0; day < ROUTE_DAYS; day++) { + routes.add(route(day)); + } + for (final Duration duration : Arrays.asList(EVERY_ROUTE_DAY, null)) { + clearInvocations(client); + final Set values = dao.queryTagAutocompleteValues(TagType.TRACE, "http.route", 3, duration); + assertEquals(3, values.size()); + assertTrue(routes.containsAll(values), values.toString()); + verify(client, never()).scroll(any(), any()); + } + assertEquals(routes, new TreeSet<>( + dao.queryTagAutocompleteValues(TagType.TRACE, "http.route", ROUTE_DAYS + 10, EVERY_ROUTE_DAY))); + } + + @Test + public void readsNothingForALimitOfZero() throws Exception { + assertEquals(Collections.emptySet(), dao.queryTagAutocompleteKeys(TagType.TRACE, 0, WITHIN_ONE_DAY)); + assertEquals(Collections.emptySet(), + dao.queryTagAutocompleteValues(TagType.TRACE, "http.method", 0, WITHIN_ONE_DAY)); + } + + private static String route(final int day) { + return String.format("/route-%02d", day); + } + + /** + * A tag as the autocompletion dispatcher stores it, under the first minute of the day it was seen. + */ + private static TagAutocompleteData tag(final String key, final String value, final long seen) { + final TagAutocompleteData data = new TagAutocompleteData(); + data.setTagType(TagType.TRACE.name()); + data.setTagKey(key); + data.setTagValue(value); + data.setTimeBucket(TimeBucket.retainToDay4MinuteBucket(TimeBucket.getMinuteTimeBucket(seen))); + return data; + } + + /** + * A query duration from {@code start} to {@code end}, as the query protocol writes it for {@code step}. + */ + private static Duration duration(final long start, final long end, final Step step) { + final String pattern = step == Step.DAY ? "yyyy-MM-dd" : "yyyy-MM-dd HHmm"; + final Duration duration = new Duration(); + duration.setStep(step); + duration.setStart(new DateTime(start).toString(pattern)); + duration.setEnd(new DateTime(end).toString(pattern)); + return duration; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryITBase.java new file mode 100644 index 000000000000..b4ccde0b1049 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TopologyQueryITBase.java @@ -0,0 +1,779 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessDetectType; +import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.endpoint.EndpointRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.process.ProcessRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.process.ProcessRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.IntList; +import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; +import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; +import org.apache.skywalking.oap.server.core.query.MetadataQueryService; +import org.apache.skywalking.oap.server.core.query.TopologyQueryService; +import org.apache.skywalking.oap.server.core.query.enumeration.ProfilingSupportStatus; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.Call; +import org.apache.skywalking.oap.server.core.query.type.ProcessNode; +import org.apache.skywalking.oap.server.core.query.type.ProcessTopology; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.StorageModels; +import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.base.StorageEsDAO; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Reads the relations a topology is built from, the way the topology query service does, through + * {@link TopologyQueryEsDAO}: the service relations of the whole range or of some services, the instance relations + * of a service pair, the relations of an endpoint and the process relations of an instance, each from the model of + * the side that detected it, and the service and process topologies the query service builds from them. A nested + * class of a server's IT names the server and the storage mode. + * + *

The range is the minute before midnight and the minute of midnight, so every read spans two daily indices, and + * relations lie on both its bounds and in the minutes just outside them. The server and the client side each detect + * relations the other does not, and in the default mode both sides share the merged metrics index, so a read that + * lost its model filter returns the other side's relations. The relation models have no routing column: the sharded + * mode routes them by id over both shards, and the reads search every shard. A service or process relation may name + * another component in each minute, a service relation several in one minute, and each component is a call of its + * own. Every filter has a near miss: a relation of a service, an instance, an endpoint or a process not asked for, + * an instance relation from one service of the pair to a third one, an endpoint of the same name on another + * service, and relations rolled up per hour in the same index, which a minute read must not return and an hour read + * must. A scheduler service and process relation each name another component in each of twelve minutes, more than + * a terms aggregation keeps by default. + * + *

The topology builders key the calls by id, so a read is compared as the set of calls it returns. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class TopologyQueryITBase { + /** A fixed local midnight, so the daily indices the relations are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final int TOMCAT = 1; + private static final int MYSQL = 5; + private static final int GRPC = 23; + private static final int HTTP = 49; + /** Readable names of the ids the fixture builds, to print the calls read. */ + private static final Map NAMES = new HashMap<>(); + + private static final String USER = service("User", false); + private static final String GATEWAY = service("gateway", true); + private static final String CHECKOUT = service("checkout", true); + private static final String PAYMENT = service("payment", true); + private static final String LEDGER = service("ledger", true); + private static final String AUDIT = service("audit", true); + private static final String ORDERS_DB = service("orders-db:3306", false); + + private static final String GATEWAY_1 = instance(GATEWAY, "gateway-1"); + private static final String GATEWAY_2 = instance(GATEWAY, "gateway-2"); + private static final String CHECKOUT_1 = instance(CHECKOUT, "checkout-1"); + private static final String PAYMENT_1 = instance(PAYMENT, "payment-1"); + + private static final String CHECKOUT_1_ENVOY = process(CHECKOUT_1, "envoy"); + private static final String CHECKOUT_1_JAVA = process(CHECKOUT_1, "java"); + private static final String CHECKOUT_1_PYTHON = process(CHECKOUT_1, "python"); + private static final String PAYMENT_1_ENVOY = process(PAYMENT_1, "envoy"); + private static final String PAYMENT_1_JAVA = process(PAYMENT_1, "java"); + + private static final String SCHEDULER = service("scheduler", true); + private static final String SCHEDULER_1 = instance(SCHEDULER, "scheduler-1"); + private static final String SCHEDULER_1_CRON = process(SCHEDULER_1, "cron"); + private static final String SCHEDULER_1_WORKER = process(SCHEDULER_1, "worker"); + /** The first of the minutes in which a scheduler relation names another component each. */ + private static final int FIRST_COMPONENT_MINUTE = 10; + /** One component per minute, more than a terms aggregation keeps by default. */ + private static final int COMPONENTS = 12; + /** The names the component library gives the component ids of the fixture. */ + private static final Map COMPONENT_NAMES = componentNames(); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private TopologyQueryEsDAO dao; + private TopologyQueryService topologyQueryService; + + TopologyQueryITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeRelations() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + // As at boot, every model is installed before anything is written: a daily index is created from the index + // template at its first write, and maps only the columns the template had then. + final Model services = support.installMetrics(ServiceRelationServerSideMetrics.class); + final Model hourlyServices = support.installMetrics(ServiceRelationServerSideMetrics.class, DownSampling.Hour); + final Model servicesAtClient = support.installMetrics(ServiceRelationClientSideMetrics.class); + final Model instances = support.installMetrics(ServiceInstanceRelationServerSideMetrics.class); + final Model instancesAtClient = support.installMetrics(ServiceInstanceRelationClientSideMetrics.class); + final Model endpoints = support.installMetrics(EndpointRelationServerSideMetrics.class); + final Model processes = support.installMetrics(ProcessRelationServerSideMetrics.class); + final Model processesAtClient = support.installMetrics(ProcessRelationClientSideMetrics.class); + final Model processTrafficModel = support.installMetrics(ProcessTraffic.class); + + support.writeMetrics( + services, + new ServiceRelationServerSideMetrics.Builder(), + Arrays.asList( + serverSide(USER, GATEWAY, minute(-1), HTTP), + serverSide(USER, GATEWAY, minute(0), HTTP), + serverSide(GATEWAY, CHECKOUT, minute(-1), HTTP), + serverSide(CHECKOUT, PAYMENT, minute(-1), HTTP), + serverSide(CHECKOUT, PAYMENT, minute(0), GRPC), + serverSide(PAYMENT, CHECKOUT, minute(0), GRPC, HTTP), + serverSide(LEDGER, AUDIT, minute(-2), HTTP), + serverSide(LEDGER, AUDIT, minute(1), HTTP))); + support.writeMetrics( + hourlyServices, + new ServiceRelationServerSideMetrics.Builder(), + Arrays.asList( + serverSide(AUDIT, GATEWAY, hour(-1), HTTP), + serverSide(GATEWAY, AUDIT, hour(0), TOMCAT), + serverSide(CHECKOUT, PAYMENT, hour(-1), GRPC), + serverSide(LEDGER, GATEWAY, hour(-2), HTTP), + serverSide(LEDGER, GATEWAY, hour(1), HTTP))); + support.writeMetrics( + servicesAtClient, + new ServiceRelationClientSideMetrics.Builder(), + Arrays.asList( + clientSide(GATEWAY, CHECKOUT, minute(0), HTTP), + clientSide(CHECKOUT, PAYMENT, minute(-1), HTTP), + clientSide(CHECKOUT, PAYMENT, minute(0), HTTP, GRPC), + clientSide(PAYMENT, CHECKOUT, minute(0), GRPC), + clientSide(CHECKOUT, ORDERS_DB, minute(-1), MYSQL), + clientSide(CHECKOUT, ORDERS_DB, minute(0), MYSQL), + clientSide(LEDGER, AUDIT, minute(-2), HTTP), + clientSide(LEDGER, AUDIT, minute(1), HTTP))); + + support.writeMetrics( + instances, + new ServiceInstanceRelationServerSideMetrics.Builder(), + Arrays.asList( + instanceServerSide(GATEWAY_1, CHECKOUT_1, -1), + instanceServerSide(GATEWAY_2, CHECKOUT_1, 0), + instanceServerSide(CHECKOUT_1, GATEWAY_1, 0), + instanceServerSide(GATEWAY_1, PAYMENT_1, -1), + instanceServerSide(PAYMENT_1, CHECKOUT_1, 0), + instanceServerSide(CHECKOUT_1, GATEWAY_2, -2), + instanceServerSide(CHECKOUT_1, GATEWAY_2, 1))); + support.writeMetrics( + instancesAtClient, + new ServiceInstanceRelationClientSideMetrics.Builder(), + Arrays.asList( + instanceClientSide(GATEWAY_1, CHECKOUT_1, -1), + instanceClientSide(GATEWAY_1, CHECKOUT_1, 0), + instanceClientSide(CHECKOUT_1, GATEWAY_2, 0), + instanceClientSide(PAYMENT_1, CHECKOUT_1, -1), + instanceClientSide(GATEWAY_2, CHECKOUT_1, -2), + instanceClientSide(GATEWAY_2, CHECKOUT_1, 1))); + + support.writeMetrics( + endpoints, + new EndpointRelationServerSideMetrics.Builder(), + Arrays.asList( + endpointRelation(GATEWAY, "/checkout", CHECKOUT, "/pay", -1), + endpointRelation(CHECKOUT, "/pay", PAYMENT, "/charge", 0), + endpointRelation(GATEWAY, "/cart", CHECKOUT, "/stock", -1), + endpointRelation(GATEWAY, "/cart", CHECKOUT, "/stock", 0), + endpointRelation(GATEWAY, "/pay", PAYMENT, "/charge", 0), + endpointRelation(LEDGER, "/refund", CHECKOUT, "/pay", -2), + endpointRelation(LEDGER, "/refund", CHECKOUT, "/pay", 1))); + + support.writeMetrics( + processes, + new ProcessRelationServerSideMetrics.Builder(), + Arrays.asList( + processServerSide(CHECKOUT_1, CHECKOUT_1_ENVOY, CHECKOUT_1_JAVA, -1, HTTP), + processServerSide(CHECKOUT_1, CHECKOUT_1_ENVOY, CHECKOUT_1_JAVA, 0, HTTP), + processServerSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_ENVOY, -1, GRPC), + processServerSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_ENVOY, 0, HTTP), + processServerSide(PAYMENT_1, PAYMENT_1_ENVOY, PAYMENT_1_JAVA, 0, HTTP), + processServerSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_PYTHON, -2, HTTP), + processServerSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_PYTHON, 1, HTTP))); + support.writeMetrics( + processesAtClient, + new ProcessRelationClientSideMetrics.Builder(), + Arrays.asList( + processClientSide(CHECKOUT_1, CHECKOUT_1_ENVOY, CHECKOUT_1_JAVA, -1, HTTP), + processClientSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_PYTHON, 0, GRPC), + processClientSide(PAYMENT_1, PAYMENT_1_ENVOY, PAYMENT_1_JAVA, -1, HTTP), + processClientSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_ENVOY, -2, HTTP), + processClientSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_ENVOY, 1, HTTP))); + + final List schedulerCalls = new ArrayList<>(); + final List schedulerCallsAtClient = new ArrayList<>(); + final List cronCalls = new ArrayList<>(); + final List cronCallsAtClient = new ArrayList<>(); + for (int component = 1; component <= COMPONENTS; component++) { + final int minute = FIRST_COMPONENT_MINUTE + component - 1; + schedulerCalls.add(serverSide(SCHEDULER, LEDGER, minute(minute), component)); + schedulerCallsAtClient.add(clientSide(SCHEDULER, LEDGER, minute(minute), component)); + cronCalls.add(processServerSide(SCHEDULER_1, SCHEDULER_1_CRON, SCHEDULER_1_WORKER, minute, component)); + cronCallsAtClient.add( + processClientSide(SCHEDULER_1, SCHEDULER_1_CRON, SCHEDULER_1_WORKER, minute, component)); + } + support.writeMetrics(services, new ServiceRelationServerSideMetrics.Builder(), schedulerCalls); + support.writeMetrics(servicesAtClient, new ServiceRelationClientSideMetrics.Builder(), schedulerCallsAtClient); + support.writeMetrics(processes, new ProcessRelationServerSideMetrics.Builder(), cronCalls); + support.writeMetrics(processesAtClient, new ProcessRelationClientSideMetrics.Builder(), cronCallsAtClient); + + support.writeMetrics( + processTrafficModel, + new ProcessTraffic.Builder(), + Arrays.asList( + processTraffic(CHECKOUT_1, "envoy", -1), + processTraffic(CHECKOUT_1, "java", -1), + processTraffic(CHECKOUT_1, "python", 0), + processTraffic(PAYMENT_1, "envoy", 0), + processTraffic(PAYMENT_1, "java", 0), + processTraffic(SCHEDULER_1, "cron", -1), + processTraffic(SCHEDULER_1, "worker", 0))); + + dao = new TopologyQueryEsDAO(support.getClient()); + topologyQueryService = topologyQueryService(new StorageEsDAO(support.getClient()), processTrafficModel); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsEveryServiceRelationDetectedAtTheServerSide() throws Exception { + assertEquals( + set( + "checkout -> payment #23 SERVER", + "checkout -> payment #49 SERVER", + "gateway -> checkout #49 SERVER", + "payment -> checkout #23 SERVER", + "payment -> checkout #49 SERVER", + "User -> gateway #49 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(minutes()))); + } + + @Test + public void readsEveryServiceRelationDetectedAtTheClientSide() throws Exception { + assertEquals( + set( + "checkout -> orders-db:3306 #5 CLIENT", + "checkout -> payment #23 CLIENT", + "checkout -> payment #49 CLIENT", + "gateway -> checkout #49 CLIENT", + "payment -> checkout #23 CLIENT"), + calls(dao.loadServiceRelationDetectedAtClientSide(minutes()))); + } + + @Test + public void readsTheServiceRelationsOfOneServiceEitherWay() throws Exception { + final List services = Collections.singletonList(CHECKOUT); + assertEquals( + set( + "checkout -> payment #23 SERVER", + "checkout -> payment #49 SERVER", + "gateway -> checkout #49 SERVER", + "payment -> checkout #23 SERVER", + "payment -> checkout #49 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(minutes(), services))); + assertEquals( + set( + "checkout -> orders-db:3306 #5 CLIENT", + "checkout -> payment #23 CLIENT", + "checkout -> payment #49 CLIENT", + "gateway -> checkout #49 CLIENT", + "payment -> checkout #23 CLIENT"), + calls(dao.loadServiceRelationDetectedAtClientSide(minutes(), services))); + } + + @Test + public void readsTheServiceRelationsOfSomeServices() throws Exception { + assertEquals( + set( + "gateway -> checkout #49 SERVER", + "User -> gateway #49 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(minutes(), Arrays.asList(GATEWAY, ORDERS_DB)))); + assertEquals( + set( + "checkout -> orders-db:3306 #5 CLIENT", + "gateway -> checkout #49 CLIENT"), + calls(dao.loadServiceRelationDetectedAtClientSide(minutes(), Arrays.asList(GATEWAY, ORDERS_DB)))); + } + + @Test + public void readsNoServiceRelationOfServicesThatOnlyCalledOutsideTheRange() throws Exception { + final List services = Arrays.asList(LEDGER, AUDIT); + assertEquals(set(), calls(dao.loadServiceRelationsDetectedAtServerSide(minutes(), services))); + assertEquals(set(), calls(dao.loadServiceRelationDetectedAtClientSide(minutes(), services))); + } + + @Test + public void readsTheHourlyServiceRelationsAtAnHourStep() throws Exception { + assertEquals( + set( + "audit -> gateway #49 SERVER", + "checkout -> payment #23 SERVER", + "gateway -> audit #1 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(hours()))); + assertEquals( + set( + "audit -> gateway #49 SERVER", + "gateway -> audit #1 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(hours(), Collections.singletonList(GATEWAY)))); + } + + @Test + public void readsTheInstanceRelationsOfAServicePairBothWays() throws Exception { + final Set serverSide = set( + "checkout-1 -> gateway-1 #null SERVER", + "gateway-1 -> checkout-1 #null SERVER", + "gateway-2 -> checkout-1 #null SERVER"); + assertEquals(serverSide, calls(dao.loadInstanceRelationDetectedAtServerSide(GATEWAY, CHECKOUT, minutes()))); + assertEquals(serverSide, calls(dao.loadInstanceRelationDetectedAtServerSide(CHECKOUT, GATEWAY, minutes()))); + + final Set clientSide = set( + "checkout-1 -> gateway-2 #null CLIENT", + "gateway-1 -> checkout-1 #null CLIENT"); + assertEquals(clientSide, calls(dao.loadInstanceRelationDetectedAtClientSide(GATEWAY, CHECKOUT, minutes()))); + assertEquals(clientSide, calls(dao.loadInstanceRelationDetectedAtClientSide(CHECKOUT, GATEWAY, minutes()))); + } + + @Test + public void readsTheRelationsOfAnEndpointEitherWay() throws Exception { + assertEquals( + set( + "checkout:/pay -> payment:/charge #0 SERVER", + "gateway:/checkout -> checkout:/pay #0 SERVER"), + calls(dao.loadEndpointRelation(minutes(), endpoint(CHECKOUT, "/pay")))); + assertEquals( + set("gateway:/cart -> checkout:/stock #0 SERVER"), + calls(dao.loadEndpointRelation(minutes(), endpoint(GATEWAY, "/cart")))); + assertEquals(set(), calls(dao.loadEndpointRelation(minutes(), endpoint(LEDGER, "/refund")))); + } + + @Test + public void readsTheProcessRelationsOfAnInstance() throws Exception { + assertEquals( + set( + "checkout-1/envoy -> checkout-1/java #49 SERVER", + "checkout-1/java -> checkout-1/envoy #23 SERVER", + "checkout-1/java -> checkout-1/envoy #49 SERVER"), + calls(dao.loadProcessRelationDetectedAtServerSide(CHECKOUT_1, minutes()))); + assertEquals( + set( + "checkout-1/envoy -> checkout-1/java #49 CLIENT", + "checkout-1/java -> checkout-1/python #23 CLIENT"), + calls(dao.loadProcessRelationDetectedAtClientSide(CHECKOUT_1, minutes()))); + assertEquals( + set("payment-1/envoy -> payment-1/java #49 SERVER"), + calls(dao.loadProcessRelationDetectedAtServerSide(PAYMENT_1, minutes()))); + } + + @Test + public void readsEveryComponentOfARelationThatChangesItEveryMinute() throws Exception { + final Duration range = componentMinutes(); + final Set services = new TreeSet<>(); + final Set servicesAtClient = new TreeSet<>(); + final Set processes = new TreeSet<>(); + final Set processesAtClient = new TreeSet<>(); + for (int component = 1; component <= COMPONENTS; component++) { + services.add("scheduler -> ledger #" + component + " SERVER"); + servicesAtClient.add("scheduler -> ledger #" + component + " CLIENT"); + processes.add("scheduler-1/cron -> scheduler-1/worker #" + component + " SERVER"); + processesAtClient.add("scheduler-1/cron -> scheduler-1/worker #" + component + " CLIENT"); + } + assertEquals(services, calls(dao.loadServiceRelationsDetectedAtServerSide(range))); + assertEquals(services, calls(dao.loadServiceRelationsDetectedAtServerSide( + range, Collections.singletonList(SCHEDULER)))); + assertEquals(servicesAtClient, calls(dao.loadServiceRelationDetectedAtClientSide(range))); + assertEquals(processes, calls(dao.loadProcessRelationDetectedAtServerSide(SCHEDULER_1, range))); + assertEquals(processesAtClient, calls(dao.loadProcessRelationDetectedAtClientSide(SCHEDULER_1, range))); + } + + /** + * A built call keeps every component its relation named in the range: the client side's as its source + * components, the server side's as its target components. No network address alias and no service is known, so + * no call is folded onto another service. + */ + @Test + public void buildsTheServiceTopologyWithEveryComponentOfBothSides() throws Exception { + assertEquals( + sorted( + "User -> gateway [SERVER] [] [http]", + "checkout -> orders-db:3306 [CLIENT] [Mysql] []", + "checkout -> payment [CLIENT, SERVER] [GRPC, http] [GRPC, http]", + "gateway -> checkout [CLIENT, SERVER] [http] [http]", + "payment -> checkout [CLIENT, SERVER] [GRPC] [GRPC, http]"), + builtCalls(topologyQueryService.getGlobalTopology(minutes(), null).getCalls())); + assertEquals( + sorted("scheduler -> ledger [CLIENT, SERVER] " + library() + " " + library()), + builtCalls(topologyQueryService.getGlobalTopology(componentMinutes(), null).getCalls())); + } + + @Test + public void buildsTheProcessTopologyWithEveryComponentOfBothSides() throws Exception { + final ProcessTopology checkout = topologyQueryService.getProcessTopology(CHECKOUT_1, minutes()); + assertEquals( + sorted( + "checkout-1/envoy -> checkout-1/java [CLIENT, SERVER] [http] [http]", + "checkout-1/java -> checkout-1/envoy [SERVER] [] [GRPC, http]", + "checkout-1/java -> checkout-1/python [CLIENT] [GRPC] []"), + builtCalls(checkout.getCalls())); + assertEquals( + sorted("checkout-1/envoy of checkout", "checkout-1/java of checkout", "checkout-1/python of checkout"), + processNodes(checkout.getNodes())); + + final ProcessTopology scheduler = topologyQueryService.getProcessTopology(SCHEDULER_1, componentMinutes()); + assertEquals( + sorted("scheduler-1/cron -> scheduler-1/worker [CLIENT, SERVER] " + library() + " " + library()), + builtCalls(scheduler.getCalls())); + assertEquals( + sorted("scheduler-1/cron of scheduler", "scheduler-1/worker of scheduler"), + processNodes(scheduler.getNodes())); + } + + /** + * The calls read, as {@code source -> target #component detect point}. A call whose id is not the relation of + * its source and target is printed with its id. + */ + private static Set calls(final List calls) { + return calls.stream().map(call -> { + final String relation = name(call.getSource()) + " -> " + name(call.getTarget()); + final String id = relation.equals(name(call.getId())) ? "" : " id " + name(call.getId()); + return relation + " #" + call.getComponentId() + " " + call.getDetectPoint() + id; + }).collect(Collectors.toCollection(TreeSet::new)); + } + + /** + * The calls of a built topology, sorted, as {@code source -> target [detect points] [source components] [target + * components]}. A call whose id is not the relation of its source and target is printed with its id. + */ + private static List builtCalls(final List calls) { + return calls.stream().map(call -> { + final String relation = name(call.getSource()) + " -> " + name(call.getTarget()); + final String id = relation.equals(name(call.getId())) ? "" : " id " + name(call.getId()); + return relation + " " + sortedValues(call.getDetectPoints()) + + " " + sortedValues(call.getSourceComponents()) + + " " + sortedValues(call.getTargetComponents()) + id; + }).sorted().collect(Collectors.toList()); + } + + /** The nodes of a process topology, sorted, as {@code instance/process of service}. */ + private static List processNodes(final List nodes) { + return nodes.stream() + .map(node -> node.getServiceInstanceName() + "/" + node.getName() + " of " + node.getServiceName() + + (name(node.getId()).equals(node.getServiceInstanceName() + "/" + node.getName()) + ? "" : " id " + name(node.getId())) + + (node.isReal() ? "" : " virtual")) + .sorted() + .collect(Collectors.toList()); + } + + private static List sortedValues(final List values) { + return values.stream().map(String::valueOf).sorted().collect(Collectors.toList()); + } + + /** The names of the components the scheduler relations name, one per minute. */ + private static List library() { + return sortedValues( + IntStream.rangeClosed(1, COMPONENTS).mapToObj(COMPONENT_NAMES::get).collect(Collectors.toList())); + } + + private static List sorted(final String... values) { + return Arrays.stream(values).sorted().collect(Collectors.toList()); + } + + private static Map componentNames() { + final Map names = new HashMap<>(); + final String[] library = { + "Tomcat", "HttpClient", "Dubbo", "H2", "Mysql", "ORACLE", "Redis", "Motan", "MongoDB", "Resin", "Feign", + "OKHttp" + }; + for (int i = 0; i < library.length; i++) { + names.put(i + 1, library[i]); + } + names.put(GRPC, "GRPC"); + names.put(HTTP, "http"); + return names; + } + + /** + * The topology query service over the DAO under test, with OAP's storage DAO to read the process traffic. + */ + private TopologyQueryService topologyQueryService(final StorageDAO storageDAO, final Model processTraffic) { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleServiceHolder storage = services(moduleManager, StorageModule.NAME); + when(storage.getService(ITopologyQueryDAO.class)).thenReturn(dao); + when(storage.getService(StorageDAO.class)).thenReturn(storageDAO); + final IComponentLibraryCatalogService components = mock(IComponentLibraryCatalogService.class); + when(components.getComponentName(anyInt())).thenAnswer( + invocation -> COMPONENT_NAMES.get(invocation.getArgument(0))); + final ModuleServiceHolder core = services(moduleManager, CoreModule.NAME); + when(core.getService(IComponentLibraryCatalogService.class)).thenReturn(components); + when(core.getService(NetworkAddressAliasCache.class)).thenReturn( + new NetworkAddressAliasCache(new CoreModuleConfig())); + when(core.getService(MetadataQueryService.class)).thenReturn(mock(MetadataQueryService.class)); + final StorageModels models = mock(StorageModels.class); + when(models.allModels()).thenReturn(Collections.singletonList(processTraffic)); + return new TopologyQueryService(moduleManager, models); + } + + private static ModuleServiceHolder services(final ModuleManager moduleManager, final String moduleName) { + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(moduleManager.find(moduleName)).thenReturn(provider); + when(provider.provider()).thenReturn(services); + return services; + } + + private static Set set(final String... calls) { + return new TreeSet<>(Arrays.asList(calls)); + } + + private static String name(final String id) { + return NAMES.getOrDefault(id, id); + } + + private static Duration minutes() { + return duration(MIDNIGHT - 60_000L, MIDNIGHT, Step.MINUTE, "yyyy-MM-dd HHmm"); + } + + private static Duration minutes(final long first, final long last) { + return duration(MIDNIGHT + first * 60_000L, MIDNIGHT + last * 60_000L, Step.MINUTE, "yyyy-MM-dd HHmm"); + } + + private static Duration componentMinutes() { + return minutes(FIRST_COMPONENT_MINUTE, FIRST_COMPONENT_MINUTE + COMPONENTS - 1); + } + + private static Duration hours() { + return duration(MIDNIGHT - 3_600_000L, MIDNIGHT, Step.HOUR, "yyyy-MM-dd HH"); + } + + private static Duration duration(final long start, final long end, final Step step, final String pattern) { + final Duration duration = new Duration(); + duration.setStep(step); + duration.setStart(new DateTime(start).toString(pattern)); + duration.setEnd(new DateTime(end).toString(pattern)); + return duration; + } + + private static long minute(final long minute) { + return TimeBucket.getMinuteTimeBucket(MIDNIGHT + minute * 60_000L); + } + + private static long hour(final long hour) { + return TimeBucket.getTimeBucket(MIDNIGHT + hour * 3_600_000L, DownSampling.Hour); + } + + private static String service(final String name, final boolean normal) { + final String id = IDManager.ServiceID.buildId(name, normal); + NAMES.put(id, name); + return id; + } + + private static String instance(final String serviceId, final String name) { + final String id = IDManager.ServiceInstanceID.buildId(serviceId, name); + NAMES.put(id, name); + return id; + } + + private static String endpoint(final String serviceId, final String name) { + final String id = IDManager.EndpointID.buildId(serviceId, name); + NAMES.put(id, NAMES.get(serviceId) + ":" + name); + return id; + } + + private static String process(final String instanceId, final String name) { + final String id = IDManager.ProcessID.buildId(instanceId, name); + NAMES.put(id, NAMES.get(instanceId) + "/" + name); + return id; + } + + private static String relation(final String relationId, final String source, final String dest) { + NAMES.put(relationId, name(source) + " -> " + name(dest)); + return relationId; + } + + private static String serviceRelation(final String source, final String dest) { + return relation(IDManager.ServiceID.buildRelationId( + new IDManager.ServiceID.ServiceRelationDefine(source, dest)), source, dest); + } + + private static String instanceRelation(final String source, final String dest) { + return relation(IDManager.ServiceInstanceID.buildRelationId( + new IDManager.ServiceInstanceID.ServiceInstanceRelationDefine(source, dest)), source, dest); + } + + private static String processRelation(final String source, final String dest) { + return relation(IDManager.ProcessID.buildRelationId( + new IDManager.ProcessID.ProcessRelationDefine(source, dest)), source, dest); + } + + private static ProcessTraffic processTraffic(final String instanceId, final String name, + final long firstSeenMinute) { + final ProcessTraffic process = new ProcessTraffic(); + process.setServiceId(IDManager.ServiceInstanceID.analysisId(instanceId).getServiceId()); + process.setInstanceId(instanceId); + process.setName(name); + process.setAgentId("rover-1"); + process.setDetectType(ProcessDetectType.VM.value()); + process.setProfilingSupportStatus(ProfilingSupportStatus.SUPPORT_EBPF_PROFILING.value()); + process.setLabelsJson(Const.EMPTY_STRING); + process.setTimeBucket(minute(firstSeenMinute)); + process.setLastPingTimestamp(minute(firstSeenMinute)); + return process; + } + + private static IntList components(final int... ids) { + final IntList components = new IntList(ids.length); + for (final int id : ids) { + components.add(id); + } + return components; + } + + private static ServiceRelationServerSideMetrics serverSide(final String source, final String dest, + final long timeBucket, final int... components) { + final ServiceRelationServerSideMetrics metrics = new ServiceRelationServerSideMetrics(); + metrics.setSourceServiceId(source); + metrics.setDestServiceId(dest); + metrics.setComponentIds(components(components)); + metrics.setEntityId(serviceRelation(source, dest)); + metrics.setTimeBucket(timeBucket); + return metrics; + } + + private static ServiceRelationClientSideMetrics clientSide(final String source, final String dest, + final long timeBucket, final int... components) { + final ServiceRelationClientSideMetrics metrics = new ServiceRelationClientSideMetrics(); + metrics.setSourceServiceId(source); + metrics.setDestServiceId(dest); + metrics.setComponentIds(components(components)); + metrics.setEntityId(serviceRelation(source, dest)); + metrics.setTimeBucket(timeBucket); + return metrics; + } + + private static ServiceInstanceRelationServerSideMetrics instanceServerSide(final String source, final String dest, + final long minute) { + final ServiceInstanceRelationServerSideMetrics metrics = new ServiceInstanceRelationServerSideMetrics(); + metrics.setSourceServiceId(IDManager.ServiceInstanceID.analysisId(source).getServiceId()); + metrics.setSourceServiceInstanceId(source); + metrics.setDestServiceId(IDManager.ServiceInstanceID.analysisId(dest).getServiceId()); + metrics.setDestServiceInstanceId(dest); + metrics.setEntityId(instanceRelation(source, dest)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } + + private static ServiceInstanceRelationClientSideMetrics instanceClientSide(final String source, final String dest, + final long minute) { + final ServiceInstanceRelationClientSideMetrics metrics = new ServiceInstanceRelationClientSideMetrics(); + metrics.setSourceServiceId(IDManager.ServiceInstanceID.analysisId(source).getServiceId()); + metrics.setSourceServiceInstanceId(source); + metrics.setDestServiceId(IDManager.ServiceInstanceID.analysisId(dest).getServiceId()); + metrics.setDestServiceInstanceId(dest); + metrics.setEntityId(instanceRelation(source, dest)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } + + private static EndpointRelationServerSideMetrics endpointRelation(final String sourceService, + final String source, + final String destService, + final String dest, + final long minute) { + final EndpointRelationServerSideMetrics metrics = new EndpointRelationServerSideMetrics(); + final String sourceId = endpoint(sourceService, source); + final String destId = endpoint(destService, dest); + metrics.setSourceEndpoint(sourceId); + metrics.setDestEndpoint(destId); + metrics.setComponentId(HTTP); + metrics.setEntityId(relation( + IDManager.EndpointID.buildRelationId( + new IDManager.EndpointID.EndpointRelationDefine(sourceService, source, destService, dest)), + sourceId, destId)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } + + private static ProcessRelationServerSideMetrics processServerSide(final String instance, final String source, + final String dest, final long minute, + final int component) { + final ProcessRelationServerSideMetrics metrics = new ProcessRelationServerSideMetrics(); + metrics.setServiceInstanceId(instance); + metrics.setSourceProcessId(source); + metrics.setDestProcessId(dest); + metrics.setComponentId(component); + metrics.setEntityId(processRelation(source, dest)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } + + private static ProcessRelationClientSideMetrics processClientSide(final String instance, final String source, + final String dest, final long minute, + final int component) { + final ProcessRelationClientSideMetrics metrics = new ProcessRelationClientSideMetrics(); + metrics.setServiceInstanceId(instance); + metrics.setSourceProcessId(source); + metrics.setDestProcessId(dest); + metrics.setComponentId(component); + metrics.setEntityId(processRelation(source, dest)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAOITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAOITBase.java new file mode 100644 index 000000000000..86e914950a1e --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/TraceQueryEsDAOITBase.java @@ -0,0 +1,526 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanType; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.BasicTrace; +import org.apache.skywalking.oap.server.core.query.type.QueryOrder; +import org.apache.skywalking.oap.server.core.query.type.TraceState; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads native segments the way the trace list and trace profiling do on Elasticsearch and OpenSearch, through + * {@link TraceQueryEsDAO}: the trace search, the segments of one trace by its id, and the two reads of a trace + * profiling task, its profiled segments by id and then the segments of their traces on the profiled instances. A + * nested class of a server's IT names the server and the storage mode. + * + *

The search range is a minute either side of midnight, so it reads two daily indices. Each search condition is + * tried alone, in both orders, and with all the others. For every condition a near-miss segment matches all the + * others but that one, and the two matching segments lie on the bounds of the range and of the latency. The range is + * read from its first millisecond through its end instant, {@link Duration#getEndTimestamp()}, the first millisecond + * after its last second, so the segments just outside it start a millisecond before and a millisecond after. A segment + * of its own starts on the end instant of a range ending in a day's last second: the next midnight, in the next day's + * index. The search lists segments, not traces: the trace that matches every + * condition has a matching segment either side of midnight, and a third segment, of another service, after the range. + * The reads by trace id and by segment id are not bounded by the range on this storage. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class TraceQueryEsDAOITBase { + /** A fixed local midnight, so the daily indices the segments are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + private static final long START = MIDNIGHT - 60_000L; + /** The range's end instant, the first millisecond after its last second. */ + private static final long END = MIDNIGHT + 60_000L; + private static final Duration AROUND_MIDNIGHT = duration(START, END - 1_000L); + /** The midnight a day before {@link #MIDNIGHT}. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + + private static final String CHECKOUT = "checkout"; + private static final String CANARY = "checkout-canary"; + private static final String INVENTORY = "inventory"; + private static final String CHECKOUT_1 = "checkout-1"; + private static final String OTHER_INSTANCE = "checkout-2"; + private static final String POST_ORDERS = "POST:/orders"; + private static final String OTHER_ENDPOINT = "GET:/orders"; + private static final int MIN_LATENCY = 100; + private static final int MAX_LATENCY = 900; + private static final Tag METHOD = new Tag("http.method", "POST"); + private static final Tag STATUS = new Tag("http.status_code", "500"); + + /** The trace matching every condition. */ + private static final String TRACE = "4a2b6c8d0e1f.37.17684352000000001"; + /** One digit off the matching trace's id. */ + private static final String OTHER_TRACE = "4a2b6c8d0e1f.37.17684352000000002"; + private static final String UNKNOWN_TRACE = "4a2b6c8d0e1f.37.17684352000000009"; + + /** On the range's start, before midnight, at the least latency. */ + private static final String FIRST_MATCH = "segment-first-match"; + /** On the range's end instant, after midnight, at the most latency. */ + private static final String LAST_MATCH = "segment-last-match"; + private static final String OTHER_SERVICE = "segment-other-service"; + private static final String OTHER_INSTANCE_SEGMENT = "segment-other-instance"; + private static final String OTHER_ENDPOINT_SEGMENT = "segment-other-endpoint"; + private static final String OTHER_TRACE_SEGMENT = "segment-other-trace"; + private static final String BELOW_MIN_LATENCY = "segment-below-min-latency"; + private static final String ABOVE_MAX_LATENCY = "segment-above-max-latency"; + private static final String SUCCEEDED = "segment-succeeded"; + /** Has one of the two tags searched for, and the other with another value. */ + private static final String OTHER_TAG_VALUE = "segment-other-tag-value"; + private static final String BEFORE_START = "segment-before-start"; + private static final String AFTER_END = "segment-after-end"; + private static final String ON_PREVIOUS_MIDNIGHT = "segment-on-previous-midnight"; + /** The matching trace's call to the inventory, after the range. */ + private static final String DOWNSTREAM = "segment-downstream"; + + private static final Map SEGMENTS = fixture(); + /** The segments in the range, latest start first. */ + private static final List IN_RANGE = Arrays.asList( + LAST_MATCH, OTHER_TRACE_SEGMENT, OTHER_TAG_VALUE, SUCCEEDED, ABOVE_MAX_LATENCY, BELOW_MIN_LATENCY, + OTHER_ENDPOINT_SEGMENT, OTHER_INSTANCE_SEGMENT, OTHER_SERVICE, FIRST_MATCH); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private TraceQueryEsDAO dao; + + TraceQueryEsDAOITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeSegments() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + final Model model = support.installRecord(SegmentRecord.class); + // Neither in start order nor against it. + final List records = new ArrayList<>(); + for (final String id : Arrays.asList( + OTHER_TAG_VALUE, FIRST_MATCH, AFTER_END, OTHER_INSTANCE_SEGMENT, DOWNSTREAM, SUCCEEDED, LAST_MATCH, + BELOW_MIN_LATENCY, OTHER_SERVICE, BEFORE_START, OTHER_TRACE_SEGMENT, ABOVE_MAX_LATENCY, + OTHER_ENDPOINT_SEGMENT, ON_PREVIOUS_MIDNIGHT)) { + records.add(SEGMENTS.get(id).record()); + } + support.writeRecords(model, new SegmentRecord.Builder(), records); + dao = new TraceQueryEsDAO(support.getClient(), support.getConfig().getSegmentQueryMaxSize()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @ParameterizedTest(name = "{0} {1}") + @MethodSource("conditionsInBothOrders") + public void findsTheSegmentsMatchingOneCondition(final Condition condition, final QueryOrder order) + throws IOException { + final List expected = expected(condition.matching(), order); + assertEquals(expected, search(condition.criterion, order, 0, expected.size())); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(QueryOrder.class) + public void findsOnlyTheSegmentsMatchingEveryCondition(final QueryOrder order) throws IOException { + final Consumer everyCondition = search -> { + for (final Condition condition : Condition.values()) { + if (condition != Condition.SUCCESS) { + condition.criterion.accept(search); + } + } + }; + assertEquals(expected(Arrays.asList(FIRST_MATCH, LAST_MATCH), order), search(everyCondition, order, 0, 10)); + } + + @Test + public void findsNoSegmentWhenNoneMatches() throws IOException { + assertEquals(Collections.emptyList(), + search(search -> search.serviceId = serviceId("unknown"), QueryOrder.BY_START_TIME, 0, 10)); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(QueryOrder.class) + public void pagesThroughTheSegmentsOfTheRange(final QueryOrder order) throws IOException { + final List all = expected(IN_RANGE, order); + assertEquals(all.subList(0, 4), search(search -> { }, order, 0, 4)); + assertEquals(all.subList(4, 8), search(search -> { }, order, 4, 4)); + assertEquals(all.subList(8, 10), search(search -> { }, order, 8, 4)); + assertEquals(Collections.emptyList(), search(search -> { }, order, 12, 4)); + } + + /** + * A range ending in a day's last second ends at the next midnight, which is in the next day's index. + */ + @Test + public void findsASegmentOnTheEndInstantOfARangeEndingAtMidnight() throws IOException { + final Duration endingAtMidnight = duration(PREVIOUS_MIDNIGHT - 60_000L, PREVIOUS_MIDNIGHT - 1_000L); + assertEquals(expected(Collections.singletonList(ON_PREVIOUS_MIDNIGHT), QueryOrder.BY_START_TIME), + search(endingAtMidnight, search -> { }, QueryOrder.BY_START_TIME, 0, 10)); + } + + @Test + public void readsAWholeTraceByIdAcrossMidnight() throws IOException { + final List trace = records(FIRST_MATCH, LAST_MATCH, DOWNSTREAM); + assertEquals(trace, sorted(dao.queryByTraceId(TRACE, AROUND_MIDNIGHT))); + assertEquals(trace, sorted(dao.queryByTraceId(TRACE, null))); + assertEquals(Collections.emptyList(), dao.queryByTraceId(UNKNOWN_TRACE, null)); + } + + /** + * The flexible trace query reads the traces of a third-party format that has no segments, which this storage + * does not keep. + */ + @Test + public void readsNoTraceWithoutSegments() throws IOException { + assertEquals(Collections.emptyList(), dao.doFlexibleTraceQuery(TRACE)); + } + + @Test + public void readsTheProfiledSegmentsById() throws IOException { + assertEquals(records(FIRST_MATCH, LAST_MATCH, BEFORE_START), + sorted(dao.queryBySegmentIdList( + Arrays.asList(LAST_MATCH, "segment-unknown", BEFORE_START, FIRST_MATCH), AROUND_MIDNIGHT))); + } + + /** + * apache/skywalking#14239: a trace profiling task without a sampled segment has no segment id to read. + */ + @Test + public void readsNoSegmentForNoSegmentId() throws IOException { + assertEquals(Collections.emptyList(), dao.queryBySegmentIdList(Collections.emptyList(), AROUND_MIDNIGHT)); + } + + /** + * The segments of other traces on the profiled instance stay out, and so do the listed traces' segments on the + * other instances until those are asked for: the matching trace's call to the inventory, and the other listed + * trace's segment on the second checkout instance. + */ + @Test + public void readsTheSegmentsOfTheProfiledTracesOnTheProfiledInstances() throws IOException { + final List traces = Arrays.asList(TRACE, traceOf(OTHER_INSTANCE_SEGMENT)); + assertEquals(records(FIRST_MATCH, LAST_MATCH), sorted(dao.queryByTraceIdWithInstanceId( + traces, Collections.singletonList(instanceId(CHECKOUT, CHECKOUT_1)), null))); + assertEquals(records(FIRST_MATCH, LAST_MATCH, DOWNSTREAM, OTHER_INSTANCE_SEGMENT), + sorted(dao.queryByTraceIdWithInstanceId(traces, Arrays.asList( + instanceId(CHECKOUT, CHECKOUT_1), instanceId(CHECKOUT, OTHER_INSTANCE), + instanceId(INVENTORY, "inventory-1")), null))); + } + + @Test + public void readsNoSegmentForNoTraceOrNoInstance() throws IOException { + assertEquals(Collections.emptyList(), dao.queryByTraceIdWithInstanceId( + Collections.emptyList(), Collections.singletonList(instanceId(CHECKOUT, CHECKOUT_1)), null)); + assertEquals(Collections.emptyList(), dao.queryByTraceIdWithInstanceId( + Collections.singletonList(TRACE), Collections.emptyList(), null)); + } + + /** What the trace search filters by, and the segments in the range each keeps. */ + enum Condition { + TIME_RANGE_ONLY(search -> { }), + SERVICE(search -> search.serviceId = serviceId(CHECKOUT), OTHER_SERVICE), + // The other service's segment is on an instance and an endpoint of that service. + INSTANCE(search -> search.instanceId = instanceId(CHECKOUT, CHECKOUT_1), + OTHER_INSTANCE_SEGMENT, OTHER_SERVICE), + ENDPOINT(search -> search.endpointId = endpointId(CHECKOUT, POST_ORDERS), + OTHER_ENDPOINT_SEGMENT, OTHER_SERVICE), + TRACE_ID(search -> search.traceId = TRACE, only(FIRST_MATCH, LAST_MATCH)), + MIN_DURATION(search -> search.minLatency = MIN_LATENCY, BELOW_MIN_LATENCY), + MAX_DURATION(search -> search.maxLatency = MAX_LATENCY, ABOVE_MAX_LATENCY), + ERROR(search -> search.state = TraceState.ERROR, SUCCEEDED), + SUCCESS(search -> search.state = TraceState.SUCCESS, only(SUCCEEDED)), + TAGS(search -> search.tags = Arrays.asList(METHOD, STATUS), OTHER_TAG_VALUE); + + private final Consumer criterion; + private final List excluded; + private final List matching; + + Condition(final Consumer criterion, final String... excluded) { + this.criterion = criterion; + this.excluded = Arrays.asList(excluded); + this.matching = null; + } + + Condition(final Consumer criterion, final List matching) { + this.criterion = criterion; + this.excluded = Collections.emptyList(); + this.matching = matching; + } + + private List matching() { + if (matching != null) { + return matching; + } + final List kept = new ArrayList<>(IN_RANGE); + kept.removeAll(excluded); + return kept; + } + + private static List only(final String... segments) { + return Arrays.asList(segments); + } + } + + static Stream conditionsInBothOrders() { + return Arrays.stream(Condition.values()).flatMap(condition -> Stream.of( + Arguments.of(condition, QueryOrder.BY_START_TIME), + Arguments.of(condition, QueryOrder.BY_DURATION))); + } + + /** The arguments of a trace search besides the range, the order and the page. */ + private static final class Search { + private long minLatency; + private long maxLatency; + private String serviceId; + private String instanceId; + private String endpointId; + private String traceId; + private TraceState state = TraceState.ALL; + private List tags; + } + + private List search(final Consumer criteria, final QueryOrder order, final int from, + final int limit) throws IOException { + return search(AROUND_MIDNIGHT, criteria, order, from, limit); + } + + private List search(final Duration range, final Consumer criteria, final QueryOrder order, + final int from, final int limit) throws IOException { + final Search search = new Search(); + criteria.accept(search); + final List found = new ArrayList<>(); + for (final BasicTrace trace : dao.queryBasicTraces( + range, search.minLatency, search.maxLatency, search.serviceId, search.instanceId, + search.endpointId, search.traceId, limit, from, search.state, order, search.tags).getTraces()) { + found.add(render(trace)); + } + return found; + } + + /** The segments as the search lists them, in the order asked for. */ + private static List expected(final List segments, final QueryOrder order) { + final Comparator latestFirst = Comparator.comparingLong(segment -> -segment.start); + final Comparator slowestFirst = Comparator.comparingInt(segment -> -segment.latency); + return segments.stream() + .map(SEGMENTS::get) + .sorted(order == QueryOrder.BY_DURATION ? slowestFirst : latestFirst) + .map(StoredSegment::listed) + .collect(Collectors.toList()); + } + + private static String render(final BasicTrace trace) { + return trace.getSegmentId() + " of " + trace.getTraceIds() + " at " + trace.getEndpointNames() + + ", start " + trace.getStart() + ", " + trace.getDuration() + " ms, error " + trace.isError(); + } + + /** Every column of the segment the trace and profiling queries use. */ + private static String render(final SegmentRecord segment) { + return segment.getSegmentId() + " of " + segment.getTraceId() + " on " + segment.getServiceId() + " / " + + segment.getServiceInstanceId() + " / " + segment.getEndpointId() + ", start " + segment.getStartTime() + + ", " + segment.getLatency() + " ms, error " + segment.getIsError() + ", " + + Base64.getEncoder().encodeToString(segment.getDataBinary()); + } + + private static List records(final String... segments) { + return Arrays.stream(segments).sorted().map(id -> render(SEGMENTS.get(id).record())) + .collect(Collectors.toList()); + } + + private static List sorted(final List segments) { + return segments.stream() + .sorted(Comparator.comparing(SegmentRecord::getSegmentId)) + .map(TraceQueryEsDAOITBase::render) + .collect(Collectors.toList()); + } + + private static Map fixture() { + final Map segments = new LinkedHashMap<>(); + add(segments, new StoredSegment(FIRST_MATCH, START, MIN_LATENCY).trace(TRACE)); + add(segments, new StoredSegment(OTHER_SERVICE, MIDNIGHT - 50_000L, 150).service(CANARY)); + add(segments, new StoredSegment(OTHER_INSTANCE_SEGMENT, MIDNIGHT - 45_000L, 200).instance(OTHER_INSTANCE)); + add(segments, new StoredSegment(OTHER_ENDPOINT_SEGMENT, MIDNIGHT - 40_000L, 250).endpoint(OTHER_ENDPOINT)); + add(segments, new StoredSegment(BELOW_MIN_LATENCY, MIDNIGHT - 30_000L, MIN_LATENCY - 1)); + add(segments, new StoredSegment(ABOVE_MAX_LATENCY, MIDNIGHT - 20_000L, MAX_LATENCY + 1)); + add(segments, new StoredSegment(SUCCEEDED, MIDNIGHT - 10_000L, 300).succeeded()); + add(segments, new StoredSegment(OTHER_TAG_VALUE, MIDNIGHT + 10_000L, 350) + .tags(METHOD, new Tag(STATUS.getKey(), "200"))); + add(segments, new StoredSegment(OTHER_TRACE_SEGMENT, MIDNIGHT + 20_000L, 400).trace(OTHER_TRACE)); + add(segments, new StoredSegment(LAST_MATCH, END, MAX_LATENCY).trace(TRACE)); + add(segments, new StoredSegment(BEFORE_START, START - 1, 500)); + add(segments, new StoredSegment(AFTER_END, END + 1, 550)); + add(segments, new StoredSegment(ON_PREVIOUS_MIDNIGHT, PREVIOUS_MIDNIGHT, 600)); + add(segments, new StoredSegment(DOWNSTREAM, END + 1_000L, 50) + .trace(TRACE).service(INVENTORY).instance("inventory-1").endpoint("GET:/stock").tags()); + return segments; + } + + private static void add(final Map segments, final StoredSegment segment) { + segments.put(segment.id, segment); + } + + /** A segment of its own trace matching every condition unless told otherwise. */ + private static final class StoredSegment { + private final String id; + private final long start; + private final int latency; + private String traceId; + private String service = CHECKOUT; + private String instance = CHECKOUT_1; + private String endpoint = POST_ORDERS; + private boolean error = true; + private List tags = Arrays.asList(METHOD, STATUS); + + private StoredSegment(final String id, final long start, final int latency) { + this.id = id; + this.start = start; + this.latency = latency; + this.traceId = traceOf(id); + } + + private StoredSegment trace(final String traceId) { + this.traceId = traceId; + return this; + } + + private StoredSegment service(final String service) { + this.service = service; + return this; + } + + private StoredSegment instance(final String instance) { + this.instance = instance; + return this; + } + + private StoredSegment endpoint(final String endpoint) { + this.endpoint = endpoint; + return this; + } + + private StoredSegment succeeded() { + this.error = false; + return this; + } + + private StoredSegment tags(final Tag... tags) { + this.tags = Arrays.asList(tags); + return this; + } + + /** The record {@code SegmentAnalysisListener} stores for the segment, with its searchable tags. */ + private SegmentRecord record() { + final SegmentObject segment = SegmentObject.newBuilder() + .setTraceId(traceId) + .setTraceSegmentId(id) + .setService(service) + .setServiceInstance(instance) + .addSpans(SpanObject.newBuilder() + .setSpanId(0) + .setParentSpanId(-1) + .setStartTime(start) + .setEndTime(start + latency) + .setOperationName(endpoint) + .setSpanType(SpanType.Entry) + .setIsError(error)) + .build(); + final SegmentRecord record = new SegmentRecord(); + record.setSegmentId(id); + record.setTraceId(traceId); + record.setServiceId(serviceId(service)); + record.setServiceInstanceId(instanceId(service, instance)); + record.setEndpointId(endpointId(service, endpoint)); + record.setStartTime(start); + record.setLatency(latency); + record.setIsError(error ? 1 : 0); + record.setDataBinary(segment.toByteArray()); + record.setTags(tags.stream().map(Tag::toString).collect(Collectors.toList())); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(start)); + return record; + } + + /** The segment as the trace search lists it. */ + private String listed() { + final BasicTrace trace = new BasicTrace(); + trace.setSegmentId(id); + trace.getTraceIds().add(traceId); + trace.getEndpointNames().add(endpoint); + trace.setStart(String.valueOf(start)); + trace.setDuration(latency); + trace.setError(error); + return render(trace); + } + } + + private static String traceOf(final String segment) { + return "trace-of-" + segment; + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + private static String instanceId(final String service, final String instance) { + return IDManager.ServiceInstanceID.buildId(serviceId(service), instance); + } + + private static String endpointId(final String service, final String endpoint) { + return IDManager.EndpointID.buildId(serviceId(service), endpoint); + } + + /** The duration the query protocol passes for a range read by the second. */ + private static Duration duration(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.SECOND); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmmss")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmmss")); + return duration; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/otlp/OTLPTraceQueryEsDAOITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/otlp/OTLPTraceQueryEsDAOITBase.java new file mode 100644 index 000000000000..2280aa35b855 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/otlp/OTLPTraceQueryEsDAOITBase.java @@ -0,0 +1,775 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.otlp; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.function.IntFunction; +import java.util.function.IntPredicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.otlp.OTLPServiceRelationTraffic; +import org.apache.skywalking.oap.server.core.otlp.OTLPServiceSpanTraffic; +import org.apache.skywalking.oap.server.core.otlp.OTLPServiceTraffic; +import org.apache.skywalking.oap.server.core.otlp.OTLPSpanRecord; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.OTLPTraceQueryCondition; +import org.apache.skywalking.oap.server.core.query.type.QueryOrder; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.query.proto.Source; +import org.apache.skywalking.oap.server.core.storage.query.proto.SpanWrapper; +import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.testcontainers.elasticsearch.ElasticsearchContainer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Reads natively stored OTLP spans the way the TraceQL API does on Elasticsearch and OpenSearch, through + * {@link OTLPTraceQueryEsDAO}: the trace search, the read of one trace by id, and the catalogs of services, span names + * and peer services. A nested class of a server's IT names the server and the storage mode. + * + *

The search range is a minute either side of midnight, so it reads two daily indices. Each search condition is + * tried alone, in both orders, and with all the others. For every condition a near-miss trace matches all the others + * but that one, the two matching traces lie on the bounds of the range (a span is kept by its start millisecond, the + * range's end counted) and of the span duration, and a search asks for exactly as many traces as should match, so a + * near miss that leaks in pushes a match out. A search ranks a trace by its earliest matching span, or by its longest, + * and returns it whole: the first matching trace has a span of another service after midnight, the last one a root + * span of another service before the range. + * + *

The DAO serves {@code data_binary} untouched, so each span stores a label there instead of a + * {@code ResourceSpans}. The catalogs keep each name once, in the minute it was first seen, on either side of + * midnight; a service name that arrived with two layers is two rows. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class OTLPTraceQueryEsDAOITBase { + /** A fixed local midnight, so the daily indices the spans are written to are known. */ + private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); + /** + * A minute either side of midnight, read by the second: its end timestamp, which the search counts, is the first + * millisecond after its last second. + */ + private static final Duration AROUND_MIDNIGHT = duration(MIDNIGHT - 60_000L, MIDNIGHT + 59_000L); + private static final long START = AROUND_MIDNIGHT.getStartTimestamp(); + private static final long END = AROUND_MIDNIGHT.getEndTimestamp(); + + private static final long HOUR = 3_600_000L; + private static final long DAY = 24 * HOUR; + private static final OTLPSpanRecord.Builder BUILDER = new OTLPSpanRecord.Builder(); + /** The service of the ranking cases, whose spans are days after the other fixtures' and in their own indices. */ + private static final String RANKED = "ranked"; + private static final int RANKED_LIMIT = 2; + /** More than the traces a shard answers a terms aggregation of {@link #RANKED_LIMIT} with by default. */ + private static final int RANKED_OTHERS = RANKED_LIMIT * 3 / 2 + 10 + 3; + + private static final String CHECKOUT = "checkout"; + private static final String CANARY = "checkout-canary"; + private static final String INVENTORY = "inventory"; + private static final String FRONTEND = "frontend"; + private static final String CHECKOUT_1 = "checkout-1"; + private static final String OKHTTP = "io.opentelemetry.okhttp-3.0"; + private static final String POST_STOCK = "POST /stock"; + private static final String PEER = INVENTORY; + private static final int INTERNAL = 1; + private static final int SERVER = 2; + private static final int CLIENT = 3; + private static final int STATUS_UNSET = 0; + private static final int STATUS_OK = 1; + private static final int STATUS_ERROR = 2; + private static final long MIN_DURATION_NS = 1_000_000L; + private static final long MAX_DURATION_NS = 9_000_000L; + private static final String ENVIRONMENT = "deployment.environment"; + private static final String TENANT = "tenant"; + + /** Found by its root span on the range's start; its child, after midnight, is the inventory's. */ + private static final String FIRST_MATCH = "4bf92f3577b34da6a3ce929d0e0e0a01"; + /** Found by a span on the range's end, whose root, before the range, is the frontend's; one digit off the first. */ + private static final String LAST_MATCH = "4bf92f3577b34da6a3ce929d0e0e0a02"; + private static final String OTHER_SERVICE = "4bf92f3577b34da6a3ce929d0e0e0b01"; + private static final String OTHER_INSTANCE = "4bf92f3577b34da6a3ce929d0e0e0b02"; + private static final String OTHER_SCOPE = "4bf92f3577b34da6a3ce929d0e0e0b03"; + private static final String OTHER_SPAN_NAME = "4bf92f3577b34da6a3ce929d0e0e0b04"; + private static final String OTHER_PEER = "4bf92f3577b34da6a3ce929d0e0e0b05"; + private static final String OTHER_KIND = "4bf92f3577b34da6a3ce929d0e0e0b06"; + private static final String OTHER_STATUS = "4bf92f3577b34da6a3ce929d0e0e0b07"; + private static final String BELOW_MIN_DURATION = "4bf92f3577b34da6a3ce929d0e0e0b08"; + private static final String ABOVE_MAX_DURATION = "4bf92f3577b34da6a3ce929d0e0e0b09"; + private static final String OTHER_RESOURCE_ATTRIBUTE = "4bf92f3577b34da6a3ce929d0e0e0b0a"; + private static final String OTHER_SPAN_ATTRIBUTE = "4bf92f3577b34da6a3ce929d0e0e0b0b"; + /** Has the tenant as a resource attribute, not as a span attribute. */ + private static final String TENANT_OF_RESOURCE = "4bf92f3577b34da6a3ce929d0e0e0b0c"; + private static final String BEFORE_START = "4bf92f3577b34da6a3ce929d0e0e0b0d"; + private static final String AFTER_END = "4bf92f3577b34da6a3ce929d0e0e0b0e"; + private static final String UNKNOWN = "4bf92f3577b34da6a3ce929d0e0e0fff"; + + /** The span each trace is found by. */ + private static final Map MATCHED_BY = new LinkedHashMap<>(); + /** Every trace's spans. */ + private static final Map> TRACES = fixture(); + /** The traces in the range, latest first. */ + private static final List IN_RANGE = Arrays.asList( + LAST_MATCH, TENANT_OF_RESOURCE, OTHER_SPAN_ATTRIBUTE, OTHER_RESOURCE_ATTRIBUTE, ABOVE_MAX_DURATION, + BELOW_MIN_DURATION, OTHER_STATUS, OTHER_KIND, OTHER_PEER, OTHER_SPAN_NAME, OTHER_SCOPE, OTHER_INSTANCE, + OTHER_SERVICE, FIRST_MATCH); + + private final ElasticsearchContainer server; + private final ElasticSearchQueryITSupport.Mode mode; + private ElasticSearchQueryITSupport support; + private OTLPTraceQueryEsDAO dao; + private Model spans; + private long yesterday; + private long today; + + protected OTLPTraceQueryEsDAOITBase(final ElasticsearchContainer server, + final ElasticSearchQueryITSupport.Mode mode) { + this.server = server; + this.mode = mode; + } + + @BeforeAll + public void writeSpans() throws Exception { + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + spans = support.installRecord(OTLPSpanRecord.class); + final Model services = support.installMetrics(OTLPServiceTraffic.class); + final Model spanNames = support.installMetrics(OTLPServiceSpanTraffic.class); + final Model peers = support.installMetrics(OTLPServiceRelationTraffic.class); + + // Neither in time order nor against it, and each trace's spans child first. + final List records = new ArrayList<>(); + for (final String traceId : Arrays.asList( + OTHER_STATUS, LAST_MATCH, BELOW_MIN_DURATION, OTHER_SPAN_ATTRIBUTE, AFTER_END, OTHER_SERVICE, FIRST_MATCH, + OTHER_PEER, TENANT_OF_RESOURCE, OTHER_SCOPE, BEFORE_START, ABOVE_MAX_DURATION, OTHER_INSTANCE, + OTHER_RESOURCE_ATTRIBUTE, OTHER_KIND, OTHER_SPAN_NAME)) { + final List trace = new ArrayList<>(TRACES.get(traceId)); + Collections.reverse(trace); + for (final StoredSpan span : trace) { + records.add(span.record()); + } + } + support.writeRecords(spans, new OTLPSpanRecord.Builder(), records); + + yesterday = TimeBucket.getMinuteTimeBucket(MIDNIGHT - 120_000L); + today = TimeBucket.getMinuteTimeBucket(MIDNIGHT + 60_000L); + support.writeMetrics(services, new OTLPServiceTraffic.Builder(), Arrays.asList( + service(CHECKOUT, Layer.GENERAL.name(), yesterday), + service(CANARY, Layer.GENERAL.name(), yesterday), + service(FRONTEND, Layer.GENERAL.name(), yesterday), + service(CHECKOUT, Layer.MESH.name(), today), + service(INVENTORY, Layer.UNDEFINED.name(), today))); + support.writeMetrics(spanNames, new OTLPServiceSpanTraffic.Builder(), Arrays.asList( + spanName(CHECKOUT, POST_STOCK, yesterday), + spanName(CHECKOUT, "GET /stock", yesterday), + spanName(CANARY, POST_STOCK, yesterday), + spanName(CHECKOUT, "POST /orders", today), + spanName(INVENTORY, "POST /stock/reserve", today))); + support.writeMetrics(peers, new OTLPServiceRelationTraffic.Builder(), Arrays.asList( + peer(CHECKOUT, PEER, yesterday), + peer(CANARY, PEER, yesterday), + peer(CHECKOUT, "payments", today))); + + dao = new OTLPTraceQueryEsDAO(support.getClient()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void listsEveryServiceRowOfEveryLayer() { + assertEquals(sorted(Arrays.asList( + CANARY + " in GENERAL since " + yesterday, + CHECKOUT + " in GENERAL since " + yesterday, + CHECKOUT + " in MESH since " + today, + FRONTEND + " in GENERAL since " + yesterday, + INVENTORY + " in UNDEFINED since " + today)), services(null)); + } + + @Test + public void listsTheServicesOfOneLayer() { + assertEquals(sorted(Arrays.asList( + CANARY + " in GENERAL since " + yesterday, + CHECKOUT + " in GENERAL since " + yesterday, + FRONTEND + " in GENERAL since " + yesterday)), services(Layer.GENERAL.name())); + assertEquals(Collections.singletonList(CHECKOUT + " in MESH since " + today), services(Layer.MESH.name())); + assertEquals(Collections.emptyList(), services(Layer.K8S.name())); + } + + @Test + public void listsTheSpanNamesOfEachService() { + assertEquals(Arrays.asList("GET /stock", "POST /orders", POST_STOCK), sorted(dao.getSpanNames(CHECKOUT))); + assertEquals(Collections.singletonList(POST_STOCK), sorted(dao.getSpanNames(CANARY))); + assertEquals(Collections.emptyList(), dao.getSpanNames("unknown")); + } + + @Test + public void listsThePeerServicesEachServiceCalled() { + assertEquals(Arrays.asList(PEER, "payments"), sorted(dao.getPeerServiceNames(CHECKOUT))); + assertEquals(Collections.singletonList(PEER), sorted(dao.getPeerServiceNames(CANARY))); + assertEquals(Collections.emptyList(), dao.getPeerServiceNames(INVENTORY)); + } + + /** + * A range bounds the read to the spans that started in it, the range's end counted; no range reads them all. + */ + @Test + public void readsATraceById() { + assertEquals(trace(FIRST_MATCH), render(dao.queryTraceById(FIRST_MATCH, AROUND_MIDNIGHT))); + assertEquals(trace(FIRST_MATCH), render(dao.queryTraceById(FIRST_MATCH, null))); + assertEquals(trace(LAST_MATCH), render(dao.queryTraceById(LAST_MATCH, null))); + assertEquals(Collections.singletonList(MATCHED_BY.get(LAST_MATCH).render()), + render(dao.queryTraceById(LAST_MATCH, AROUND_MIDNIGHT))); + assertEquals(Collections.emptyList(), dao.queryTraceById(BEFORE_START, AROUND_MIDNIGHT)); + assertEquals(Collections.emptyList(), dao.queryTraceById(AFTER_END, AROUND_MIDNIGHT)); + assertEquals(Collections.emptyList(), dao.queryTraceById(UNKNOWN, null)); + } + + /** + * A scroll context left open is held for the scroll's keep-alive, and a node refuses every new scroll, of any + * query, past {@code search.max_open_scroll_context}. The read by id and the search's read of the traces found + * both scroll. + */ + @Test + public void leavesNoScrollContextOpen() throws Exception { + final long openBefore = support.openScrollContexts(); + dao.queryTraceById(UNKNOWN, null); + dao.queryTraceById(FIRST_MATCH, AROUND_MIDNIGHT); + search(condition -> { }, QueryOrder.BY_START_TIME, 2); + final long openAfter = support.openScrollContexts(); + assertTrue(openAfter <= openBefore, + "scroll contexts open before the reads: " + openBefore + ", after them: " + openAfter); + } + + @ParameterizedTest(name = "{0} {1}") + @MethodSource("conditionsInBothOrders") + public void findsTheTracesMatchingOneCondition(final Condition condition, final QueryOrder order) { + final List> expected = expected(condition.matching(), order); + assertEquals(expected, search(condition.criterion, order, expected.size())); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(QueryOrder.class) + public void findsOnlyTheTracesMatchingEveryCondition(final QueryOrder order) { + final Consumer everyCondition = condition -> { + for (final Condition each : Condition.values()) { + if (each != Condition.TRACE_ID) { + each.criterion.accept(condition); + } + } + }; + assertEquals(expected(Arrays.asList(FIRST_MATCH, LAST_MATCH), order), search(everyCondition, order, 2)); + assertEquals(expected(Collections.singletonList(FIRST_MATCH), order), + search(everyCondition.andThen(Condition.TRACE_ID.criterion), order, 2)); + } + + @Test + public void findsNoTraceWhenNoSpanMatches() { + assertEquals(Collections.emptyList(), + search(condition -> condition.setServiceName("unknown"), QueryOrder.BY_START_TIME, 10)); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(QueryOrder.class) + public void keepsAsManyTracesAsTheLimit(final QueryOrder order) { + final List> all = expected(Condition.SERVICE.matching(), order); + for (final int limit : new int[] {1, 3, 5}) { + assertEquals(all.subList(0, limit), search(Condition.SERVICE.criterion, order, limit), "limit " + limit); + } + } + + /** + * The first matching trace's span after midnight ranks it among the latest traces of that day, but the trace + * ranks by its earliest span, before midnight, so the four latest traces are all others. + */ + @Test + public void ranksATraceOfTwoDaysByItsEarliestSpan() { + assertEquals(expected(IN_RANGE, QueryOrder.BY_START_TIME).subList(0, 4), + search(condition -> { }, QueryOrder.BY_START_TIME, 4)); + } + + /** + * A trace ranks by its earliest matching span, but the search's terms aggregation lets each shard rank a trace by + * the spans it holds and answer with its first {@code shard_size} traces, an approximation accepted for its cost + * (see {@link OTLPTraceQueryEsDAO#queryTraces(OTLPTraceQueryCondition)}). The older trace's first span shares its + * shard with more than {@code shard_size} later traces, and its later span, newer than all of them, is routed by + * its document id to another shard, which ranks the trace first: the search returns the older trace first, then + * the newest of the others. Custom routing keeps a trace's spans on one shard of a daily index, and there the + * search is exact. The JDBC storages, which rank over every table, return the exact traces in both cases. + */ + @Test + public void keepsAnOlderTraceWithALaterSpanOnAnotherShardOnlyWithDefaultRouting() throws Exception { + final long first = MIDNIGHT + 2 * DAY + 12 * HOUR; + final RankedCase ranked = writeRanked(first, first + 30_000L, false); + assertEquals(customRouting() ? ranked.exact() : ranked.approximate(), + searchRanked(first, first + 30_000L)); + } + + /** + * The older trace's first span is in the earlier daily index, on a shard with more than {@code shard_size} later + * traces, and its later span is in the next daily index, which ranks the trace first; routing keeps a trace on + * one shard of a daily index only. So with either routing the search returns the older trace first, then the + * newest of the others, where the JDBC storages return the two newest others. + */ + @Test + public void keepsAnOlderTraceThatCrossesMidnightByItsLaterSpan() throws Exception { + final long midnight = MIDNIGHT + 4 * DAY; + final RankedCase ranked = writeRanked(midnight - 50_000L, midnight + 10_000L, true); + assertEquals(ranked.approximate(), searchRanked(midnight - 50_000L, midnight + 10_000L)); + } + + /** What the TraceQL search filters by, and the traces in the range each keeps. */ + enum Condition { + TIME_RANGE_ONLY(condition -> { }), + SERVICE(condition -> condition.setServiceName(CHECKOUT), OTHER_SERVICE), + INSTANCE(condition -> condition.setServiceInstance(CHECKOUT_1), OTHER_INSTANCE), + SCOPE(condition -> condition.setScopeName(OKHTTP), OTHER_SCOPE), + SPAN_NAME(condition -> condition.setSpanName(POST_STOCK), OTHER_SPAN_NAME), + PEER_SERVICE(condition -> condition.setPeerService(PEER), OTHER_PEER), + KIND(condition -> condition.setKind(CLIENT), OTHER_KIND), + STATUS(condition -> condition.setStatusCode(STATUS_ERROR), OTHER_STATUS), + MIN_DURATION(condition -> condition.setMinDurationNanos(MIN_DURATION_NS), BELOW_MIN_DURATION), + MAX_DURATION(condition -> condition.setMaxDurationNanos(MAX_DURATION_NS), ABOVE_MAX_DURATION), + RESOURCE_ATTRIBUTE(condition -> condition.getTags().add( + new Tag(OTLPSpanRecord.RESOURCE_TAG_PREFIX + ENVIRONMENT, "production")), OTHER_RESOURCE_ATTRIBUTE), + SPAN_ATTRIBUTE(condition -> condition.getTags().add(new Tag(OTLPSpanRecord.SPAN_TAG_PREFIX + TENANT, "acme")), + OTHER_SPAN_ATTRIBUTE, TENANT_OF_RESOURCE), + ATTRIBUTE_OF_EITHER_SCOPE(condition -> condition.getTags().add(new Tag(TENANT, "acme")), OTHER_SPAN_ATTRIBUTE), + // The last matching trace's id is one digit off the first's. + TRACE_ID(condition -> condition.setTraceId(FIRST_MATCH), only(FIRST_MATCH)); + + private final Consumer criterion; + private final List excluded; + private final List matching; + + Condition(final Consumer criterion, final String... excluded) { + this.criterion = criterion; + this.excluded = Arrays.asList(excluded); + this.matching = null; + } + + Condition(final Consumer criterion, final List matching) { + this.criterion = criterion; + this.excluded = Collections.emptyList(); + this.matching = matching; + } + + private List matching() { + if (matching != null) { + return matching; + } + final List kept = new ArrayList<>(IN_RANGE); + kept.removeAll(excluded); + return kept; + } + + private static List only(final String... traceIds) { + return Arrays.asList(traceIds); + } + } + + static Stream conditionsInBothOrders() { + return Arrays.stream(Condition.values()).flatMap(condition -> Stream.of( + Arguments.of(condition, QueryOrder.BY_START_TIME), + Arguments.of(condition, QueryOrder.BY_DURATION))); + } + + private List> search(final Consumer criteria, final QueryOrder order, + final int limit) { + final OTLPTraceQueryCondition condition = new OTLPTraceQueryCondition(); + criteria.accept(condition); + condition.setQueryDuration(AROUND_MIDNIGHT); + condition.setQueryOrder(order); + condition.setLimit(limit); + final List> traces = new ArrayList<>(); + for (final List trace : dao.queryTraces(condition)) { + traces.add(render(trace)); + } + return traces; + } + + /** The traces whole, the latest or the longest first by the span each is found by. */ + private static List> expected(final List traceIds, final QueryOrder order) { + final Comparator latestFirst = Comparator.comparingLong(traceId -> -MATCHED_BY.get(traceId).start); + final Comparator longestFirst = + Comparator.comparingLong(traceId -> -MATCHED_BY.get(traceId).duration); + return traceIds.stream() + .sorted(order == QueryOrder.BY_DURATION ? longestFirst : latestFirst) + .map(OTLPTraceQueryEsDAOITBase::trace) + .collect(Collectors.toList()); + } + + private static List trace(final String traceId) { + return TRACES.get(traceId).stream().map(StoredSpan::render).sorted().collect(Collectors.toList()); + } + + /** The trace's spans in label order: a trace's spans come in no promised order. */ + private static List render(final List trace) { + return trace.stream() + .map(span -> span.getSpan().toStringUtf8() + " of " + span.getService() + ", " + span.getSource()) + .sorted() + .collect(Collectors.toList()); + } + + private List services(final String layer) { + return dao.getServices(layer).stream() + .map(service -> service.getServiceName() + " in " + service.getLayer() + " since " + + service.getTimeBucket()) + .sorted() + .collect(Collectors.toList()); + } + + private static List sorted(final List names) { + final List sorted = new ArrayList<>(names); + Collections.sort(sorted); + return sorted; + } + + private static Map> fixture() { + final Map> traces = new LinkedHashMap<>(); + found(traces, FIRST_MATCH, new StoredSpan(FIRST_MATCH, "first match", START, MIN_DURATION_NS), + new StoredSpan(FIRST_MATCH, "first match, the inventory's", MIDNIGHT + 40_000L, 10_000L) + .childOf("first match").at(INVENTORY, "inventory-1", "io.opentelemetry.servlet-5.0") + .call("POST /stock/reserve", SERVER, STATUS_UNSET, "").attributes("production", "acme")); + found(traces, OTHER_SERVICE, new StoredSpan(OTHER_SERVICE, "other service", MIDNIGHT - 55_000L, 1_500_000L) + .at(CANARY, CHECKOUT_1, OKHTTP)); + found(traces, OTHER_INSTANCE, new StoredSpan(OTHER_INSTANCE, "other instance", MIDNIGHT - 50_000L, 2_000_000L) + .at(CHECKOUT, "checkout-2", OKHTTP)); + found(traces, OTHER_SCOPE, new StoredSpan(OTHER_SCOPE, "other scope", MIDNIGHT - 45_000L, 2_500_000L) + .at(CHECKOUT, CHECKOUT_1, "io.opentelemetry.java-http-client")); + found(traces, OTHER_SPAN_NAME, new StoredSpan(OTHER_SPAN_NAME, "other span name", MIDNIGHT - 40_000L, + 3_000_000L).call("GET /stock", CLIENT, STATUS_ERROR, PEER)); + found(traces, OTHER_PEER, new StoredSpan(OTHER_PEER, "other peer", MIDNIGHT - 35_000L, 3_500_000L) + .call(POST_STOCK, CLIENT, STATUS_ERROR, "payments")); + found(traces, OTHER_KIND, new StoredSpan(OTHER_KIND, "other kind", MIDNIGHT - 30_000L, 4_000_000L) + .call(POST_STOCK, INTERNAL, STATUS_ERROR, PEER)); + found(traces, OTHER_STATUS, new StoredSpan(OTHER_STATUS, "other status", MIDNIGHT - 25_000L, 4_500_000L) + .call(POST_STOCK, CLIENT, STATUS_OK, PEER)); + found(traces, BELOW_MIN_DURATION, new StoredSpan(BELOW_MIN_DURATION, "below the minimum duration", + MIDNIGHT - 20_000L, MIN_DURATION_NS - 1)); + found(traces, ABOVE_MAX_DURATION, new StoredSpan(ABOVE_MAX_DURATION, "above the maximum duration", + MIDNIGHT - 15_000L, MAX_DURATION_NS + 1)); + found(traces, OTHER_RESOURCE_ATTRIBUTE, new StoredSpan(OTHER_RESOURCE_ATTRIBUTE, "other environment", + MIDNIGHT + 10_000L, 5_000_000L) + .attributes("staging", "acme")); + found(traces, OTHER_SPAN_ATTRIBUTE, new StoredSpan(OTHER_SPAN_ATTRIBUTE, "other tenant", MIDNIGHT + 20_000L, + 5_500_000L).attributes("production", "globex")); + found(traces, TENANT_OF_RESOURCE, new StoredSpan(TENANT_OF_RESOURCE, "tenant of the resource", + MIDNIGHT + 30_000L, 6_000_000L).tenantOfResource()); + found(traces, LAST_MATCH, new StoredSpan(LAST_MATCH, "last match", END, MAX_DURATION_NS).childOf("last root"), + new StoredSpan(LAST_MATCH, "last root", START - 30_000L, 125_000_000_000L) + .at(FRONTEND, "frontend-1", "io.opentelemetry.servlet-5.0") + .call("GET /checkout", SERVER, STATUS_ERROR, "")); + found(traces, BEFORE_START, new StoredSpan(BEFORE_START, "before the start", START - 1, 7_000_000L)); + found(traces, AFTER_END, new StoredSpan(AFTER_END, "after the end", END + 1, 7_500_000L)); + return traces; + } + + /** + * @param spans the span the trace is found by first + */ + private static void found(final Map> traces, final String traceId, + final StoredSpan... spans) { + MATCHED_BY.put(traceId, spans[0]); + traces.put(traceId, Arrays.asList(spans)); + } + + /** A span matching every condition unless told otherwise. */ + private static final class StoredSpan { + private final String traceId; + private final String label; + private final long start; + private final long duration; + private String parentLabel; + private String service = CHECKOUT; + private String instance = CHECKOUT_1; + private String scope = OKHTTP; + private String name = POST_STOCK; + private int kind = CLIENT; + private int status = STATUS_ERROR; + private String peer = PEER; + private String environment = "production"; + private String tenant = "acme"; + private String tenantScope = OTLPSpanRecord.SPAN_TAG_PREFIX; + + private StoredSpan(final String traceId, final String label, final long start, final long duration) { + this.traceId = traceId; + this.label = label; + this.start = start; + this.duration = duration; + } + + private StoredSpan childOf(final String parentLabel) { + this.parentLabel = parentLabel; + return this; + } + + private StoredSpan at(final String service, final String instance, final String scope) { + this.service = service; + this.instance = instance; + this.scope = scope; + return this; + } + + private StoredSpan call(final String name, final int kind, final int status, final String peer) { + this.name = name; + this.kind = kind; + this.status = status; + this.peer = peer; + return this; + } + + private StoredSpan attributes(final String environment, final String tenant) { + this.environment = environment; + this.tenant = tenant; + return this; + } + + private StoredSpan tenantOfResource() { + this.tenantScope = OTLPSpanRecord.RESOURCE_TAG_PREFIX; + return this; + } + + /** The record the OTLP receiver stores for the span. */ + private OTLPSpanRecord record() { + final OTLPSpanRecord record = new OTLPSpanRecord(); + record.setTraceId(traceId); + record.setSpanId(spanId(label)); + record.setParentSpanId(parentLabel == null ? "" : spanId(parentLabel)); + record.setServiceName(service); + record.setServiceInstance(instance); + record.setScopeName(scope); + record.setName(name); + record.setKind(kind); + record.setStatusCode(status); + record.setPeerService(peer); + record.setStartTime(start); + record.setDuration(duration); + record.setTags(Arrays.asList( + OTLPSpanRecord.RESOURCE_TAG_PREFIX + "service.name=" + service, + OTLPSpanRecord.RESOURCE_TAG_PREFIX + ENVIRONMENT + "=" + environment, + tenantScope + TENANT + "=" + tenant)); + record.setDataBinary(label.getBytes(StandardCharsets.UTF_8)); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(start)); + return record; + } + + /** The span as the DAO serves it. */ + private String render() { + return label + " of " + service + ", " + Source.OTLP; + } + } + + /** + * Writes the older trace of the {@link #RANKED} service and {@link #RANKED_OTHERS} later single-span traces + * between its first span and its later one, placed on shards as the server routes them, and checks the placement. + * + * @param crossesMidnight whether the later span is in the next daily index, where its shard does not matter + */ + private RankedCase writeRanked(final long first, final long later, + final boolean crossesMidnight) throws Exception { + final String olderTrace = String.format("%032x", (first / 1000) << 16); + final OTLPSpanRecord firstSpan = rankedSpan(olderTrace, "000000000000d001", first); + support.writeRecords(spans, BUILDER, Collections.singletonList(firstSpan)); + final int shard = support.shardOf(spans, BUILDER, firstSpan); + final boolean customRouting = customRouting(); + + // The next daily index does not exist yet, and custom routing places the span with the first one. + final OTLPSpanRecord laterSpan = crossesMidnight || customRouting + ? rankedSpan(olderTrace, "000000000000d100", later) + : placed(i -> rankedSpan(olderTrace, String.format("%016x", 0xd100L + i), later), 1, + placedOn -> placedOn != shard).get(0); + final List onTheShard = placed( + i -> rankedSpan(String.format("%032x", ((first / 1000) << 16) + 0x100 + i), "000000000000e001", first), + RANKED_OTHERS, placedOn -> placedOn == shard); + final List others = new ArrayList<>(); + for (int i = 0; i < onTheShard.size(); i++) { + final OTLPSpanRecord candidate = onTheShard.get(i); + others.add(rankedSpan(candidate.getTraceId(), candidate.getSpanId(), first + (i + 1) * 1000L)); + } + final List records = new ArrayList<>(); + records.add(laterSpan); + records.addAll(others); + support.writeRecords(spans, BUILDER, records); + + assertEquals(shard, support.shardHolding(spans, firstSpan)); + if (!crossesMidnight) { + assertEquals(customRouting, support.shardHolding(spans, laterSpan) == shard, + "the later span shares the first span's shard"); + } + for (final OTLPSpanRecord other : others) { + assertEquals(shard, support.shardHolding(spans, other), other.getTraceId()); + } + return new RankedCase(Arrays.asList(firstSpan, laterSpan), others); + } + + private boolean customRouting() { + return mode == ElasticSearchQueryITSupport.Mode.LOGIC_SHARDING_AND_CUSTOM_ROUTING; + } + + /** The older trace and the later traces of a ranking case, the later ones in start order. */ + private static final class RankedCase { + private final List older; + private final List others; + + private RankedCase(final List older, final List others) { + this.older = older; + this.others = others; + } + + /** The {@link #RANKED_LIMIT} newest of the later traces, newest first. */ + private List> exact() { + final List> latest = new ArrayList<>(); + for (int i = others.size() - 1; i >= others.size() - RANKED_LIMIT; i--) { + latest.add(rendered(Collections.singletonList(others.get(i)))); + } + return latest; + } + + /** The older trace first, ranked by its later span, then the newest of the later traces. */ + private List> approximate() { + final List> kept = new ArrayList<>(); + kept.add(rendered(older)); + for (int i = others.size() - 1; i > others.size() - RANKED_LIMIT; i--) { + kept.add(rendered(Collections.singletonList(others.get(i)))); + } + return kept; + } + + /** The trace's spans as the DAO serves them, in label order. */ + private static List rendered(final List trace) { + return trace.stream() + .map(span -> new String(span.getDataBinary(), StandardCharsets.UTF_8) + " of " + RANKED + ", " + + Source.OTLP) + .sorted() + .collect(Collectors.toList()); + } + } + + /** The latest {@link #RANKED_LIMIT} traces of the {@link #RANKED} service from a second before the first span. */ + private List> searchRanked(final long first, final long end) { + final OTLPTraceQueryCondition condition = new OTLPTraceQueryCondition(); + condition.setServiceName(RANKED); + condition.setQueryDuration(duration(first - 1_000L, end)); + condition.setQueryOrder(QueryOrder.BY_START_TIME); + condition.setLimit(RANKED_LIMIT); + final List> traces = new ArrayList<>(); + for (final List trace : dao.queryTraces(condition)) { + traces.add(render(trace)); + } + return traces; + } + + /** + * The first {@code count} candidates the server routes to a shard the predicate takes, asked of the server + * before they are written. + */ + private List placed(final IntFunction candidates, final int count, + final IntPredicate shard) throws Exception { + final List placed = new ArrayList<>(); + for (int i = 0; placed.size() < count; i++) { + assertTrue(i < 10_000, "no candidate placed as asked after " + i); + final OTLPSpanRecord candidate = candidates.apply(i); + if (shard.test(support.shardOf(spans, BUILDER, candidate))) { + placed.add(candidate); + } + } + return placed; + } + + /** A single-span trace's span of the {@link #RANKED} service, labelled with its trace and span id. */ + private static OTLPSpanRecord rankedSpan(final String traceId, final String spanId, final long start) { + final OTLPSpanRecord record = new OTLPSpanRecord(); + record.setTraceId(traceId); + record.setSpanId(spanId); + record.setParentSpanId(""); + record.setServiceName(RANKED); + record.setServiceInstance(RANKED + "-1"); + record.setScopeName(OKHTTP); + record.setName(POST_STOCK); + record.setKind(SERVER); + record.setStatusCode(STATUS_UNSET); + record.setPeerService(""); + record.setStartTime(start); + record.setDuration(1_000_000L); + record.setTags(Collections.singletonList(OTLPSpanRecord.RESOURCE_TAG_PREFIX + "service.name=" + RANKED)); + record.setDataBinary((traceId + "/" + spanId).getBytes(StandardCharsets.UTF_8)); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(start)); + return record; + } + + /** A span id derived from the span's label, unique within the fixture. */ + private static String spanId(final String label) { + return String.format("%016x", label.hashCode() & 0xffffffffL); + } + + private static OTLPServiceTraffic service(final String name, final String layer, final long minute) { + final OTLPServiceTraffic traffic = new OTLPServiceTraffic(); + traffic.setServiceName(name); + traffic.setLayer(layer); + traffic.setTimeBucket(minute); + return traffic; + } + + private static OTLPServiceSpanTraffic spanName(final String service, final String name, final long minute) { + final OTLPServiceSpanTraffic traffic = new OTLPServiceSpanTraffic(); + traffic.setServiceName(service); + traffic.setSpanName(name); + traffic.setTimeBucket(minute); + return traffic; + } + + private static OTLPServiceRelationTraffic peer(final String service, final String peer, final long minute) { + final OTLPServiceRelationTraffic traffic = new OTLPServiceRelationTraffic(); + traffic.setServiceName(service); + traffic.setPeerService(peer); + traffic.setTimeBucket(minute); + return traffic; + } + + /** The duration the TraceQL API passes for a range read by the second. */ + private static Duration duration(final long start, final long end) { + final Duration duration = new Duration(); + duration.setStep(Step.SECOND); + duration.setStart(new DateTime(start).toString("yyyy-MM-dd HHmmss")); + duration.setEnd(new DateTime(end).toString("yyyy-MM-dd HHmmss")); + return duration; + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch9ShardedIT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch9ShardedIT.java deleted file mode 100644 index 0f91b43c9b1d..000000000000 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch9ShardedIT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin; - -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Tag("slow") -@Tag("elastic-it") -@Testcontainers -public class ZipkinQueryEsDAOElasticsearch9ShardedIT extends ZipkinQueryEsDAOITBase { - @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("9.3.0"); - - public ZipkinQueryEsDAOElasticsearch9ShardedIT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.LOGIC_SHARDING_AND_CUSTOM_ROUTING); - } -} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOITBase.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOITBase.java index 6edbf4eae719..20e069cb57db 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOITBase.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOITBase.java @@ -27,8 +27,9 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.IntFunction; +import java.util.function.IntPredicate; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.query.enumeration.Step; import org.apache.skywalking.oap.server.core.query.input.Duration; @@ -53,11 +54,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.testcontainers.shaded.org.awaitility.Awaitility.await; /** * Reads Zipkin data the way the Zipkin query API does, through {@link ZipkinQueryEsDAO}, from what OAP's own write - * path stored in Elasticsearch or OpenSearch. A subclass names the server and the storage mode. + * path stored in Elasticsearch or OpenSearch. A nested class of a server's IT names the server and the storage mode. * *

The search range spans a midnight, so it reads two daily indices. Each search criterion is tried alone and * with all the others. For every criterion a near-miss trace matches all the others but that one, the matching traces @@ -70,7 +70,7 @@ * {@code QueryRequest}), by their first span. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) -abstract class ZipkinQueryEsDAOITBase { +public abstract class ZipkinQueryEsDAOITBase { /** A fixed local midnight, so the daily indices the spans are written to are known. */ private static final long MIDNIGHT = new DateTime(2026, 1, 15, 0, 0).getMillis(); /** The main search range: a minute either side of midnight. */ @@ -80,6 +80,15 @@ abstract class ZipkinQueryEsDAOITBase { /** Ten to five minutes before midnight, apart from the main range. */ private static final Range BEFORE_MIDNIGHT = Range.ending(MIDNIGHT - 301_000L, 299_000L); + private static final long HOUR = 3_600_000L; + private static final long DAY = 24 * HOUR; + private static final ZipkinSpanRecord.Builder BUILDER = new ZipkinSpanRecord.Builder(); + /** The service of the ranking cases, whose spans are days after the other fixtures' and in their own indices. */ + private static final String RANKED = "ranked"; + private static final int RANKED_LIMIT = 2; + /** More than the traces a shard answers a terms aggregation of {@link #RANKED_LIMIT} with by default. */ + private static final int RANKED_OTHERS = RANKED_LIMIT * 3 / 2 + 10 + 3; + private static final String BACKEND = "backend"; private static final String CANARY = "backend-canary"; private static final String FRONTEND = "frontend"; @@ -143,16 +152,17 @@ abstract class ZipkinQueryEsDAOITBase { private final ElasticSearchQueryITSupport.Mode mode; private ElasticSearchQueryITSupport support; private ZipkinQueryEsDAO dao; + private Model spans; - ZipkinQueryEsDAOITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { + protected ZipkinQueryEsDAOITBase(final ElasticsearchContainer server, final ElasticSearchQueryITSupport.Mode mode) { this.server = server; this.mode = mode; } @BeforeAll public void writeSpans() throws Exception { - support = ElasticSearchQueryITSupport.connect(server, mode); - final Model spans = support.installRecord(ZipkinSpanRecord.class); + support = ElasticSearchQueryITSupport.connect(server, mode, getClass()); + spans = support.installRecord(ZipkinSpanRecord.class); final Model services = support.installMetrics(ZipkinServiceTraffic.class); final Model remoteServices = support.installMetrics(ZipkinServiceRelationTraffic.class); final Model spanNames = support.installMetrics(ZipkinServiceSpanTraffic.class); @@ -171,10 +181,6 @@ public void writeSpans() throws Exception { support.writeMetrics(spanNames, new ZipkinServiceSpanTraffic.Builder(), spanTraffic()); dao = new ZipkinQueryEsDAO(support.getClient()); - // A write is searchable after its index's next refresh, every 30 s as OAP configures the indices. - await().atMost(90, TimeUnit.SECONDS) - .pollInterval(1, TimeUnit.SECONDS) - .untilAsserted(this::assertEverythingSearchable); } @AfterAll @@ -270,6 +276,36 @@ public void keepsTheTracesThatStartedLast(final int limit) { canonical(search(BEFORE_MIDNIGHT, request -> request.serviceName(SCHEDULER), limit))); } + /** + * A trace ranks by its first matching span, but the search's terms aggregation lets each shard rank a trace by the + * spans it holds and answer with its first {@code shard_size} traces, an approximation accepted for its cost (see + * {@link ZipkinQueryEsDAO#getTraces(QueryRequest, Duration)}). The older trace's first span shares its shard with + * more than {@code shard_size} later traces, and its later span, newer than all of them, is routed by its document + * id to another shard, which ranks the trace first: the search keeps the older trace and the newest of the others. + * Custom routing keeps a trace's spans on one shard of a daily index, and there the search is exact. The JDBC + * storages, which rank over every table, keep the exact traces in both cases. + */ + @Test + public void keepsAnOlderTraceWithALaterSpanOnAnotherShardOnlyWithDefaultRouting() throws Exception { + final long first = MIDNIGHT + 2 * DAY + 12 * HOUR; + final RankedCase ranked = writeRanked(first, first + 30_000L, false); + assertEquals(customRouting() ? ranked.exact() : ranked.approximate(), + searchRanked(first, first + 30_000L)); + } + + /** + * The older trace's first span is in the earlier daily index, on a shard with more than {@code shard_size} later + * traces, and its later span is in the next daily index, which ranks the trace first; routing keeps a trace on + * one shard of a daily index only. So with either routing the search keeps the older trace and the newest of the + * others, where the JDBC storages keep the two newest others. + */ + @Test + public void keepsAnOlderTraceThatCrossesMidnightByItsLaterSpan() throws Exception { + final long midnight = MIDNIGHT + 4 * DAY; + final RankedCase ranked = writeRanked(midnight - 50_000L, midnight + 10_000L, true); + assertEquals(ranked.approximate(), searchRanked(midnight - 50_000L, midnight + 10_000L)); + } + /** What a search can filter by, each with the trace in the main range that matches every other criterion. */ enum Criterion { TIME_RANGE_ONLY(request -> { }, null), @@ -300,22 +336,6 @@ private List> search(final Range range, final Consumer> trace : TRACES.entrySet()) { - assertEquals(trace.getValue().size(), dao.getTrace(trace.getKey(), null).size(), trace.getKey()); - } - assertTrue(dao.getServiceNames().containsAll(SERVICES), "searchable services"); - for (final String service : SERVICES) { - assertTrue(dao.getRemoteServiceNames(service).containsAll(REMOTE_SERVICES.get(service)), service); - assertTrue(dao.getSpanNames(service).containsAll(SPAN_NAMES.get(service)), service); - } - } - private static Map> fixture() { final Map> traces = new LinkedHashMap<>(); traces.put(FIRST_MATCH, Arrays.asList( @@ -492,6 +512,110 @@ private static List sorted(final List names) { return sorted; } + /** + * Writes the older trace of the {@link #RANKED} service and {@link #RANKED_OTHERS} later single-span traces + * between its first span and its later one, placed on shards as the server routes them, and checks the placement. + * + * @param crossesMidnight whether the later span is in the next daily index, where its shard does not matter + */ + private RankedCase writeRanked(final long first, final long later, + final boolean crossesMidnight) throws Exception { + final String olderTrace = String.format("%032x", (first / 1000) << 16); + final Span firstSpan = rankedSpan(olderTrace, "000000000000d001", first); + support.writeRecords(spans, BUILDER, Collections.singletonList(ZipkinSpanRecords.of(firstSpan))); + final int shard = support.shardOf(spans, BUILDER, ZipkinSpanRecords.of(firstSpan)); + final boolean customRouting = customRouting(); + + // The next daily index does not exist yet, and custom routing places the span with the first one. + final Span laterSpan = crossesMidnight || customRouting + ? rankedSpan(olderTrace, "000000000000d100", later) + : placed(i -> rankedSpan(olderTrace, String.format("%016x", 0xd100L + i), later), 1, + placedOn -> placedOn != shard).get(0); + final List others = new ArrayList<>(); + final List onTheShard = placed( + i -> rankedSpan(String.format("%032x", ((first / 1000) << 16) + 0x100 + i), "000000000000e001", first), + RANKED_OTHERS, placedOn -> placedOn == shard); + for (int i = 0; i < onTheShard.size(); i++) { + others.add(onTheShard.get(i).toBuilder().timestamp((first + (i + 1) * 1000L) * 1000L).build()); + } + final List records = new ArrayList<>(); + records.add(ZipkinSpanRecords.of(laterSpan)); + others.forEach(span -> records.add(ZipkinSpanRecords.of(span))); + support.writeRecords(spans, BUILDER, records); + + assertEquals(shard, support.shardHolding(spans, ZipkinSpanRecords.of(firstSpan))); + if (!crossesMidnight) { + assertEquals(customRouting, support.shardHolding(spans, ZipkinSpanRecords.of(laterSpan)) == shard, + "the later span shares the first span's shard"); + } + for (final Span other : others) { + assertEquals(shard, support.shardHolding(spans, ZipkinSpanRecords.of(other)), other.traceId()); + } + return new RankedCase(Arrays.asList(firstSpan, laterSpan), others); + } + + private boolean customRouting() { + return mode == ElasticSearchQueryITSupport.Mode.LOGIC_SHARDING_AND_CUSTOM_ROUTING; + } + + /** The older trace and the later traces of a ranking case, the later ones in start order. */ + private static final class RankedCase { + private final List older; + private final List others; + + private RankedCase(final List older, final List others) { + this.older = older; + this.others = others; + } + + /** The {@link #RANKED_LIMIT} newest of the later traces. */ + private List> exact() { + final List> latest = new ArrayList<>(); + for (int i = others.size() - RANKED_LIMIT; i < others.size(); i++) { + latest.add(Collections.singletonList(others.get(i))); + } + return canonical(latest); + } + + /** The older trace, ranked by its later span, and the newest of the later traces after it. */ + private List> approximate() { + final List> kept = new ArrayList<>(); + kept.add(older); + for (int i = others.size() - RANKED_LIMIT + 1; i < others.size(); i++) { + kept.add(Collections.singletonList(others.get(i))); + } + return canonical(kept); + } + } + + /** The latest {@link #RANKED_LIMIT} traces of the {@link #RANKED} service from a second before the first span. */ + private List> searchRanked(final long first, final long end) { + return canonical(search(Range.ending(end, end - first + 1_000L), + request -> request.serviceName(RANKED), RANKED_LIMIT)); + } + + /** + * The first {@code count} candidates the server routes to a shard the predicate takes, asked of the server + * before they are written. + */ + private List placed(final IntFunction candidates, final int count, + final IntPredicate shard) throws Exception { + final List placed = new ArrayList<>(); + for (int i = 0; placed.size() < count; i++) { + assertTrue(i < 10_000, "no candidate placed as asked after " + i); + final Span candidate = candidates.apply(i); + if (shard.test(support.shardOf(spans, BUILDER, ZipkinSpanRecords.of(candidate)))) { + placed.add(candidate); + } + } + return placed; + } + + private static Span rankedSpan(final String traceId, final String spanId, final long timestampMillis) { + return span(traceId, spanId, null, Span.Kind.SERVER, "get /rank", RANKED, timestampMillis) + .duration(1_000L).build(); + } + /** A search's time range, from the Zipkin API's {@code endTs} and {@code lookback}. */ private static final class Range { private final long endTs; diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch3ShardedIT.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch3ShardedIT.java deleted file mode 100644 index 99d34994bb9d..000000000000 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch3ShardedIT.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin; - -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; - -@Tag("slow") -@Tag("elastic-it") -@Testcontainers -public class ZipkinQueryEsDAOOpenSearch3ShardedIT extends ZipkinQueryEsDAOITBase { - @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("3.0.0"); - - public ZipkinQueryEsDAOOpenSearch3ShardedIT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.LOGIC_SHARDING_AND_CUSTOM_ROUTING); - } -} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml index 886696cb5098..0108d44bcdbf 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/pom.xml @@ -70,5 +70,19 @@ ${project.version} test + + + org.apache.skywalking + oal-rt + ${project.version} + test + + + + org.apache.skywalking + zipkin-query-plugin + ${project.version} + test + diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/TableHelper.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/TableHelper.java index 84409c408dfe..5d41f128a3c0 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/TableHelper.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/TableHelper.java @@ -141,6 +141,16 @@ public List getTablesForRead(String modelName, long timeBucketStart, lon .collect(toList()); } + /** + * Similar to {@link #getTablesForRead(String, long, long)}, for the days from one instant through another, both + * included. A query duration reads through its end instant, the first instant after its last step, so a duration + * ending in a day's last step reads the next day's table too. + */ + public List getTablesBetweenTimestamps(String modelName, long startTimestamp, long endTimestamp) { + return getTablesForRead( + modelName, TimeBucket.getRecordTimeBucket(startTimestamp), TimeBucket.getRecordTimeBucket(endTimestamp)); + } + /** * Similar to {@link #getTablesForRead(String, long, long)}, but don't check the table existence. */ diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryDAO.java index 5455e71b7a91..554679163acf 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryDAO.java @@ -31,7 +31,6 @@ import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.aiagent.AIAgentSessionDataRecord; import org.apache.skywalking.oap.server.core.analysis.manual.aiagent.AIAgentSessionFlowRecord; -import org.apache.skywalking.oap.server.core.analysis.record.Record; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.storage.model.ColumnName; import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; @@ -94,8 +93,8 @@ public List queryRounds(final String serviceId, final boolean includeBody) { final List tables = duration == null ? tableHelper.getTablesWithinTTL(AIAgentSessionFlowRecord.INDEX_NAME) - : tableHelper.getTablesForRead( - AIAgentSessionFlowRecord.INDEX_NAME, duration.getStartTimeBucket(), duration.getEndTimeBucket()); + : tableHelper.getTablesBetweenTimestamps( + AIAgentSessionFlowRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp()); final List columns = new ArrayList<>(ROUND_COLUMNS); if (includeBody) { columns.add(AIAgentSessionFlowRecord.BODY); @@ -109,12 +108,11 @@ public List queryRounds(final String serviceId, .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); parameters.add(AIAgentSessionFlowRecord.INDEX_NAME); if (duration != null) { - sql.append(" and ").append(column(AIAgentSessionFlowRecord.INDEX_NAME, Record.TIME_BUCKET)) - .append(" >= ?"); - parameters.add(duration.getStartTimeBucketInSec()); - sql.append(" and ").append(column(AIAgentSessionFlowRecord.INDEX_NAME, Record.TIME_BUCKET)) - .append(" <= ?"); - parameters.add(duration.getEndTimeBucketInSec()); + final String timestamp = column(AIAgentSessionFlowRecord.INDEX_NAME, AIAgentSessionFlowRecord.TIMESTAMP); + sql.append(" and ").append(timestamp).append(" >= ?"); + parameters.add(duration.getStartTimestamp()); + sql.append(" and ").append(timestamp).append(" <= ?"); + parameters.add(duration.getEndTimestamp()); } sql.append(" and ").append(column(AIAgentSessionFlowRecord.INDEX_NAME, AIAgentSessionFlowRecord.SERVICE_ID)) .append(" = ?"); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryDAO.java index 031b60c41784..ae4f1a323467 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryDAO.java @@ -90,8 +90,8 @@ public Alarms getAlarm(Integer scopeId, String keyword, int limit, int from, return new Alarms(); } - final var tables = tableHelper.getTablesForRead( - AlarmRecord.INDEX_NAME, duration.getStartTimeBucket(), duration.getEndTimeBucket() + final var tables = tableHelper.getTablesBetweenTimestamps( + AlarmRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp() ); final var alarmMsgs = new ArrayList(); @@ -145,8 +145,8 @@ public Alarms queryAlarms(final AlarmQueryCondition condition, final int limit, } final List entityConstraints = resolveEntityFilters(condition.getEntities()); final Duration duration = condition.getDuration(); - final var tables = tableHelper.getTablesForRead( - AlarmRecord.INDEX_NAME, duration.getStartTimeBucket(), duration.getEndTimeBucket() + final var tables = tableHelper.getTablesBetweenTimestamps( + AlarmRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp() ); final var alarmMsgs = new ArrayList(); for (final var table : tables) { @@ -191,12 +191,6 @@ protected SQLAndParameters buildSQLForQueryAlarms(final AlarmQueryCondition cond final List tags = condition.getTags(); final var sql = new StringBuilder(); final var parameters = new ArrayList<>(); - long startTB = 0; - long endTB = 0; - if (nonNull(duration)) { - startTB = duration.getStartTimeBucketInSec(); - endTB = duration.getEndTimeBucketInSec(); - } sql.append("select * from ").append(table); final var timeBucket = TableHelper.getTimeBucket(table); final var tagTable = TableHelper.getTable(AlarmRecord.ADDITIONAL_TAG_TABLE, timeBucket); @@ -212,12 +206,7 @@ protected SQLAndParameters buildSQLForQueryAlarms(final AlarmQueryCondition cond sql.append(" where ") .append(table).append(".").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? "); parameters.add(AlarmRecord.INDEX_NAME); - if (startTB != 0 && endTB != 0) { - sql.append(" and ").append(table).append(".").append(AlarmRecord.TIME_BUCKET).append(" >= ?"); - parameters.add(startTB); - sql.append(" and ").append(table).append(".").append(AlarmRecord.TIME_BUCKET).append(" <= ?"); - parameters.add(endTB); - } + appendStartTimeWithin(duration, table, sql, parameters); if (!Strings.isNullOrEmpty(condition.getKeyword())) { sql.append(" and ").append(AlarmRecord.ALARM_MESSAGE).append(" like concat('%',?,'%') "); parameters.add(condition.getKeyword()); @@ -286,8 +275,9 @@ private Map getAlarmRecoveryRecord(List uuids = msgs.stream().map(AlarmMessage::getUuid).collect(toList()); - final var tables = tableHelper.getTablesForRead( - AlarmRecoveryRecord.INDEX_NAME, duration.getStartTimeBucket(), duration.getEndTimeBucket() + // A recovery is stored in the day of its alarm's start. + final var tables = tableHelper.getTablesBetweenTimestamps( + AlarmRecoveryRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp() ); for (final var table : tables) { final var sqlAndParameters = buildSQL4Recovery(uuids, table); @@ -309,12 +299,6 @@ protected SQLAndParameters buildSQL(Integer scopeId, String keyword, int limit, final var sql = new StringBuilder(); final var parameters = new ArrayList<>(); - long startTB = 0; - long endTB = 0; - if (nonNull(duration)) { - startTB = duration.getStartTimeBucketInSec(); - endTB = duration.getEndTimeBucketInSec(); - } sql.append("select * from ").append(table); /* * This is an AdditionalEntity feature, see: @@ -338,12 +322,7 @@ protected SQLAndParameters buildSQL(Integer scopeId, String keyword, int limit, sql.append(" and ").append(AlarmRecord.SCOPE).append(" = ?"); parameters.add(scopeId); } - if (startTB != 0 && endTB != 0) { - sql.append(" and ").append(table).append(".").append(AlarmRecord.TIME_BUCKET).append(" >= ?"); - parameters.add(startTB); - sql.append(" and ").append(table).append(".").append(AlarmRecord.TIME_BUCKET).append(" <= ?"); - parameters.add(endTB); - } + appendStartTimeWithin(duration, table, sql, parameters); if (!Strings.isNullOrEmpty(keyword)) { sql.append(" and ").append(AlarmRecord.ALARM_MESSAGE).append(" like concat('%',?,'%') "); @@ -362,6 +341,19 @@ protected SQLAndParameters buildSQL(Integer scopeId, String keyword, int limit, return new SQLAndParameters(sql.toString(), parameters); } + /** + * Keeps the alarms starting from the duration's start through its end instant, {@link Duration#getEndTimestamp()}. + */ + private static void appendStartTimeWithin(final Duration duration, final String table, + final StringBuilder sql, final List parameters) { + if (nonNull(duration)) { + sql.append(" and ").append(table).append(".").append(AlarmRecord.START_TIME).append(" >= ?"); + parameters.add(duration.getStartTimestamp()); + sql.append(" and ").append(table).append(".").append(AlarmRecord.START_TIME).append(" <= ?"); + parameters.add(duration.getEndTimestamp()); + } + } + private SQLAndParameters buildSQL4Recovery(List uuids, String table) { final var sql = new StringBuilder(); final var parameters = new ArrayList<>(); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBrowserLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBrowserLogQueryDAO.java index 012160bcb95c..54db0255fb89 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBrowserLogQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBrowserLogQueryDAO.java @@ -53,8 +53,8 @@ public BrowserErrorLogs queryBrowserErrorLogs(String serviceId, int from) { final var logs = new ArrayList(); final var tables = nonNull(duration) - ? tableHelper.getTablesForRead( - BrowserErrorLogRecord.INDEX_NAME, duration.getStartTimeBucket(), duration.getEndTimeBucket()) + ? tableHelper.getTablesBetweenTimestamps( + BrowserErrorLogRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp()) : tableHelper.getTablesWithinTTL(BrowserErrorLogRecord.INDEX_NAME); for (String table : tables) { @@ -99,16 +99,11 @@ protected SQLAndParameters buildSQL( .append(" where ").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? "); parameters.add(BrowserErrorLogRecord.INDEX_NAME); - long startSecondTB = 0, endSecondTB = 0; if (nonNull(duration)) { - startSecondTB = duration.getStartTimeBucketInSec(); - endSecondTB = duration.getEndTimeBucketInSec(); - } - if (startSecondTB != 0 && endSecondTB != 0) { - sql.append(" and ").append(BrowserErrorLogRecord.TIME_BUCKET).append(" >= ?"); - parameters.add(startSecondTB); - sql.append(" and ").append(BrowserErrorLogRecord.TIME_BUCKET).append(" <= ?"); - parameters.add(endSecondTB); + sql.append(" and ").append(BrowserErrorLogRecord.TIMESTAMP).append(" >= ?"); + parameters.add(duration.getStartTimestamp()); + sql.append(" and ").append(BrowserErrorLogRecord.TIMESTAMP).append(" <= ?"); + parameters.add(duration.getEndTimestamp()); } if (StringUtil.isNotEmpty(serviceId)) { diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingScheduleDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingScheduleDAO.java index 3520fad1bdb1..c6ffd2b14792 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingScheduleDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingScheduleDAO.java @@ -32,6 +32,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; @RequiredArgsConstructor @@ -54,6 +55,8 @@ public List querySchedules(String taskId) { ) ); } + // The query service lists the schedules as read: latest start first, whichever day table each is in. + schedules.sort(Comparator.comparingLong(EBPFProfilingSchedule::getStartTime).reversed()); return schedules; } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEventQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEventQueryDAO.java index 34c64035352f..1bca54844b1e 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEventQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEventQueryDAO.java @@ -214,13 +214,14 @@ protected Tuple2, Stream> buildQuery(final EventQueryCond final Duration time = condition.getTime(); if (time != null) { - // The window is [start, end): end is the first millisecond after the last step of the duration. + // Both bounds are inclusive. The end is the first millisecond after the last step of the duration, so an + // event on it is also read by the next window. if (time.getStartTimestamp() > 0) { conditions.add(Event.TIMESTAMP + ">=?"); parameters.add(time.getStartTimestamp()); } if (time.getEndTimestamp() > 0) { - conditions.add(Event.TIMESTAMP + " tables; if (nonNull(duration)) { - tables = tableHelper.getTablesForRead( + tables = tableHelper.getTablesBetweenTimestamps( GenAIEvaluationRecord.INDEX_NAME, - duration.getStartTimeBucket(), - duration.getEndTimeBucket() + duration.getStartTimestamp(), + duration.getEndTimestamp() ); } else { tables = tableHelper.getTablesWithinTTL(GenAIEvaluationRecord.INDEX_NAME); @@ -175,13 +175,6 @@ protected SQLAndParameters buildSQL(final String serviceId, final int limit, final Duration duration, final String table) { - long startSecondTB = 0; - long endSecondTB = 0; - if (nonNull(duration)) { - startSecondTB = duration.getStartTimeBucketInSec(); - endSecondTB = duration.getEndTimeBucketInSec(); - } - final StringBuilder sql = new StringBuilder("select "); final List parameters = new ArrayList<>(10); sql.append(selectColumns()) @@ -192,11 +185,11 @@ protected SQLAndParameters buildSQL(final String serviceId, .append(" = ?"); parameters.add(GenAIEvaluationRecord.INDEX_NAME); - if (startSecondTB != 0 && endSecondTB != 0) { - sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TIME_BUCKET)).append(" >= ?"); - parameters.add(startSecondTB); - sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.TIME_BUCKET)).append(" <= ?"); - parameters.add(endSecondTB); + if (nonNull(duration)) { + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.EVALUATION_TIME)).append(" >= ?"); + parameters.add(duration.getStartTimestamp()); + sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.EVALUATION_TIME)).append(" <= ?"); + parameters.add(duration.getEndTimestamp()); } if (StringUtil.isNotEmpty(serviceId)) { sql.append(" and ").append(storageColumn(GenAIEvaluationRecord.SERVICE_ID)).append(" = ?"); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryDAO.java index fabbad866f09..b465e228142e 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryDAO.java @@ -104,10 +104,10 @@ public Logs queryLogs(String serviceId, } List tables; if (nonNull(duration)) { - tables = tableHelper.getTablesForRead( + tables = tableHelper.getTablesBetweenTimestamps( LogRecord.INDEX_NAME, - duration.getStartTimeBucket(), - duration.getEndTimeBucket() + duration.getStartTimestamp(), + duration.getEndTimestamp() ); } else { tables = tableHelper.getTablesWithinTTL(LogRecord.INDEX_NAME); @@ -179,12 +179,6 @@ protected SQLAndParameters buildSQL( final List keywordsOfContent, final List excludingKeywordsOfContent, final String table) { - long startSecondTB = 0; - long endSecondTB = 0; - if (nonNull(duration)) { - startSecondTB = duration.getStartTimeBucketInSec(); - endSecondTB = duration.getEndTimeBucketInSec(); - } StringBuilder sql = new StringBuilder(); List parameters = new ArrayList<>(10); @@ -206,11 +200,11 @@ protected SQLAndParameters buildSQL( sql.append(" where "); sql.append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); parameters.add(LogRecord.INDEX_NAME); - if (startSecondTB != 0 && endSecondTB != 0) { - sql.append(" and ").append(table).append(".").append(AbstractLogRecord.TIME_BUCKET).append(" >= ?"); - parameters.add(startSecondTB); - sql.append(" and ").append(table).append(".").append(AbstractLogRecord.TIME_BUCKET).append(" <= ?"); - parameters.add(endSecondTB); + if (nonNull(duration)) { + sql.append(" and ").append(table).append(".").append(TIMESTAMP).append(" >= ?"); + parameters.add(duration.getStartTimestamp()); + sql.append(" and ").append(table).append(".").append(TIMESTAMP).append(" <= ?"); + parameters.add(duration.getEndTimestamp()); } if (StringUtil.isNotEmpty(serviceId)) { diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryDAO.java index f5a2d1fab182..396eb8696167 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryDAO.java @@ -27,7 +27,6 @@ import lombok.SneakyThrows; import org.apache.commons.lang3.StringUtils; import org.apache.skywalking.oap.server.core.analysis.IDManager; -import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessDetectType; @@ -141,8 +140,8 @@ protected SQLAndParameters buildSQLForListInstances(String serviceId, Duration d .append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); parameters.add(InstanceTraffic.INDEX_NAME); if (duration != null) { - final var startMinuteTimeBucket = TimeBucket.getMinuteTimeBucket(duration.getStartTimestamp()); - final var endMinuteTimeBucket = TimeBucket.getMinuteTimeBucket(duration.getEndTimestamp()); + final var startMinuteTimeBucket = duration.getStartTimeBucketInMin(); + final var endMinuteTimeBucket = duration.getEndTimeBucketInMin(); sql.append(" and ").append(InstanceTraffic.LAST_PING_TIME_BUCKET).append(" >= ?"); parameters.add(startMinuteTimeBucket); sql.append(" and ").append(InstanceTraffic.TIME_BUCKET).append(" <= ?"); @@ -227,8 +226,8 @@ public List findEndpoint(String keyword, String serviceId, int limit, condition.add(keyword); } if (duration != null) { - final var startMinuteTimeBucket = TimeBucket.getMinuteTimeBucket(duration.getStartTimestamp()); - final var endMinuteTimeBucket = TimeBucket.getMinuteTimeBucket(duration.getEndTimestamp()); + final var startMinuteTimeBucket = duration.getStartTimeBucketInMin(); + final var endMinuteTimeBucket = duration.getEndTimeBucketInMin(); sql.append(" and ").append(EndpointTraffic.LAST_PING_TIME_BUCKET).append(" >= ?"); condition.add(startMinuteTimeBucket); sql.append(" and ").append(EndpointTraffic.TIME_BUCKET).append(" <= ?"); @@ -321,8 +320,9 @@ public List listProcesses(String serviceInstanceId, Duration duration, } protected SQLAndParameters buildSQLForListProcesses(String serviceInstanceId, Duration duration, boolean includeVirtual, String table) { - final var lastPingStartTimeBucket = duration.getStartTimeBucket(); - final var lastPingEndTimeBucket = duration.getEndTimeBucket(); + // Traffic is stored by minute, whatever the step of the duration. + final var lastPingStartTimeBucket = duration.getStartTimeBucketInMin(); + final var lastPingEndTimeBucket = duration.getEndTimeBucketInMin(); final var sql = new StringBuilder(); final var condition = new ArrayList<>(); sql.append("select * from ").append(table); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsDAO.java index db036e86c2d5..c3d5293d9a8d 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsDAO.java @@ -19,6 +19,9 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; import lombok.RequiredArgsConstructor; +import org.apache.skywalking.oap.server.core.UnexpectedException; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; import org.apache.skywalking.oap.server.core.storage.SessionCacheCallback; @@ -32,7 +35,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Calendar; import java.util.List; +import java.util.concurrent.TimeUnit; import static java.util.stream.Collectors.toList; @@ -40,6 +45,7 @@ public class JDBCMetricsDAO extends JDBCSQLExecutor implements IMetricsDAO { private final JDBCClient jdbcClient; private final StorageBuilder storageBuilder; + private volatile FirstKeptDay firstKeptDay; @Override public List multiGet(Model model, List metrics) throws Exception { @@ -61,4 +67,70 @@ public SQLExecutor prepareBatchInsert(Model model, Metrics metrics, SessionCache public SQLExecutor prepareBatchUpdate(Model model, Metrics metrics, SessionCacheCallback callback) { return getUpdateExecutor(model, metrics, metrics.getTimeBucket(), storageBuilder, callback); } + + /** + * Expired once the metric's day table is past the TTL: {@link JDBCHistoryDeleteDAO} drops whole day tables, + * keeping those from the day {@code ttl - 1} days before now, so a metric of an earlier day is gone however late in + * that day it is. The metrics workers ask this for every metric they receive, so the metric's day is read off its + * time bucket, and the first kept day is reused for as long as it stays the same. + */ + @Override + public boolean isExpiredCache(Model model, Metrics cachedValue, long currentTimeMillis, int ttl) { + return dayOf(cachedValue.getTimeBucket(), model.getDownsampling()) < firstKeptDay(currentTimeMillis, ttl); + } + + private long firstKeptDay(final long currentTimeMillis, final int ttl) { + FirstKeptDay kept = firstKeptDay; + if (kept == null || !kept.holdsFor(currentTimeMillis, ttl)) { + kept = new FirstKeptDay(currentTimeMillis, ttl); + firstKeptDay = kept; + } + return kept.day; + } + + private static long dayOf(final long timeBucket, final DownSampling downSampling) { + switch (downSampling) { + case Second: + return timeBucket / 1_000_000; + case Minute: + return timeBucket / 10_000; + case Hour: + return timeBucket / 100; + case Day: + return timeBucket; + default: + throw new UnexpectedException("Unknown downsampling value " + downSampling); + } + } + + /** + * The first day table the TTL keeps, counted as {@link JDBCHistoryDeleteDAO} counts it: the day of {@code ttl - 1} + * days of elapsed time before now. It stays the same from the instant that point reaches the start of the day until + * it reaches the start of the next one. + */ + private static final class FirstKeptDay { + private final int ttl; + private final long from; + private final long until; + private final long day; + + private FirstKeptDay(final long currentTimeMillis, final int ttl) { + final long kept = TimeUnit.DAYS.toMillis(ttl - 1); + final Calendar dayStart = Calendar.getInstance(); + dayStart.setTimeInMillis(currentTimeMillis - kept); + dayStart.set(Calendar.HOUR_OF_DAY, 0); + dayStart.set(Calendar.MINUTE, 0); + dayStart.set(Calendar.SECOND, 0); + dayStart.set(Calendar.MILLISECOND, 0); + this.ttl = ttl; + this.day = TimeBucket.getTimeBucket(currentTimeMillis - kept, DownSampling.Day); + this.from = dayStart.getTimeInMillis() + kept; + dayStart.add(Calendar.DAY_OF_MONTH, 1); + this.until = dayStart.getTimeInMillis() + kept; + } + + private boolean holdsFor(final long currentTimeMillis, final int ttl) { + return this.ttl == ttl && currentTimeMillis >= from && currentTimeMillis < until; + } + } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCNetworkAddressAliasDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCNetworkAddressAliasDAO.java index e12be5eb25e8..27b8e571f83e 100755 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCNetworkAddressAliasDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCNetworkAddressAliasDAO.java @@ -51,7 +51,7 @@ public List loadLastUpdate(long lastUpdateTime) { .append("select * from ").append(table) .append(" where ") .append(JDBCTableInstaller.TABLE_COLUMN).append(" = ? ") - .append(" and ").append(NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET).append(">?"); + .append(" and ").append(NetworkAddressAlias.LAST_UPDATE_TIME_BUCKET).append(">=?"); results.addAll( jdbcClient.executeQuery( diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryDAO.java index 2ecb2171fc5a..3efb6fa98a4b 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryDAO.java @@ -24,13 +24,11 @@ import java.util.ArrayList; import java.util.Base64; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -53,19 +51,15 @@ /** * Reads {@code otlp_span} on the JDBC storages. A tag condition is one inner join on the {@code otlp_span_tag} - * additional table, the way the segment and Zipkin DAOs search their tag tables; the spans of the selected traces - * are then fetched and grouped by trace in Java. + * additional table, the way the segment and Zipkin DAOs search their tag tables. A search ranks the traces over the + * day tables of its range in one query ({@link TraceRankingAcrossTables}); the spans of the selected traces are then + * fetched and grouped by trace in Java. */ @RequiredArgsConstructor public class JDBCOTLPTraceQueryDAO implements IOTLPTraceQueryDAO { private static final int NAME_QUERY_MAX_SIZE = Integer.MAX_VALUE; private static final OTLPServiceTraffic.Builder SERVICE_TRAFFIC_BUILDER = new OTLPServiceTraffic.Builder(); - /** - * The alias of the aggregate a search ranks traces by, {@code min(start_time)} or {@code max(duration)}. - */ - private static final String RANK_COLUMN = "rank_value"; - private final JDBCClient jdbcClient; private final TableHelper tableHelper; @@ -160,100 +154,107 @@ public List queryTraceById(final String traceId, @Nullable final Du } /** - * The day tables a time range covers, every table within the TTL without one. + * The day tables a time range covers through its end instant, which the reads count, every table within the TTL + * without one. */ private List tablesFor(@Nullable final Duration duration) { return duration == null ? tableHelper.getTablesWithinTTL(OTLPSpanRecord.INDEX_NAME) - : tableHelper.getTablesForRead( - OTLPSpanRecord.INDEX_NAME, duration.getStartTimeBucket(), duration.getEndTimeBucket()); + : tableHelper.getTablesBetweenTimestamps( + OTLPSpanRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp()); } @Override @SneakyThrows public List> queryTraces(final OTLPTraceQueryCondition condition) { - final Duration duration = condition.getQueryDuration(); - final List tables = tablesFor(duration); + final List tables = tablesFor(condition.getQueryDuration()); + if (tables.isEmpty() || condition.getLimit() <= 0) { + return new ArrayList<>(); + } final boolean byDuration = condition.getQueryOrder() == QueryOrder.BY_DURATION; - final String orderColumn = byDuration - ? "max(" + OTLPSpanRecord.DURATION + ")" : "min(" + OTLPSpanRecord.START_TIME + ")"; - // Every day table answers its own top `limit`; a trace that crosses midnight is in two of them, so the - // candidates are merged on the rank value and ranked once more before the limit applies to the whole range. - final Map rankByTraceId = new HashMap<>(); + final String aggregate = byDuration ? "max" : "min"; + final String rank = aggregate + "(" + (byDuration ? OTLPSpanRecord.DURATION : OTLPSpanRecord.START_TIME) + ")"; + final List parameters = new ArrayList<>(); + final List tableRankings = new ArrayList<>(tables.size()); for (final String table : tables) { - final StringBuilder sql = new StringBuilder(); - final List parameters = new ArrayList<>(); - final List tags = condition.getTags() == null ? Collections.emptyList() : condition.getTags(); - sql.append("select ").append(table).append(".").append(OTLPSpanRecord.TRACE_ID).append(", ") - .append(orderColumn).append(" as ").append(RANK_COLUMN).append(" from ").append(table); - final long timeBucket = TableHelper.getTimeBucket(table); - final String tagTable = TableHelper.getTable(OTLPSpanRecord.ADDITIONAL_TAG_TABLE, timeBucket); - for (int i = 0; i < tags.size(); i++) { - sql.append(" inner join ").append(tagTable).append(" ").append(tagTable).append(i); - sql.append(" on ").append(table).append(".").append(JDBCTableInstaller.ID_COLUMN).append(" = ") - .append(tagTable).append(i).append(".").append(JDBCTableInstaller.ID_COLUMN); - } - sql.append(" where ").append(table).append(".").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); - parameters.add(OTLPSpanRecord.INDEX_NAME); - if (duration != null && duration.getStartTimestamp() > 0 && duration.getEndTimestamp() > 0) { - sql.append(" and ").append(OTLPSpanRecord.START_TIME).append(" >= ?"); - parameters.add(duration.getStartTimestamp()); - sql.append(" and ").append(OTLPSpanRecord.START_TIME).append(" <= ?"); - parameters.add(duration.getEndTimestamp()); - } - appendEquals(sql, parameters, table, OTLPSpanRecord.TRACE_ID, condition.getTraceId()); - appendEquals(sql, parameters, table, OTLPSpanRecord.SERVICE_NAME, condition.getServiceName()); - appendEquals(sql, parameters, table, OTLPSpanRecord.SERVICE_INSTANCE, condition.getServiceInstance()); - appendEquals(sql, parameters, table, OTLPSpanRecord.SCOPE_NAME, condition.getScopeName()); - appendEquals(sql, parameters, table, OTLPSpanRecord.NAME, condition.getSpanName()); - appendEquals(sql, parameters, table, OTLPSpanRecord.PEER_SERVICE, condition.getPeerService()); - if (condition.getKind() != null) { - sql.append(" and ").append(OTLPSpanRecord.KIND).append(" = ?"); - parameters.add(condition.getKind()); - } - if (condition.getStatusCode() != null) { - sql.append(" and ").append(OTLPSpanRecord.STATUS_CODE).append(" = ?"); - parameters.add(condition.getStatusCode()); - } - if (condition.getMinDurationNanos() != null) { - sql.append(" and ").append(OTLPSpanRecord.DURATION).append(" >= ?"); - parameters.add(condition.getMinDurationNanos()); - } - if (condition.getMaxDurationNanos() != null) { - sql.append(" and ").append(OTLPSpanRecord.DURATION).append(" <= ?"); - parameters.add(condition.getMaxDurationNanos()); - } - for (int i = 0; i < tags.size(); i++) { - // one form for a scoped key, either of the two for an unscoped one - final List forms = OTLPSpanRecord.indexedTags(tags.get(i).getKey(), tags.get(i).getValue()); - sql.append(" and ").append(tagTable).append(i).append(".").append(OTLPSpanRecord.TAGS); - if (forms.size() == 1) { - sql.append(" = ?"); - } else { - sql.append(" in (").append(String.join(",", Collections.nCopies(forms.size(), "?"))).append(")"); - } - parameters.addAll(forms); - } - sql.append(" group by ").append(table).append(".").append(OTLPSpanRecord.TRACE_ID); - sql.append(" order by ").append(orderColumn).append(" desc"); - sql.append(" limit ").append(condition.getLimit()); - jdbcClient.executeQuery(sql.toString(), resultSet -> { + tableRankings.add(rankSql(table, rank, condition, parameters)); + } + final Set traceIds = new LinkedHashSet<>(); + jdbcClient.executeQuery( + TraceRankingAcrossTables.sql(OTLPSpanRecord.TRACE_ID, tableRankings, aggregate, condition.getLimit()), + resultSet -> { while (resultSet.next()) { - rankByTraceId.merge( - resultSet.getString(OTLPSpanRecord.TRACE_ID), resultSet.getLong(RANK_COLUMN), - byDuration ? Math::max : Math::min); + traceIds.add(resultSet.getString(OTLPSpanRecord.TRACE_ID)); } return null; }, parameters.toArray(new Object[0])); - } - final Set traceIds = rankByTraceId.entrySet().stream() - .sorted(Map.Entry.comparingByValue().reversed()) - .limit(condition.getLimit()) - .map(Map.Entry::getKey) - .collect(Collectors.toCollection(LinkedHashSet::new)); return fetchTraces(traceIds); } + /** + * The rank of each trace with spans in the table that match the condition, over those spans. + */ + private static String rankSql(final String table, + final String rank, + final OTLPTraceQueryCondition condition, + final List parameters) { + final Duration duration = condition.getQueryDuration(); + final StringBuilder sql = new StringBuilder(); + final List tags = condition.getTags() == null ? Collections.emptyList() : condition.getTags(); + sql.append("select ").append(table).append(".").append(OTLPSpanRecord.TRACE_ID).append(", ") + .append(rank).append(" as ").append(TraceRankingAcrossTables.RANK_COLUMN).append(" from ").append(table); + final long timeBucket = TableHelper.getTimeBucket(table); + final String tagTable = TableHelper.getTable(OTLPSpanRecord.ADDITIONAL_TAG_TABLE, timeBucket); + for (int i = 0; i < tags.size(); i++) { + sql.append(" inner join ").append(tagTable).append(" ").append(tagTable).append(i); + sql.append(" on ").append(table).append(".").append(JDBCTableInstaller.ID_COLUMN).append(" = ") + .append(tagTable).append(i).append(".").append(JDBCTableInstaller.ID_COLUMN); + } + sql.append(" where ").append(table).append(".").append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); + parameters.add(OTLPSpanRecord.INDEX_NAME); + if (duration != null && duration.getStartTimestamp() > 0 && duration.getEndTimestamp() > 0) { + sql.append(" and ").append(OTLPSpanRecord.START_TIME).append(" >= ?"); + parameters.add(duration.getStartTimestamp()); + sql.append(" and ").append(OTLPSpanRecord.START_TIME).append(" <= ?"); + parameters.add(duration.getEndTimestamp()); + } + appendEquals(sql, parameters, table, OTLPSpanRecord.TRACE_ID, condition.getTraceId()); + appendEquals(sql, parameters, table, OTLPSpanRecord.SERVICE_NAME, condition.getServiceName()); + appendEquals(sql, parameters, table, OTLPSpanRecord.SERVICE_INSTANCE, condition.getServiceInstance()); + appendEquals(sql, parameters, table, OTLPSpanRecord.SCOPE_NAME, condition.getScopeName()); + appendEquals(sql, parameters, table, OTLPSpanRecord.NAME, condition.getSpanName()); + appendEquals(sql, parameters, table, OTLPSpanRecord.PEER_SERVICE, condition.getPeerService()); + if (condition.getKind() != null) { + sql.append(" and ").append(OTLPSpanRecord.KIND).append(" = ?"); + parameters.add(condition.getKind()); + } + if (condition.getStatusCode() != null) { + sql.append(" and ").append(OTLPSpanRecord.STATUS_CODE).append(" = ?"); + parameters.add(condition.getStatusCode()); + } + if (condition.getMinDurationNanos() != null) { + sql.append(" and ").append(OTLPSpanRecord.DURATION).append(" >= ?"); + parameters.add(condition.getMinDurationNanos()); + } + if (condition.getMaxDurationNanos() != null) { + sql.append(" and ").append(OTLPSpanRecord.DURATION).append(" <= ?"); + parameters.add(condition.getMaxDurationNanos()); + } + for (int i = 0; i < tags.size(); i++) { + // one form for a scoped key, either of the two for an unscoped one + final List forms = OTLPSpanRecord.indexedTags(tags.get(i).getKey(), tags.get(i).getValue()); + sql.append(" and ").append(tagTable).append(i).append(".").append(OTLPSpanRecord.TAGS); + if (forms.size() == 1) { + sql.append(" = ?"); + } else { + sql.append(" in (").append(String.join(",", Collections.nCopies(forms.size(), "?"))).append(")"); + } + parameters.addAll(forms); + } + sql.append(" group by ").append(table).append(".").append(OTLPSpanRecord.TRACE_ID); + return sql.toString(); + } + private static void appendEquals(final StringBuilder sql, final List parameters, final String table, diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCRecordsQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCRecordsQueryDAO.java index e20c8a1c9e6b..8d79768eb1e8 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCRecordsQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCRecordsQueryDAO.java @@ -47,10 +47,10 @@ public class JDBCRecordsQueryDAO implements IRecordsQueryDAO { public List readRecords(final RecordCondition condition, final String valueColumnName, final Duration duration) { - final var tables = tableHelper.getTablesForRead( + final var tables = tableHelper.getTablesBetweenTimestamps( condition.getName(), - duration.getStartTimeBucket(), - duration.getEndTimeBucket() + duration.getStartTimestamp(), + duration.getEndTimestamp() ); final var results = new ArrayList(); @@ -92,10 +92,10 @@ protected static SQLAndParameters buildSQL( parameters.add(condition.getName()); sql.append(" and ").append(TopN.ENTITY_ID).append(" = ? and"); parameters.add(condition.getParentEntity().buildId()); - sql.append(" ").append(TopN.TIME_BUCKET).append(" >= ?"); - parameters.add(duration.getStartTimeBucketInSec()); - sql.append(" and ").append(TopN.TIME_BUCKET).append(" <= ?"); - parameters.add(duration.getEndTimeBucketInSec()); + sql.append(" ").append(TopN.TIMESTAMP).append(" >= ?"); + parameters.add(duration.getStartTimestamp()); + sql.append(" and ").append(TopN.TIMESTAMP).append(" <= ?"); + parameters.add(duration.getEndTimestamp()); sql.append(" order by ").append(valueColumnName); if (condition.getOrder().equals(Order.DES)) { diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java index 3b6b3cd26708..f869470c4b48 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTagAutoCompleteQueryDAO.java @@ -130,6 +130,11 @@ protected SQLAndParameters buildSQLForQueryValues(TagType tagType, String tagKey return new SQLAndParameters(sql.toString(), condition); } + /** + * A tag is kept under the first minute of the day it is seen, so the read takes whole days: from the day of the + * duration's start through the day of its last step. The end instant, {@link Duration#getEndTimestamp()}, opens + * the next day, and honoring it would read all of that day's tags. + */ private void appendTagAutocompleteCondition(final TagType tagType, final Duration duration, final StringBuilder sql, diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTraceQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTraceQueryDAO.java index a320211628bd..0e533c37386b 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTraceQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTraceQueryDAO.java @@ -104,15 +104,9 @@ public TraceBrief queryBasicTraces(Duration duration, return new TraceBrief(); } - long startSecondTB = 0; - long endSecondTB = 0; - if (nonNull(duration)) { - startSecondTB = duration.getStartTimeBucketInSec(); - endSecondTB = duration.getEndTimeBucketInSec(); - } - - final var tables = startSecondTB > 0 && endSecondTB > 0 ? - tableHelper.getTablesForRead(SegmentRecord.INDEX_NAME, startSecondTB, endSecondTB) : + final var tables = nonNull(duration) ? + tableHelper.getTablesBetweenTimestamps( + SegmentRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp()) : tableHelper.getTablesWithinTTL(SegmentRecord.INDEX_NAME); final var traces = new ArrayList(); @@ -139,11 +133,11 @@ public TraceBrief queryBasicTraces(Duration duration, sql.append(" where "); sql.append(JDBCTableInstaller.TABLE_COLUMN).append(" = ?"); parameters.add(SegmentRecord.INDEX_NAME); - if (startSecondTB != 0 && endSecondTB != 0) { - sql.append(" and ").append(table).append(".").append(SegmentRecord.TIME_BUCKET).append(" >= ?"); - parameters.add(startSecondTB); - sql.append(" and ").append(table).append(".").append(SegmentRecord.TIME_BUCKET).append(" <= ?"); - parameters.add(endSecondTB); + if (nonNull(duration)) { + sql.append(" and ").append(table).append(".").append(SegmentRecord.START_TIME).append(" >= ?"); + parameters.add(duration.getStartTimestamp()); + sql.append(" and ").append(table).append(".").append(SegmentRecord.START_TIME).append(" <= ?"); + parameters.add(duration.getEndTimestamp()); } if (minDuration != 0) { sql.append(" and ").append(SegmentRecord.LATENCY).append(" >= ?"); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryDAO.java index 1f4184b7da76..8a51e3608757 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryDAO.java @@ -43,14 +43,12 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; -import static java.util.stream.Collectors.toCollection; import static java.util.stream.Collectors.toList; @RequiredArgsConstructor @@ -177,106 +175,114 @@ public List getTrace(final String traceId, @Nullable final Duration durati @Override @SneakyThrows public List> getTraces(final QueryRequest request, Duration duration) { - final var tables = tableHelper.getTablesForRead( + // Through the end instant, which the search counts. + final var tables = tableHelper.getTablesBetweenTimestamps( ZipkinSpanRecord.INDEX_NAME, - duration.getStartTimeBucket(), - duration.getEndTimeBucket() + duration.getStartTimestamp(), + duration.getEndTimestamp() ); + if (tables.isEmpty()) { + return new ArrayList<>(); + } + final String rank = "min(" + ZipkinSpanRecord.TIMESTAMP_MILLIS + ")"; + final List condition = new ArrayList<>(); + final List tableRankings = new ArrayList<>(tables.size()); + for (final String table : tables) { + tableRankings.add(rankSql(table, rank, request, duration, condition)); + } + final Set traceIds = new LinkedHashSet<>(); + h2Client.executeQuery( + TraceRankingAcrossTables.sql(ZipkinSpanRecord.TRACE_ID, tableRankings, "min", request.limit()), + resultSet -> { + while (resultSet.next()) { + traceIds.add(resultSet.getString(ZipkinSpanRecord.TRACE_ID)); + } + return null; + }, condition.toArray(new Object[0])); + return getTraces(traceIds, duration); + } + + /** + * The first match of each trace with spans in the table that match the request. + */ + private String rankSql(final String table, + final String rank, + final QueryRequest request, + final Duration duration, + final List condition) { final long startTimeMillis = duration.getStartTimestamp(); final long endTimeMillis = duration.getEndTimestamp(); - final Map firstMatchByTraceId = new HashMap<>(); - - for (String table : tables) { - StringBuilder sql = new StringBuilder(); - List condition = new ArrayList<>(5); - List> annotations = new ArrayList<>(request.annotationQuery().entrySet()); - sql.append("select ").append(table).append(".").append(ZipkinSpanRecord.TRACE_ID).append(", ") - .append("min(").append(ZipkinSpanRecord.TIMESTAMP_MILLIS).append(")").append(" from "); - sql.append(table); - /* - * This is an AdditionalEntity feature, see: - * {@link org.apache.skywalking.oap.server.core.storage.annotation.SQLDatabase.AdditionalEntity} - */ - final var timeBucket = TableHelper.getTimeBucket(table); - final var tagTable = TableHelper.getTable(ZipkinSpanRecord.ADDITIONAL_QUERY_TABLE, timeBucket); - if (!CollectionUtils.isEmpty(annotations)) { - for (int i = 0; i < annotations.size(); i++) { - sql.append(" inner join ").append(tagTable).append(" "); - sql.append(tagTable + i); - sql.append(" on ").append(table).append(".").append(JDBCTableInstaller.ID_COLUMN).append(" = "); - sql.append(tagTable + i).append(".").append(JDBCTableInstaller.ID_COLUMN); - } - } - sql.append(" where "); - sql.append(" 1=1 "); - if (startTimeMillis > 0 && endTimeMillis > 0) { - sql.append(" and "); - sql.append(ZipkinSpanRecord.TIMESTAMP_MILLIS).append(" >= ?"); - condition.add(startTimeMillis); - sql.append(" and "); - sql.append(ZipkinSpanRecord.TIMESTAMP_MILLIS).append(" <= ?"); - condition.add(endTimeMillis); - buildShardingCondition(sql, condition, startTimeMillis, endTimeMillis); - } - if (request.minDuration() != null) { - sql.append(" and "); - sql.append(ZipkinSpanRecord.DURATION).append(" >= ?"); - condition.add(request.minDuration()); + final List> annotations = new ArrayList<>(request.annotationQuery().entrySet()); + final StringBuilder sql = new StringBuilder(); + sql.append("select ").append(table).append(".").append(ZipkinSpanRecord.TRACE_ID).append(", ") + .append(rank).append(" as ").append(TraceRankingAcrossTables.RANK_COLUMN).append(" from "); + sql.append(table); + /* + * This is an AdditionalEntity feature, see: + * {@link org.apache.skywalking.oap.server.core.storage.annotation.SQLDatabase.AdditionalEntity} + */ + final var timeBucket = TableHelper.getTimeBucket(table); + final var tagTable = TableHelper.getTable(ZipkinSpanRecord.ADDITIONAL_QUERY_TABLE, timeBucket); + if (!CollectionUtils.isEmpty(annotations)) { + for (int i = 0; i < annotations.size(); i++) { + sql.append(" inner join ").append(tagTable).append(" "); + sql.append(tagTable + i); + sql.append(" on ").append(table).append(".").append(JDBCTableInstaller.ID_COLUMN).append(" = "); + sql.append(tagTable + i).append(".").append(JDBCTableInstaller.ID_COLUMN); } - if (request.maxDuration() != null) { - sql.append(" and "); - sql.append(ZipkinSpanRecord.DURATION).append(" <= ?"); - condition.add(request.maxDuration()); - } - if (!StringUtil.isEmpty(request.serviceName())) { - sql.append(" and "); - sql.append(ZipkinSpanRecord.LOCAL_ENDPOINT_SERVICE_NAME).append(" = ?"); - condition.add(request.serviceName()); - } - if (!StringUtil.isEmpty(request.remoteServiceName())) { - sql.append(" and "); - sql.append(ZipkinSpanRecord.REMOTE_ENDPOINT_SERVICE_NAME).append(" = ?"); - condition.add(request.remoteServiceName()); - } - if (!StringUtil.isEmpty(request.spanName())) { - sql.append(" and "); - sql.append(ZipkinSpanRecord.NAME).append(" = ?"); - condition.add(request.spanName()); - } - if (CollectionUtils.isNotEmpty(annotations)) { - for (int i = 0; i < annotations.size(); i++) { - Map.Entry annotation = annotations.get(i); - if (annotation.getValue().isEmpty()) { - sql.append(" and ").append(tagTable).append(i).append("."); - sql.append(ZipkinSpanRecord.QUERY).append(" = ?"); - condition.add(annotation.getKey()); - } else { - sql.append(" and ").append(tagTable).append(i).append("."); - sql.append(ZipkinSpanRecord.QUERY).append(" = ?"); - condition.add(annotation.getKey() + "=" + annotation.getValue()); - } + } + sql.append(" where "); + sql.append(" 1=1 "); + if (startTimeMillis > 0 && endTimeMillis > 0) { + sql.append(" and "); + sql.append(ZipkinSpanRecord.TIMESTAMP_MILLIS).append(" >= ?"); + condition.add(startTimeMillis); + sql.append(" and "); + sql.append(ZipkinSpanRecord.TIMESTAMP_MILLIS).append(" <= ?"); + condition.add(endTimeMillis); + buildShardingCondition(sql, condition, startTimeMillis, endTimeMillis); + } + if (request.minDuration() != null) { + sql.append(" and "); + sql.append(ZipkinSpanRecord.DURATION).append(" >= ?"); + condition.add(request.minDuration()); + } + if (request.maxDuration() != null) { + sql.append(" and "); + sql.append(ZipkinSpanRecord.DURATION).append(" <= ?"); + condition.add(request.maxDuration()); + } + if (!StringUtil.isEmpty(request.serviceName())) { + sql.append(" and "); + sql.append(ZipkinSpanRecord.LOCAL_ENDPOINT_SERVICE_NAME).append(" = ?"); + condition.add(request.serviceName()); + } + if (!StringUtil.isEmpty(request.remoteServiceName())) { + sql.append(" and "); + sql.append(ZipkinSpanRecord.REMOTE_ENDPOINT_SERVICE_NAME).append(" = ?"); + condition.add(request.remoteServiceName()); + } + if (!StringUtil.isEmpty(request.spanName())) { + sql.append(" and "); + sql.append(ZipkinSpanRecord.NAME).append(" = ?"); + condition.add(request.spanName()); + } + if (CollectionUtils.isNotEmpty(annotations)) { + for (int i = 0; i < annotations.size(); i++) { + Map.Entry annotation = annotations.get(i); + if (annotation.getValue().isEmpty()) { + sql.append(" and ").append(tagTable).append(i).append("."); + sql.append(ZipkinSpanRecord.QUERY).append(" = ?"); + condition.add(annotation.getKey()); + } else { + sql.append(" and ").append(tagTable).append(i).append("."); + sql.append(ZipkinSpanRecord.QUERY).append(" = ?"); + condition.add(annotation.getKey() + "=" + annotation.getValue()); } } - sql.append(" group by ").append(table).append(".").append(ZipkinSpanRecord.TRACE_ID); - sql.append(" order by min(").append(ZipkinSpanRecord.TIMESTAMP_MILLIS).append(") desc"); - sql.append(" limit ").append(request.limit()); - h2Client.executeQuery(sql.toString(), resultSet -> { - while (resultSet.next()) { - firstMatchByTraceId.merge( - resultSet.getString(ZipkinSpanRecord.TRACE_ID), resultSet.getLong(2), Math::min); - } - return null; - }, condition.toArray(new Object[0])); } - - // Each day table answers its own top `limit`; the traces of every table are ranked together and cut once. - final Set traceIds = firstMatchByTraceId.entrySet() - .stream() - .sorted(Map.Entry.comparingByValue().reversed()) - .limit(request.limit()) - .map(Map.Entry::getKey) - .collect(toCollection(LinkedHashSet::new)); - return getTraces(traceIds, duration); + sql.append(" group by ").append(table).append(".").append(ZipkinSpanRecord.TRACE_ID); + return sql.toString(); } @Override diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/TraceRankingAcrossTables.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/TraceRankingAcrossTables.java new file mode 100644 index 000000000000..8f2e75840d32 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/TraceRankingAcrossTables.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * The query of the traces a search ranks highest over the day tables of its range. A trace's rank is the minimum or + * the maximum of a column over its matching spans, and a trace that crosses midnight has matching spans in two + * tables, so each table ranks the traces it holds, the union of those ranks is grouped by trace once more, and the + * limit applies to that: the result is exactly what one table holding every row would return, in one round trip + * that returns only the traces kept. A range within one day table is ranked and limited in that table's query alone. + * A trace ranked by its greatest value that is among the highest of the range is among the highest of the table + * holding that value, so each table's ranking is limited too; a trace ranked by its least value is not, as another + * table can hold a greater value of it that ranks it there. + */ +final class TraceRankingAcrossTables { + /** + * The column a table's ranking names a trace's rank in, and the query ranks the traces by. + */ + static final String RANK_COLUMN = "rank_value"; + + private TraceRankingAcrossTables() { + } + + /** + * @param traceIdColumn the trace id column, which each table's ranking selects under its own name + * @param tableRankings one query per day table, of the trace id and the trace's rank in that table as + * {@link #RANK_COLUMN}, grouped by trace + * @param aggregate {@code min} when a trace ranks by its least value, {@code max} by its greatest + * @return the query of the {@code limit} traces ranking highest, highest first, by trace id within a rank + */ + static String sql(final String traceIdColumn, final List tableRankings, final String aggregate, + final int limit) { + final String highestFirst = " order by " + RANK_COLUMN + " desc, " + traceIdColumn + " limit " + limit; + if (tableRankings.size() == 1) { + return tableRankings.get(0) + highestFirst; + } + final List rankings = "max".equals(aggregate) + ? tableRankings.stream().map(ranking -> "(" + ranking + highestFirst + ")").collect(Collectors.toList()) + : tableRankings; + return "select " + traceIdColumn + ", " + aggregate + "(" + RANK_COLUMN + ") as " + RANK_COLUMN + + " from (" + String.join(" union all ", rankings) + ") ranked" + + " group by " + traceIdColumn + + " order by " + aggregate + "(" + RANK_COLUMN + ") desc, " + traceIdColumn + + " limit " + limit; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/AttachedEventFixture.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/AttachedEventFixture.java new file mode 100644 index 000000000000..b87429c87d6e --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/AttachedEventFixture.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.apm.network.common.v3.Instant; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.v3.SpanAttachedEvent; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.spanattach.SWSpanAttachedEventRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.spanattach.SpanAttachedEventRecord; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.storage.StorageData; +import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import org.joda.time.DateTime; + +/** + * The time base and the span attached events of the native and the Zipkin trace fixtures, and how OAP stores an + * event. + */ +final class AttachedEventFixture { + /** Today's local midnight, so the fixture is written to yesterday's and today's tables, within the TTL. */ + static final long MIDNIGHT = JDBCQueryITSupport.MIDNIGHT; + static final long MIDNIGHT_SECOND = MIDNIGHT / 1000; + /** A minute either side of midnight, the duration every query of the suite passes. */ + static final Duration DURATION = duration(MIDNIGHT + 59_000L, 119_000L); + static final long START_SECOND = DURATION.getStartTimestamp() / 1000; + static final long END_SECOND = DURATION.getEndTimestamp() / 1000; + + private AttachedEventFixture() { + } + + /** + * The duration {@code ZipkinQueryHandler} builds from a request's {@code endTs} and {@code lookback}. + */ + static Duration duration(final long endTs, final long lookback) { + final Duration duration = new Duration(); + duration.setStep(Step.SECOND); + duration.setStart(new DateTime(endTs - lookback).toString("yyyy-MM-dd HHmmss")); + duration.setEnd(new DateTime(endTs).toString("yyyy-MM-dd HHmmss")); + return duration; + } + + /** + * An event as rover reports it, lasting 2 ms, against the trace context the traffic carried. + * + * @param tags keys and values, one after the other + */ + static SpanAttachedEvent.Builder event(final SpanAttachedEvent.SpanReferenceType type, final String traceId, + final String segmentId, final String spanId, final long startSecond, + final int startNanos, final String name, final String... tags) { + final long endNanos = startNanos + 2_000_000L; + final SpanAttachedEvent.Builder event = SpanAttachedEvent.newBuilder() + .setStartTime(Instant.newBuilder().setSeconds(startSecond).setNanos(startNanos)) + .setEndTime(Instant.newBuilder() + .setSeconds(startSecond + endNanos / 1_000_000_000L) + .setNanos((int) (endNanos % 1_000_000_000L))) + .setEvent(name) + .setTraceContext(SpanAttachedEvent.SpanReference.newBuilder() + .setType(type) + .setTraceId(traceId) + .setTraceSegmentId(segmentId) + .setSpanId(spanId)); + for (int i = 0; i < tags.length; i += 2) { + event.addTags(KeyStringValuePair.newBuilder().setKey(tags[i]).setValue(tags[i + 1])); + } + return event; + } + + /** + * The record {@code SpanAttachedEventReportServiceHandler} writes for an event of a SkyWalking trace context. + */ + static SWSpanAttachedEventRecord skyWalkingRecord(final SpanAttachedEvent event) { + final SWSpanAttachedEventRecord record = new SWSpanAttachedEventRecord(); + record.setStartTimeSecond(event.getStartTime().getSeconds()); + record.setStartTimeNanos(event.getStartTime().getNanos()); + record.setEvent(event.getEvent()); + record.setEndTimeSecond(event.getEndTime().getSeconds()); + record.setEndTimeNanos(event.getEndTime().getNanos()); + record.setTraceRefType(event.getTraceContext().getTypeValue()); + record.setRelatedTraceId(event.getTraceContext().getTraceId()); + record.setTraceSegmentId(event.getTraceContext().getTraceSegmentId()); + record.setTraceSpanId(event.getTraceContext().getSpanId()); + record.setDataBinary(event.toByteArray()); + final long timestamp = timestampOf(event); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + record.setTimestamp(timestamp); + return record; + } + + /** + * The record {@code SpanAttachedEventReportServiceHandler} writes for an event of a Zipkin trace context. + */ + static SpanAttachedEventRecord zipkinRecord(final SpanAttachedEvent event) { + final SpanAttachedEventRecord record = new SpanAttachedEventRecord(); + record.setStartTimeSecond(event.getStartTime().getSeconds()); + record.setStartTimeNanos(event.getStartTime().getNanos()); + record.setEvent(event.getEvent()); + record.setEndTimeSecond(event.getEndTime().getSeconds()); + record.setEndTimeNanos(event.getEndTime().getNanos()); + record.setTraceRefType(event.getTraceContext().getTypeValue()); + record.setRelatedTraceId(event.getTraceContext().getTraceId()); + record.setTraceSegmentId(event.getTraceContext().getTraceSegmentId()); + record.setTraceSpanId(event.getTraceContext().getSpanId()); + record.setDataBinary(event.toByteArray()); + final long timestamp = timestampOf(event); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + record.setTimestamp(timestamp); + return record; + } + + private static long timestampOf(final SpanAttachedEvent event) { + return TimeUnit.SECONDS.toMillis(event.getStartTime().getSeconds()) + + TimeUnit.NANOSECONDS.toMillis(event.getStartTime().getNanos()); + } + + /** + * Every column the records' storage builder stores but the timestamp, so records compare whole: the JDBC DAO + * does not read the timestamp back, and no caller uses it. + */ + static List> stored(final StorageBuilder builder, + final List records) { + final List> stored = new ArrayList<>(records.size()); + for (final T record : records) { + final HashMapConverter.ToStorage columns = new HashMapConverter.ToStorage(); + builder.entity2Storage(record, columns); + final Map read = new TreeMap<>(columns.obtain()); + read.remove(SpanAttachedEventRecord.TIMESTAMP); + stored.add(read); + } + return stored; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryITBase.java new file mode 100644 index 000000000000..bb1f6885f868 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryITBase.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.aiagent.AIAgentSessionDataRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.aiagent.AIAgentSessionFlowRecord; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.query.IAIAgentConversationQueryDAO; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.JdbcDatabaseContainer; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads AI agent conversations the way the conversation query service does on a JDBC storage, through + * {@link JDBCAIAgentConversationQueryDAO}: the latest rounds of a service, a sender or a conversation, with or + * without their bodies; the highest round of a conversation; the rounds of a round-number window, every copy of a + * round two senders pushed included; and the files of a session in a seq window and a timestamp range. A subclass + * names the server. + * + *

The rounds and the files are written on both sides of midnight, so a read crosses two day tables, and the + * highest round of the conversation is its oldest row. Each read has a row of another service, sender, conversation + * or session that matches it otherwise, and rows just outside the window. A round pushed by more senders than a window + * makes room for shows the window's limit. Rounds of their own lie on and just outside the bounds of a duration: on + * its first millisecond and the one before, on its last millisecond, on the instant it ends at + * ({@link Duration#getEndTimestamp()}, which a read includes) and the millisecond after. + * One more lies on the end instant of a duration ending in a day's last minute: the next midnight, in the next + * day's table. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCAIAgentConversationQueryITBase { + private static final long MINUTE = 60_000L; + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + private static final Duration ENDING = minutes(MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE); + /** Yesterday's midnight. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = minutes(PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE); + + private static final String SERVICE = "ai-agent"; + private static final String OTHER_SERVICE = "other-agent"; + private static final String CONVERSATION = "conversation-read"; + private static final String SESSION = "session-read"; + /** More senders than the window of one round makes room for. */ + private static final int COPIES = IAIAgentConversationQueryDAO.MAX_COPIES + 1; + + /** Round 5 was pushed first, so the highest round is not the newest row. */ + private static final List ROUNDS = Arrays.asList( + new Round(SERVICE, "instance-0", CONVERSATION, 5, MIDNIGHT - 150_000L), + new Round(SERVICE, "instance-0", CONVERSATION, 1, MIDNIGHT - 110_000L), + new Round(SERVICE, "instance-0", CONVERSATION, 2, MIDNIGHT - 70_000L), + new Round(SERVICE, "instance-0", CONVERSATION, 3, MIDNIGHT - 30_000L), + new Round(SERVICE, "instance-1", CONVERSATION, 3, MIDNIGHT + 30_000L), + new Round(SERVICE, "instance-0", CONVERSATION, 4, MIDNIGHT + 70_000L).counted(), + new Round(SERVICE, "instance-0", "conversation-other", 9, MIDNIGHT + 110_000L), + new Round(OTHER_SERVICE, "instance-0", CONVERSATION, 7, MIDNIGHT + 150_000L), + new Round(SERVICE, "instance-0", "conversation-bounds", 1, BOUNDED.getStartTimestamp() - 1), + new Round(SERVICE, "instance-0", "conversation-bounds", 2, BOUNDED.getStartTimestamp()), + new Round(SERVICE, "instance-0", "conversation-bounds", 3, BOUNDED.getEndTimestamp() - 1), + new Round(SERVICE, "instance-0", "conversation-bounds", 4, BOUNDED.getEndTimestamp() + 1), + new Round(SERVICE, "instance-0", "conversation-bounds", 5, ENDING.getEndTimestamp()), + new Round(SERVICE, "instance-0", "conversation-bounds", 6, ENDING_AT_MIDNIGHT.getEndTimestamp()) + ); + /** Seq 3 is pushed by a second sender as well, a second later. */ + private static final List FILES = Arrays.asList( + new SessionFile(SERVICE, "instance-0", SESSION, 1), + new SessionFile(SERVICE, "instance-0", SESSION, 2), + new SessionFile(SERVICE, "instance-0", SESSION, 3), + new SessionFile(SERVICE, "instance-1", SESSION, 3), + new SessionFile(SERVICE, "instance-0", SESSION, 4), + new SessionFile(SERVICE, "instance-0", SESSION, 5), + new SessionFile(SERVICE, "instance-0", SESSION, 6), + new SessionFile(SERVICE, "instance-0", "session-other", 3), + new SessionFile(OTHER_SERVICE, "instance-0", SESSION, 3) + ); + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCAIAgentConversationQueryDAO dao; + + JDBCAIAgentConversationQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void writeConversations() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + final List rounds = ROUNDS.stream().map(Round::record).collect(Collectors.toList()); + final List files = FILES.stream().map(SessionFile::record) + .collect(Collectors.toList()); + for (int i = 0; i < COPIES; i++) { + rounds.add(new Round(SERVICE, "copy-" + i, "conversation-copies", 1, MIDNIGHT + 40 * MINUTE).record()); + files.add(new SessionFile(SERVICE, "copy-" + i, "session-copies", 1).record()); + } + final Model roundModel = support.install(AIAgentSessionFlowRecord.class, + DefaultScopeDefine.AI_AGENT_SESSION_FLOW, + AIAgentSessionFlowRecord.INDEX_NAME, DownSampling.Second); + final Model fileModel = support.install(AIAgentSessionDataRecord.class, + DefaultScopeDefine.AI_AGENT_SESSION_DATA, + AIAgentSessionDataRecord.INDEX_NAME, DownSampling.Second); + support.writeRecords(roundModel, new AIAgentSessionFlowRecord.Builder(), rounds); + support.writeRecords(fileModel, new AIAgentSessionDataRecord.Builder(), files); + dao = new JDBCAIAgentConversationQueryDAO(support.getClient(), support.getTableHelper()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheLatestRoundsOfAConversationNewestFirst() { + assertEquals(Arrays.asList("4@instance-0", "3@instance-1", "3@instance-0", "2@instance-0", "1@instance-0", + "5@instance-0"), + rounds(dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 10, false))); + assertEquals(Arrays.asList("4@instance-0", "3@instance-1", "3@instance-0"), + rounds(dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 3, false))); + } + + @Test + public void readsTheRoundsOfASenderOfEveryConversationAndOfAService() { + assertEquals(Collections.singletonList("3@instance-1"), rounds(dao.queryRounds( + serviceId(SERVICE), instanceId(SERVICE, "instance-1"), CONVERSATION, AROUND_MIDNIGHT, 10, false))); + assertEquals(Arrays.asList("9@instance-0", "4@instance-0", "3@instance-1", "3@instance-0", "2@instance-0", + "1@instance-0", "5@instance-0"), + rounds(dao.queryRounds(serviceId(SERVICE), null, null, AROUND_MIDNIGHT, 10, false))); + assertEquals(Collections.singletonList("7@instance-0"), rounds(dao.queryRounds( + serviceId(OTHER_SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 10, false))); + } + + @Test + public void readsTheRoundsOfEveryDayWithoutADuration() { + assertEquals(Arrays.asList("4@instance-0", "3@instance-1", "3@instance-0", "2@instance-0", "1@instance-0", + "5@instance-0"), + rounds(dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, null, 10, false))); + } + + @Test + public void readsEachRoundWholeWithOrWithoutItsBody() { + final List withBody = new ArrayList<>(); + final List withoutBody = new ArrayList<>(); + for (int i = 5; i >= 0; i--) { + withBody.add(render(ROUNDS.get(i).record())); + final AIAgentSessionFlowRecord noBody = ROUNDS.get(i).record(); + noBody.setBody(null); + withoutBody.add(render(noBody)); + } + assertEquals(withBody, dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 10, true) + .stream().map(JDBCAIAgentConversationQueryITBase::render) + .collect(Collectors.toList())); + assertEquals(withoutBody, dao.queryRounds(serviceId(SERVICE), null, CONVERSATION, AROUND_MIDNIGHT, 10, false) + .stream().map(JDBCAIAgentConversationQueryITBase::render) + .collect(Collectors.toList())); + } + + @Test + public void keepsTheRoundsWithinTheDuration() { + assertEquals(Arrays.asList("3@instance-0", "2@instance-0"), rounds(dao.queryRounds( + serviceId(SERVICE), null, "conversation-bounds", BOUNDED, 10, false))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's table. + */ + @Test + public void readsARoundOnTheInstantTheDurationEndsAt() { + assertEquals(Collections.singletonList("5@instance-0"), rounds(dao.queryRounds( + serviceId(SERVICE), null, "conversation-bounds", ENDING, 10, false))); + assertEquals(Collections.singletonList("6@instance-0"), rounds(dao.queryRounds( + serviceId(SERVICE), null, "conversation-bounds", ENDING_AT_MIDNIGHT, 10, false))); + } + + @Test + public void readsTheHighestRoundOfAConversation() { + assertEquals(5, dao.queryHeadRound(serviceId(SERVICE), null, CONVERSATION, false)); + assertEquals(3, dao.queryHeadRound(serviceId(SERVICE), instanceId(SERVICE, "instance-1"), CONVERSATION, false)); + assertEquals(9, dao.queryHeadRound(serviceId(SERVICE), null, "conversation-other", false)); + assertEquals(7, dao.queryHeadRound(serviceId(OTHER_SERVICE), null, CONVERSATION, false)); + assertEquals(0, dao.queryHeadRound(serviceId(SERVICE), null, "conversation-missing", false)); + } + + /** + * Copies of one round come in no order. + */ + @Test + public void readsEveryCopyOfTheRoundsOfAWindowInRoundOrder() { + final List window = dao.queryRoundsByNumber( + serviceId(SERVICE), null, CONVERSATION, 2, 4, Integer.MAX_VALUE, false); + assertEquals(Arrays.asList(2L, 3L, 3L, 4L), window.stream().map(AIAgentSessionFlowRecord::getRound) + .collect(Collectors.toList())); + assertEquals(Arrays.asList(render(ROUNDS.get(2).record()), render(ROUNDS.get(3).record()), + render(ROUNDS.get(4).record()), render(ROUNDS.get(5).record())), + window.stream().sorted(Comparator.comparing(AIAgentSessionFlowRecord::getRound) + .thenComparing(AIAgentSessionFlowRecord::getServiceInstanceId)) + .map(JDBCAIAgentConversationQueryITBase::render).collect(Collectors.toList())); + assertEquals(Arrays.asList("2@instance-0", "3@instance-0", "4@instance-0"), rounds(dao.queryRoundsByNumber( + serviceId(SERVICE), instanceId(SERVICE, "instance-0"), CONVERSATION, 2, 4, Integer.MAX_VALUE, false))); + assertEquals(Collections.singletonList("5@instance-0"), rounds(dao.queryRoundsByNumber( + serviceId(SERVICE), null, CONVERSATION, 5, 5, Integer.MAX_VALUE, false))); + assertEquals(Collections.emptyList(), rounds(dao.queryRoundsByNumber( + serviceId(SERVICE), null, CONVERSATION, 6, 8, Integer.MAX_VALUE, false))); + } + + @Test + public void readsAtMostTheRowsTheWindowMakesRoomFor() { + final List rounds = dao.queryRoundsByNumber( + serviceId(SERVICE), null, "conversation-copies", 1, 1, Integer.MAX_VALUE, false); + assertEquals(Collections.nCopies(IAIAgentConversationQueryDAO.MAX_COPIES, 1L), rounds.stream() + .map(AIAgentSessionFlowRecord::getRound).collect(Collectors.toList())); + final List files = dao.queryFiles( + serviceId(SERVICE), null, "session-copies", time(1) - 1, time(1) + 1, 1, 1, Integer.MAX_VALUE, false); + assertEquals(Collections.nCopies(IAIAgentConversationQueryDAO.MAX_COPIES, 1L), files.stream() + .map(AIAgentSessionDataRecord::getSeq).collect(Collectors.toList())); + } + + /** + * Copies of one seq come in no order. + */ + @Test + public void readsTheFilesOfASeqWindowBySeq() { + final List files = dao.queryFiles( + serviceId(SERVICE), null, SESSION, time(1), time(6) + 1000L, 2, 4, Integer.MAX_VALUE, false); + assertEquals(Arrays.asList(2L, 3L, 3L, 4L), files.stream().map(AIAgentSessionDataRecord::getSeq) + .collect(Collectors.toList())); + assertEquals(Arrays.asList(render(FILES.get(1).record()), render(FILES.get(2).record()), + render(FILES.get(3).record()), render(FILES.get(4).record())), + files.stream().sorted(Comparator.comparing(AIAgentSessionDataRecord::getSeq) + .thenComparing(AIAgentSessionDataRecord::getServiceInstanceId)) + .map(JDBCAIAgentConversationQueryITBase::render).collect(Collectors.toList())); + assertEquals(Collections.singletonList("3@instance-1"), files(dao.queryFiles( + serviceId(SERVICE), instanceId(SERVICE, "instance-1"), SESSION, time(1), time(6) + 1000L, 1, 6, + Integer.MAX_VALUE, false))); + } + + /** + * Both ends of the timestamp range are inclusive. + */ + @Test + public void readsTheFilesOnAndWithinTheTimestampRange() { + assertEquals(Arrays.asList("2@instance-0", "3@instance-0", "3@instance-1", "4@instance-0", "5@instance-0"), + files(dao.queryFiles(serviceId(SERVICE), null, SESSION, time(2), time(5), 1, 6, + Integer.MAX_VALUE, false))); + assertEquals(Arrays.asList("3@instance-0", "3@instance-1", "4@instance-0"), + files(dao.queryFiles(serviceId(SERVICE), null, SESSION, time(2) + 1, time(5) - 1, 1, 6, + Integer.MAX_VALUE, false))); + } + + @Test + public void readsTheFilesOfEveryDaySinceTheEpoch() { + assertEquals(Arrays.asList("1@instance-0", "2@instance-0", "3@instance-0", "3@instance-1", "4@instance-0", + "5@instance-0", "6@instance-0"), + files(dao.queryFiles(serviceId(SERVICE), null, SESSION, 0, time(6) + 1000L, 1, 6, + Integer.MAX_VALUE, false))); + } + + private static List rounds(final List rounds) { + return rounds.stream() + .map(round -> round.getRound() + "@" + instanceName(round.getServiceInstanceId())) + .collect(Collectors.toList()); + } + + /** Copies of one seq sorted by their sender. */ + private static List files(final List files) { + return files.stream() + .sorted(Comparator.comparing(AIAgentSessionDataRecord::getSeq) + .thenComparing(AIAgentSessionDataRecord::getServiceInstanceId)) + .map(file -> file.getSeq() + "@" + instanceName(file.getServiceInstanceId())) + .collect(Collectors.toList()); + } + + private static String render(final AIAgentSessionFlowRecord round) { + return String.join("|", round.getServiceId(), round.getServiceInstanceId(), round.getConversation(), + String.valueOf(round.getRound()), String.valueOf(round.getSessionFromTime()), + round.getTitle(), String.valueOf(round.getTalks()), String.valueOf(round.getSteps()), + String.valueOf(round.getStreams()), String.valueOf(round.getSegments()), + String.valueOf(round.getUnresolved()), String.valueOf(round.getChanges()), + String.valueOf(round.getLinesAdded()), String.valueOf(round.getLinesRemoved()), + String.valueOf(round.getLlmCalls()), String.valueOf(round.getSubagents()), + String.valueOf(round.getBashRuns()), round.getDigest(), + String.valueOf(round.getTimestamp()), String.valueOf(round.getTimeBucket()), + body(round.getBody())); + } + + private static String render(final AIAgentSessionDataRecord file) { + return String.join("|", file.getServiceId(), file.getServiceInstanceId(), file.getSession(), + String.valueOf(file.getSeq()), file.getDigest(), String.valueOf(file.getTimestamp()), + String.valueOf(file.getTimeBucket()), body(file.getBody())); + } + + private static String body(final byte[] body) { + return body == null ? "no body" : new String(body, StandardCharsets.UTF_8); + } + + private static String instanceName(final String instanceId) { + return IDManager.ServiceInstanceID.analysisId(instanceId).getName(); + } + + /** The time of a session's file of the seq: seq 3 at midnight, a minute apart. */ + private static long time(final long seq) { + return MIDNIGHT + (seq - 3) * MINUTE; + } + + private static Duration minutes(final long start, final long end) { + return JDBCQueryITSupport.duration(start, end, Step.MINUTE); + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + private static String instanceId(final String service, final String instance) { + return IDManager.ServiceInstanceID.buildId(serviceId(service), instance); + } + + /** + * A conversation round as the Sessionizer pushes it; only round 4 carries the optional counts. + */ + private static final class Round { + private final String service; + private final String instance; + private final String conversation; + private final long round; + private final long time; + private boolean counted; + + Round(final String service, final String instance, final String conversation, final long round, + final long time) { + this.service = service; + this.instance = instance; + this.conversation = conversation; + this.round = round; + this.time = time; + } + + Round counted() { + counted = true; + return this; + } + + AIAgentSessionFlowRecord record() { + final AIAgentSessionFlowRecord record = new AIAgentSessionFlowRecord(); + record.setServiceId(serviceId(service)); + record.setServiceInstanceId(instanceId(service, instance)); + record.setConversation(conversation); + record.setRound(round); + record.setSessionFromTime(MIDNIGHT - 10 * MINUTE); + record.setTitle("title of " + conversation); + record.setTalks(round); + record.setSteps(round * 2); + record.setStreams(round * 3); + record.setSegments(round * 4); + record.setUnresolved(round % 2); + if (counted) { + record.setChanges(2L); + record.setLinesAdded(10L); + record.setLinesRemoved(3L); + record.setLlmCalls(7L); + record.setSubagents(1L); + record.setBashRuns(4L); + } + record.setDigest("digest-" + conversation + "-" + round); + record.setTimestamp(time); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + record.setBody(("round " + round + " of " + conversation + " by " + instance) + .getBytes(StandardCharsets.UTF_8)); + return record; + } + } + + /** + * A session file as the Sessionizer pushes it, at the time of its seq; a second sender pushes a second later. + */ + private static final class SessionFile { + private final String service; + private final String instance; + private final String session; + private final long seq; + + SessionFile(final String service, final String instance, final String session, final long seq) { + this.service = service; + this.instance = instance; + this.session = session; + this.seq = seq; + } + + AIAgentSessionDataRecord record() { + final long timestamp = "instance-1".equals(instance) ? time(seq) + 1000L : time(seq); + final AIAgentSessionDataRecord record = new AIAgentSessionDataRecord(); + record.setServiceId(serviceId(service)); + record.setServiceInstanceId(instanceId(service, instance)); + record.setSession(session); + record.setSeq(seq); + record.setDigest("file-" + session + "-" + seq); + record.setTimestamp(timestamp); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + record.setBody(("file " + seq + " of " + session + " by " + instance).getBytes(StandardCharsets.UTF_8)); + return record; + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch2IT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryMariaDBIT.java similarity index 61% rename from oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch2IT.java rename to oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryMariaDBIT.java index bf59e6f85d95..9d3f3c3b7c90 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch2IT.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryMariaDBIT.java @@ -16,22 +16,18 @@ * */ -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin; +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.containers.MariaDBContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -@Tag("slow") -@Tag("elastic-it") @Testcontainers -public class ZipkinQueryEsDAOOpenSearch2IT extends ZipkinQueryEsDAOITBase { +public class JDBCAIAgentConversationQueryMariaDBIT extends JDBCAIAgentConversationQueryITBase { @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("2.8.0"); + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); - public ZipkinQueryEsDAOOpenSearch2IT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.DEFAULT); + public JDBCAIAgentConversationQueryMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch3IT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryPostgreSQLIT.java similarity index 61% rename from oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch3IT.java rename to oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryPostgreSQLIT.java index 15beb8072d40..54e570991726 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOOpenSearch3IT.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAIAgentConversationQueryPostgreSQLIT.java @@ -16,22 +16,18 @@ * */ -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin; +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -@Tag("slow") -@Tag("elastic-it") @Testcontainers -public class ZipkinQueryEsDAOOpenSearch3IT extends ZipkinQueryEsDAOITBase { +public class JDBCAIAgentConversationQueryPostgreSQLIT extends JDBCAIAgentConversationQueryITBase { @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("3.0.0"); + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); - public ZipkinQueryEsDAOOpenSearch3IT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.DEFAULT); + public JDBCAIAgentConversationQueryPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryDAOTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryDAOTest.java index 171e385dbca2..d2851bd7d17b 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryDAOTest.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryDAOTest.java @@ -103,7 +103,7 @@ void buildSQL_withKeyword_shouldIncludeLikeCondition() { } @Test - void buildSQL_withDuration_shouldIncludeTimeBucketConditions() { + void buildSQL_withDuration_shouldIncludeTheStartTimesThroughTheEndInstant() { final Duration duration = new Duration(); duration.setStart("2026-04-06 0000"); duration.setEnd("2026-04-06 2359"); @@ -112,8 +112,9 @@ void buildSQL_withDuration_shouldIncludeTimeBucketConditions() { final SQLAndParameters result = dao.buildSQL(null, null, 10, 0, duration, null, TABLE); final String sql = result.sql(); - assertThat(sql).contains("and " + TABLE + "." + AlarmRecord.TIME_BUCKET + " >= ?"); - assertThat(sql).contains("and " + TABLE + "." + AlarmRecord.TIME_BUCKET + " <= ?"); + assertThat(sql).contains("and " + TABLE + "." + AlarmRecord.START_TIME + " >= ?"); + assertThat(sql).contains("and " + TABLE + "." + AlarmRecord.START_TIME + " <= ?"); + assertThat(result.parameters()).contains(duration.getStartTimestamp(), duration.getEndTimestamp()); } @Test diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryITBase.java new file mode 100644 index 000000000000..ef9ee3887cee --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryITBase.java @@ -0,0 +1,514 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import com.google.gson.Gson; +import com.google.gson.JsonObject; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.alarm.AlarmRecord; +import org.apache.skywalking.oap.server.core.alarm.AlarmRecoveryRecord; +import org.apache.skywalking.oap.server.core.alarm.AlarmSnapshotRecord; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.query.enumeration.Scope; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.AlarmQueryCondition; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.Entity; +import org.apache.skywalking.oap.server.core.query.mqe.MQEValue; +import org.apache.skywalking.oap.server.core.query.mqe.MQEValues; +import org.apache.skywalking.oap.server.core.query.type.AlarmMessage; +import org.apache.skywalking.oap.server.core.query.type.Alarms; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.JdbcDatabaseContainer; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; + +/** + * Reads alarms the way both alarm queries do on a JDBC storage, through {@link JDBCAlarmQueryDAO}: the legacy + * {@code getAlarm} by scope, keyword and tags, and {@code queryAlarms} by entity, layer, rule name, keyword and tags, + * both newest first and page by page, each alarm with the recovery time {@code alarm_recovery_record} holds for it. A + * subclass names the server. + * + *

The alarms are written as the alarm persistence writes them, on both sides of midnight, so a read crosses two + * day tables: the searchable tags in the tag table and every tag in the raw tags, the recovery in the day of the + * alarm's start. An entity is the primary of some alarms and the destination of a relation alarm, the relation also + * fires the other way round, and for each other filter an alarm matches every condition but that one; a tag kept in + * the raw tags only is not searchable. A page holds a dozen recovered alarms. Alarms of their own lie on and just + * outside the bounds of a duration: on its first millisecond and the one before, on its last millisecond, on the + * instant it ends at ({@link Duration#getEndTimestamp()}, which a read includes) and the millisecond after. + * One more lies on the end instant of a duration ending in a day's last minute: the next midnight, in the next + * day's table. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCAlarmQueryITBase { + private static final long MINUTE = 60_000L; + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + private static final Duration ENDING = minutes(MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE); + /** Yesterday's midnight. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = minutes(PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE); + private static final Duration RECOVERING = minutes(MIDNIGHT + 30 * MINUTE, MIDNIGHT + 30 * MINUTE); + private static final int RECOVERED_IN_ONE_PAGE = 12; + /** The tag keys the IT configures as {@code searchableAlarmTags}. */ + private static final String SEARCHABLE_ALARM_TAGS = "env,team"; + private static final Gson GSON = new Gson(); + + private static final String CHECKOUT = "checkout"; + private static final String PAYMENT = "payment"; + private static final String BOUNDS = "bounds"; + private static final String SLA = "Successful rate of service %s is lower than 80%%"; + private static final String RESP = "Response time of service %s is more than 1000ms"; + + private static final List ALARMS = Arrays.asList( + AlarmRow.service("a1", MIDNIGHT - 150_000L, CHECKOUT, String.format(RESP, CHECKOUT), + "service_resp_time_rule", Layer.GENERAL).tags("env=prod", "owner=alice").recovered() + .snapshot("avg(service_resp_time) > 1000", "service_resp_time", "1200"), + AlarmRow.service("a2", MIDNIGHT - 110_000L, PAYMENT, String.format(SLA, PAYMENT), "service_sla_rule", + Layer.MESH).tags("env=test"), + AlarmRow.endpoint("a3", MIDNIGHT - 70_000L, CHECKOUT, "POST /orders", + "Response time of endpoint POST /orders is more than 1000ms", "endpoint_resp_time_rule", + Layer.GENERAL).tags("env=prod", "team=a").recovered(), + AlarmRow.relation("a4", MIDNIGHT - 30_000L, CHECKOUT, PAYMENT, Layer.GENERAL), + AlarmRow.service("a5", MIDNIGHT + 30_000L, CHECKOUT, String.format(SLA, CHECKOUT), "service_sla_rule", + Layer.GENERAL).tags("env=prod").recovered(), + AlarmRow.instance("a6", MIDNIGHT + 70_000L, CHECKOUT, "checkout-1", + "Response time of service instance checkout-1 is more than 1000ms", + "instance_resp_time_rule", Layer.GENERAL).tags("env=prod", "team=b"), + AlarmRow.service("a7", MIDNIGHT + 110_000L, PAYMENT, "Success count of service payment is lower than 10", + "service_success_rule", Layer.MESH), + AlarmRow.relation("a8", MIDNIGHT + 150_000L, PAYMENT, CHECKOUT, Layer.MESH) + ); + private static final List BOUND_ALARMS = Arrays.asList( + AlarmRow.service("before-start", BOUNDED.getStartTimestamp() - 1, BOUNDS, "bounds", "bounds_rule", + Layer.GENERAL), + AlarmRow.service("on-start", BOUNDED.getStartTimestamp(), BOUNDS, "bounds", "bounds_rule", Layer.GENERAL), + AlarmRow.service("last-millisecond", BOUNDED.getEndTimestamp() - 1, BOUNDS, "bounds", "bounds_rule", + Layer.GENERAL), + AlarmRow.service("after-end", BOUNDED.getEndTimestamp() + 1, BOUNDS, "bounds", "bounds_rule", + Layer.GENERAL), + AlarmRow.service("on-end", ENDING.getEndTimestamp(), BOUNDS, "bounds", "bounds_rule", Layer.GENERAL) + .recovered(), + AlarmRow.service("on-midnight", ENDING_AT_MIDNIGHT.getEndTimestamp(), BOUNDS, "bounds", "bounds_rule", + Layer.GENERAL).recovered() + ); + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCAlarmQueryDAO dao; + + JDBCAlarmQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void writeAlarms() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + final List alarms = new ArrayList<>(ALARMS); + alarms.addAll(BOUND_ALARMS); + for (int i = 0; i < RECOVERED_IN_ONE_PAGE; i++) { + alarms.add(AlarmRow.service(String.format("recovered-%02d", i), RECOVERING.getStartTimestamp() + i * 1000L, + "recovering", "recovering", "recovering_rule", Layer.GENERAL).recovered()); + } + final Model alarmModel = support.install( + AlarmRecord.class, DefaultScopeDefine.ALARM, AlarmRecord.INDEX_NAME, DownSampling.Second); + final Model recoveryModel = support.install( + AlarmRecoveryRecord.class, DefaultScopeDefine.ALARM_RECOVERY, AlarmRecoveryRecord.INDEX_NAME, + DownSampling.Second); + support.writeRecords(alarmModel, new AlarmRecord.Builder(), alarms.stream() + .map(AlarmRow::record).collect(Collectors.toList())); + support.writeRecords(recoveryModel, new AlarmRecoveryRecord.Builder(), alarms.stream() + .filter(alarm -> alarm.recoveryTime != null).map(AlarmRow::recovery).collect(Collectors.toList())); + final ConfigService config = support.getModuleManager().find(CoreModule.NAME).provider() + .getService(ConfigService.class); + when(config.getSearchableAlarmTags()).thenReturn(SEARCHABLE_ALARM_TAGS); + dao = new JDBCAlarmQueryDAO(support.getClient(), support.getModuleManager(), support.getTableHelper()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void pagesThroughEveryAlarmNewestFirst() { + assertEquals(Arrays.asList("a8", "a7", "a6"), uuids(dao.queryAlarms(condition(), 3, 0))); + assertEquals(Arrays.asList("a5", "a4", "a3"), uuids(dao.queryAlarms(condition(), 3, 3))); + assertEquals(Arrays.asList("a2", "a1"), uuids(dao.queryAlarms(condition(), 3, 6))); + assertEquals(Collections.emptyList(), uuids(dao.queryAlarms(condition(), 3, 9))); + } + + /** + * A service is the primary of its own alarms and the destination of a relation alarm. + */ + @Test + public void readsTheAlarmsOfAServiceAsThePrimaryOrTheRelationsDestination() { + assertEquals(Arrays.asList("a8", "a5", "a4", "a1"), uuids(dao.queryAlarms(entities(service(CHECKOUT)), 10, 0))); + assertEquals(Arrays.asList("a8", "a7", "a4", "a2"), uuids(dao.queryAlarms(entities(service(PAYMENT)), 10, 0))); + } + + @Test + public void readsTheAlarmsOfARelationInItsDirectionOnly() { + assertEquals(Collections.singletonList("a4"), uuids(dao.queryAlarms( + entities(relation(CHECKOUT, PAYMENT)), 10, 0))); + } + + @Test + public void readsTheAlarmsOfAnInstanceAndOfAnEndpoint() { + final Entity instance = new Entity(); + instance.setScope(Scope.ServiceInstance); + instance.setServiceName(CHECKOUT); + instance.setServiceInstanceName("checkout-1"); + assertEquals(Collections.singletonList("a6"), uuids(dao.queryAlarms(entities(instance), 10, 0))); + final Entity endpoint = new Entity(); + endpoint.setScope(Scope.Endpoint); + endpoint.setServiceName(CHECKOUT); + endpoint.setEndpointName("POST /orders"); + assertEquals(Collections.singletonList("a3"), uuids(dao.queryAlarms(entities(endpoint), 10, 0))); + } + + @Test + public void readsAnAlarmOnceWhenTwoEntitiesMatchIt() { + assertEquals(Arrays.asList("a8", "a5", "a4", "a1"), uuids(dao.queryAlarms( + entities(service(CHECKOUT), service(CHECKOUT)), 10, 0))); + assertEquals(Arrays.asList("a8", "a5", "a4", "a1"), uuids(dao.queryAlarms( + entities(service(CHECKOUT), relation(CHECKOUT, PAYMENT)), 10, 0))); + } + + @Test + public void pagesThroughTheAlarmsOfAService() { + final AlarmQueryCondition checkout = entities(service(CHECKOUT)); + assertEquals(Arrays.asList("a8", "a5"), uuids(dao.queryAlarms(checkout, 2, 0))); + assertEquals(Arrays.asList("a4", "a1"), uuids(dao.queryAlarms(checkout, 2, 2))); + assertEquals(Collections.emptyList(), uuids(dao.queryAlarms(checkout, 2, 4))); + } + + /** + * {@code Success count} is not the keyword {@code Successful}, and {@code owner} is not a searchable tag. + */ + @Test + public void filtersByLayerRuleNameKeywordAndTags() { + final AlarmQueryCondition mesh = condition(); + mesh.setLayer(Layer.MESH.name()); + assertEquals(Arrays.asList("a8", "a7", "a2"), uuids(dao.queryAlarms(mesh, 10, 0))); + final AlarmQueryCondition sla = condition(); + sla.setRuleNames(Collections.singletonList("service_sla_rule")); + assertEquals(Arrays.asList("a5", "a2"), uuids(dao.queryAlarms(sla, 10, 0))); + final AlarmQueryCondition responseTime = condition(); + responseTime.setRuleNames(Arrays.asList("service_resp_time_rule", "endpoint_resp_time_rule")); + assertEquals(Arrays.asList("a3", "a1"), uuids(dao.queryAlarms(responseTime, 10, 0))); + final AlarmQueryCondition keyword = condition(); + keyword.setKeyword("Successful"); + assertEquals(Arrays.asList("a5", "a2"), uuids(dao.queryAlarms(keyword, 10, 0))); + final AlarmQueryCondition prod = condition(); + prod.setTags(tags("env=prod")); + assertEquals(Arrays.asList("a6", "a5", "a3", "a1"), uuids(dao.queryAlarms(prod, 10, 0))); + final AlarmQueryCondition teamA = condition(); + teamA.setTags(tags("env=prod", "team=a")); + assertEquals(Collections.singletonList("a3"), uuids(dao.queryAlarms(teamA, 10, 0))); + final AlarmQueryCondition owner = condition(); + owner.setTags(tags("owner=alice")); + assertEquals(Collections.emptyList(), uuids(dao.queryAlarms(owner, 10, 0))); + } + + @Test + public void readsTheLegacyAlarmListByScopeKeywordAndTags() { + assertEquals(Arrays.asList("a7", "a5", "a2", "a1"), uuids(legacy(DefaultScopeDefine.SERVICE, null, null))); + assertEquals(Collections.singletonList("a6"), uuids(legacy(DefaultScopeDefine.SERVICE_INSTANCE, null, null))); + assertEquals(Collections.singletonList("a3"), uuids(legacy(DefaultScopeDefine.ENDPOINT, null, null))); + assertEquals(Arrays.asList("a8", "a4"), uuids(legacy(DefaultScopeDefine.SERVICE_RELATION, null, null))); + assertEquals(Arrays.asList("a5", "a2"), uuids(legacy(null, "Successful", null))); + assertEquals(Arrays.asList("a6", "a5", "a3", "a1"), uuids(legacy(null, null, tags("env=prod")))); + assertEquals(Collections.singletonList("a3"), uuids(legacy(null, null, tags("env=prod", "team=a")))); + assertEquals(Collections.emptyList(), uuids(legacy(null, null, tags("owner=alice")))); + } + + @Test + public void pagesThroughTheLegacyAlarmList() { + assertEquals(Arrays.asList("a8", "a7", "a6"), uuids(dao.getAlarm(null, null, 3, 0, AROUND_MIDNIGHT, null))); + assertEquals(Arrays.asList("a5", "a4", "a3"), uuids(dao.getAlarm(null, null, 3, 3, AROUND_MIDNIGHT, null))); + assertEquals(Arrays.asList("a2", "a1"), uuids(dao.getAlarm(null, null, 3, 6, AROUND_MIDNIGHT, null))); + assertEquals(Collections.emptyList(), uuids(dao.getAlarm(null, null, 3, 9, AROUND_MIDNIGHT, null))); + } + + @Test + public void readsEachAlarmWhole() { + final List expected = new ArrayList<>(); + for (int i = ALARMS.size() - 1; i >= 0; i--) { + expected.add(ALARMS.get(i).render()); + } + assertEquals(expected, render(dao.queryAlarms(condition(), 10, 0))); + assertEquals(expected, render(dao.getAlarm(null, null, 10, 0, AROUND_MIDNIGHT, null))); + } + + @Test + public void readsTheRecoveryTimeOfEveryAlarmOfAPage() { + final AlarmQueryCondition recovering = condition(); + recovering.setDuration(RECOVERING); + final List expected = new ArrayList<>(); + for (int i = RECOVERED_IN_ONE_PAGE - 1; i >= 0; i--) { + final long startTime = RECOVERING.getStartTimestamp() + i * 1000L; + expected.add(String.format("recovered-%02d", i) + "|" + (startTime + MINUTE)); + } + assertEquals(expected, recoveries(dao.queryAlarms(recovering, 20, 0))); + assertEquals(expected, recoveries(dao.getAlarm(null, null, 20, 0, RECOVERING, null))); + } + + @Test + public void keepsTheAlarmsWithinTheDuration() { + final AlarmQueryCondition bounded = entities(service(BOUNDS)); + bounded.setDuration(BOUNDED); + assertEquals(Arrays.asList("last-millisecond", "on-start"), uuids(dao.queryAlarms(bounded, 10, 0))); + assertEquals(Arrays.asList("last-millisecond", "on-start"), uuids(dao.getAlarm( + DefaultScopeDefine.SERVICE, null, 10, 0, BOUNDED, null))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's table; an + * alarm's recovery is stored in the time bucket of its start. + */ + @Test + public void readsAnAlarmOnTheInstantTheDurationEndsAt() { + for (final Duration duration : Arrays.asList(ENDING, ENDING_AT_MIDNIGHT)) { + final long startTime = duration.getEndTimestamp(); + final List expected = Collections.singletonList( + (duration == ENDING ? "on-end" : "on-midnight") + "|" + (startTime + MINUTE)); + final AlarmQueryCondition ending = entities(service(BOUNDS)); + ending.setDuration(duration); + assertEquals(expected, recoveries(dao.queryAlarms(ending, 10, 0))); + assertEquals(expected, recoveries(dao.getAlarm(DefaultScopeDefine.SERVICE, null, 10, 0, duration, null))); + } + } + + private Alarms legacy(final Integer scopeId, final String keyword, final List tags) { + return dao.getAlarm(scopeId, keyword, 10, 0, AROUND_MIDNIGHT, tags); + } + + private static AlarmQueryCondition condition() { + final AlarmQueryCondition condition = new AlarmQueryCondition(); + condition.setDuration(AROUND_MIDNIGHT); + return condition; + } + + private static AlarmQueryCondition entities(final Entity... entities) { + final AlarmQueryCondition condition = condition(); + condition.setEntities(Arrays.asList(entities)); + return condition; + } + + private static Entity service(final String name) { + final Entity service = new Entity(); + service.setScope(Scope.Service); + service.setServiceName(name); + service.setNormal(true); + return service; + } + + private static Entity relation(final String source, final String dest) { + final Entity relation = new Entity(); + relation.setScope(Scope.ServiceRelation); + relation.setServiceName(source); + relation.setDestServiceName(dest); + return relation; + } + + private static List tags(final String... tags) { + return Arrays.stream(tags).map(tag -> tag.split("=")).map(kv -> new Tag(kv[0], kv[1])) + .collect(Collectors.toList()); + } + + private static List uuids(final Alarms alarms) { + return alarms.getMsgs().stream().map(AlarmMessage::getUuid).collect(Collectors.toList()); + } + + private static List recoveries(final Alarms alarms) { + return alarms.getMsgs().stream().map(alarm -> alarm.getUuid() + "|" + alarm.getRecoveryTime()) + .collect(Collectors.toList()); + } + + private static List render(final Alarms alarms) { + return alarms.getMsgs().stream().map(alarm -> String.join( + "|", alarm.getUuid(), String.valueOf(alarm.getScopeId()), alarm.getId(), alarm.getId1(), alarm.getName(), + alarm.getMessage(), String.valueOf(alarm.getStartTime()), String.valueOf(alarm.getRecoveryTime()), + alarm.getTags().stream().map(tag -> tag.getKey() + "=" + tag.getValue()).collect(Collectors.joining(",")), + alarm.getSnapshot().getExpression() + ":" + alarm.getSnapshot().getMetrics().stream() + .map(metric -> metric.getName() + "=" + metric.getResults().stream() + .flatMap(values -> values.getValues().stream()).map(MQEValue::getValue) + .collect(Collectors.joining(","))) + .collect(Collectors.joining(";")) + )).collect(Collectors.toList()); + } + + private static Duration minutes(final long start, final long end) { + return JDBCQueryITSupport.duration(start, end, Step.MINUTE); + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + /** + * An alarm as the alarm persistence hands it to the record stream; a recovered one recovers a minute after it + * starts. + */ + private static final class AlarmRow { + private static final List SEARCHABLE_TAG_KEYS = Arrays.asList(SEARCHABLE_ALARM_TAGS.split(",")); + + private final String uuid; + private final long startTime; + private final int scope; + private final String id0; + private final String id1; + private final String name; + private final String message; + private final String ruleName; + private final Layer layer; + private List tags = Collections.emptyList(); + private Long recoveryTime; + private AlarmSnapshotRecord snapshot = new AlarmSnapshotRecord(); + + private AlarmRow(final String uuid, final long startTime, final int scope, final String id0, final String id1, + final String name, final String message, final String ruleName, final Layer layer) { + this.uuid = uuid; + this.startTime = startTime; + this.scope = scope; + this.id0 = id0; + this.id1 = id1; + this.name = name; + this.message = message; + this.ruleName = ruleName; + this.layer = layer; + } + + static AlarmRow service(final String uuid, final long startTime, final String service, final String message, + final String ruleName, final Layer layer) { + return new AlarmRow(uuid, startTime, DefaultScopeDefine.SERVICE, serviceId(service), "", service, + message, ruleName, layer); + } + + static AlarmRow instance(final String uuid, final long startTime, final String service, final String instance, + final String message, final String ruleName, final Layer layer) { + return new AlarmRow(uuid, startTime, DefaultScopeDefine.SERVICE_INSTANCE, + IDManager.ServiceInstanceID.buildId(serviceId(service), instance), "", + instance + " of " + service, message, ruleName, layer); + } + + static AlarmRow endpoint(final String uuid, final long startTime, final String service, final String endpoint, + final String message, final String ruleName, final Layer layer) { + return new AlarmRow(uuid, startTime, DefaultScopeDefine.ENDPOINT, + IDManager.EndpointID.buildId(serviceId(service), endpoint), "", + endpoint + " in " + service, message, ruleName, layer); + } + + static AlarmRow relation(final String uuid, final long startTime, final String source, final String dest, + final Layer layer) { + return new AlarmRow(uuid, startTime, DefaultScopeDefine.SERVICE_RELATION, serviceId(source), + serviceId(dest), source + " to " + dest, + "Response time of service " + source + " to service " + dest + " is more than 1000ms", + "service_relation_resp_time_rule", layer); + } + + AlarmRow tags(final String... tags) { + this.tags = Arrays.asList(tags); + return this; + } + + AlarmRow recovered() { + this.recoveryTime = startTime + MINUTE; + return this; + } + + AlarmRow snapshot(final String expression, final String metric, final String value) { + final MQEValue point = new MQEValue(); + point.setId(String.valueOf(TimeBucket.getMinuteTimeBucket(startTime))); + point.setValue(value); + point.setDoubleValue(Double.parseDouble(value)); + final MQEValues values = new MQEValues(); + values.getValues().add(point); + final JsonObject metrics = new JsonObject(); + metrics.addProperty(metric, GSON.toJson(Collections.singletonList(values))); + snapshot.setExpression(expression); + snapshot.setMetrics(metrics); + return this; + } + + AlarmRecord record() { + final AlarmRecord record = new AlarmRecord(); + record.setUuid(uuid); + record.setScope(scope); + record.setId0(id0); + record.setId1(id1); + record.setName(name); + record.setAlarmMessage(message); + record.setStartTime(startTime); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(startTime)); + record.setRuleName(ruleName); + record.setLayer(layer); + record.setTagsRawData(GSON.toJson(tags.stream().map(tag -> tag.split("=")) + .map(kv -> new Tag(kv[0], kv[1])).collect(Collectors.toList())) + .getBytes(StandardCharsets.UTF_8)); + record.setTagsInString(tags.stream().filter(tag -> SEARCHABLE_TAG_KEYS.contains(tag.split("=")[0])) + .collect(Collectors.toList())); + record.setSnapshot(GSON.toJson(snapshot)); + return record; + } + + AlarmRecoveryRecord recovery() { + final AlarmRecoveryRecord recovery = new AlarmRecoveryRecord(); + recovery.setUuid(uuid); + recovery.setRecoveryTime(recoveryTime); + recovery.setTimeBucket(TimeBucket.getRecordTimeBucket(startTime)); + return recovery; + } + + String render() { + final String metrics = snapshot.getMetrics() == null ? "" : snapshot.getMetrics().keySet().stream() + .map(metric -> metric + "=" + GSON.fromJson(snapshot.getMetrics().get(metric).getAsString(), + MQEValues[].class)[0].getValues().get(0).getValue()) + .collect(Collectors.joining(";")); + return String.join("|", uuid, String.valueOf(scope), id0, id1, name, message, String.valueOf(startTime), + String.valueOf(recoveryTime), String.join(",", tags), + snapshot.getExpression() + ":" + metrics); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch9IT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryMariaDBIT.java similarity index 60% rename from oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch9IT.java rename to oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryMariaDBIT.java index 7ccd47e3f450..8d35d048227e 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch9IT.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryMariaDBIT.java @@ -16,22 +16,18 @@ * */ -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin; +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.containers.MariaDBContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -@Tag("slow") -@Tag("elastic-it") @Testcontainers -public class ZipkinQueryEsDAOElasticsearch9IT extends ZipkinQueryEsDAOITBase { +public class JDBCAlarmQueryMariaDBIT extends JDBCAlarmQueryITBase { @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("9.3.0"); + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); - public ZipkinQueryEsDAOElasticsearch9IT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.DEFAULT); + public JDBCAlarmQueryMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch8IT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryPostgreSQLIT.java similarity index 60% rename from oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch8IT.java rename to oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryPostgreSQLIT.java index 718c31adb0d5..ff762d78e4a1 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/zipkin/ZipkinQueryEsDAOElasticsearch8IT.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCAlarmQueryPostgreSQLIT.java @@ -16,22 +16,18 @@ * */ -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query.zipkin; +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -@Tag("slow") -@Tag("elastic-it") @Testcontainers -public class ZipkinQueryEsDAOElasticsearch8IT extends ZipkinQueryEsDAOITBase { +public class JDBCAlarmQueryPostgreSQLIT extends JDBCAlarmQueryITBase { @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.elasticsearch("8.18.8"); + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); - public ZipkinQueryEsDAOElasticsearch8IT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.DEFAULT); + public JDBCAlarmQueryPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOITBase.java new file mode 100644 index 000000000000..a1291eea6a6d --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOITBase.java @@ -0,0 +1,402 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; +import org.apache.skywalking.oap.server.core.analysis.metrics.LongAvgMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.analysis.worker.MetricsSessionCache; +import org.apache.skywalking.oap.server.core.management.ui.template.UITemplate; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingDataRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTargetType; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTriggerType; +import org.apache.skywalking.oap.server.core.query.input.DashboardSetting; +import org.apache.skywalking.oap.server.core.query.type.DashboardConfiguration; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; +import org.apache.skywalking.oap.server.core.storage.IRecordDAO; +import org.apache.skywalking.oap.server.core.storage.SessionCacheCallback; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; +import org.apache.skywalking.oap.server.library.batchqueue.BatchQueueManager; +import org.apache.skywalking.oap.server.library.client.request.PrepareRequest; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCStorageConfig; +import org.joda.time.DateTime; +import org.joda.time.LocalDate; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.JdbcDatabaseContainer; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.testcontainers.shaded.org.awaitility.Awaitility.await; + +/** + * Writes through OAP's batch DAO and the write DAOs the storage DAO hands out, built as the storage provider builds + * them, the way OAP's workers use them, and reads each write back through its query DAO: records one by one through + * {@code insert}, as a record worker hands them over, and a persistence round's rows through {@code flush}; a task row + * at once through the none-stream DAO, as a task worker writes it; and a management row through the management DAO, + * which inserts a row only when its id is new. Metrics are persisted the way the metrics workers persist them: a + * persistence round's metrics are inserted and cached once the database has answered, the next round reads the stored + * metrics back by id ({@code multiGet}) and writes the combined ones as updates, and a cached metric expires with the + * day table the TTL drops ({@code isExpiredCache}). The records and the metrics lie on both sides of midnight, so in + * two day tables, and the records are more than a SQL batch. Beside the metrics, a metric of another name of the same + * function, so in the same table, has the same entities and minutes, and a metadata metric, whose id has no time + * bucket, is read with a later minute than it is stored with. A subclass names the server. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCBatchDAOITBase { + /** The name {@link JDBCBatchDAO} registers its queue under, one per JVM. */ + private static final String BATCH_QUEUE = "JDBC_ASYNC_BATCH_PERSISTENT"; + /** Fewer statements per SQL batch than the rows written to one day table. */ + private static final int MAX_SIZE_OF_BATCH_SQL = 3; + private static final int ROWS = 10; + private static final String RESP_TIME = "service_resp_time"; + private static final String OTHER_RESP_TIME = "service_relation_server_resp_time"; + private static final String SERVICE_ID = IDManager.ServiceID.buildId("checkout", true); + private static final String OTHER_SERVICE_ID = IDManager.ServiceID.buildId("payment", true); + private static final int[] MINUTES = {-1, 0, 1}; + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCBatchDAO batchDAO; + private StorageDAO storageDAO; + private Model dataModel; + private Model taskModel; + private Model templateModel; + private JDBCEBPFProfilingDataDAO dataDAO; + private Model respTime; + private Model traffic; + private StorageBuilder respTimeBuilder; + + JDBCBatchDAOITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void install() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + dataModel = support.install( + EBPFProfilingDataRecord.class, DefaultScopeDefine.EBPF_PROFILING_DATA, EBPFProfilingDataRecord.INDEX_NAME, + DownSampling.Second); + taskModel = support.install( + EBPFProfilingTaskRecord.class, DefaultScopeDefine.EBPF_PROFILING_TASK, EBPFProfilingTaskRecord.INDEX_NAME, + DownSampling.Minute); + templateModel = support.install( + UITemplate.class, DefaultScopeDefine.UI_TEMPLATE, UITemplate.INDEX_NAME, DownSampling.None); + batchDAO = new JDBCBatchDAO( + support.getClient(), MAX_SIZE_OF_BATCH_SQL, new JDBCStorageConfig().getAsyncBatchPersistentPoolSize()); + storageDAO = new JDBCStorageDAO(support.getClient()); + dataDAO = new JDBCEBPFProfilingDataDAO(support.getClient(), support.getTableHelper()); + + final Class respTimeClass = OALTestMetrics.metricsClass(RESP_TIME); + final Class otherRespTimeClass = OALTestMetrics.metricsClass(OTHER_RESP_TIME); + respTime = support.install( + respTimeClass, OALTestMetrics.scopeId(respTimeClass), RESP_TIME, DownSampling.Minute); + final Model otherRespTime = support.install( + otherRespTimeClass, OALTestMetrics.scopeId(otherRespTimeClass), OTHER_RESP_TIME, DownSampling.Minute); + traffic = support.install( + ServiceTraffic.class, DefaultScopeDefine.SERVICE, ServiceTraffic.INDEX_NAME, DownSampling.Minute); + respTimeBuilder = OALTestMetrics.builder(respTimeClass); + final List sameEntities = new ArrayList<>(); + for (final int minute : MINUTES) { + for (final String entityId : Arrays.asList(SERVICE_ID, OTHER_SERVICE_ID)) { + sameEntities.add(respTime(OTHER_RESP_TIME, entityId, minute, 7L)); + } + } + support.writeMetrics(otherRespTime, OALTestMetrics.builder(otherRespTimeClass), sameEntities); + } + + @AfterAll + public void close() { + BatchQueueManager.shutdown(BATCH_QUEUE); + if (support != null) { + support.close(); + } + } + + /** + * {@code insert} only queues a row; the queue's consumers write it shortly after. + */ + @Test + public void insertsRecordsThroughItsQueue() throws Exception { + final List rows = rows("insert-schedule"); + final IRecordDAO recordDAO = storageDAO.newRecordDao(new EBPFProfilingDataRecord.Builder()); + for (final EBPFProfilingDataRecord row : rows) { + batchDAO.insert(recordDAO.prepareBatchInsert(dataModel, row)); + } + + await().atMost(30, TimeUnit.SECONDS).untilAsserted( + () -> assertEquals(describe(rows), read("insert-schedule"))); + } + + /** + * The persistence timer waits for a round's flush before the next round, so the rows are written when it + * answers. + */ + @Test + public void flushesTheRowsOfARoundBeforeItAnswers() throws Exception { + final List rows = rows("flush-schedule"); + final IRecordDAO recordDAO = storageDAO.newRecordDao(new EBPFProfilingDataRecord.Builder()); + final List round = new ArrayList<>(); + for (final EBPFProfilingDataRecord row : rows) { + round.add(recordDAO.prepareBatchInsert(dataModel, row)); + } + + batchDAO.flush(round).get(30, TimeUnit.SECONDS); + + assertEquals(describe(rows), read("flush-schedule")); + } + + /** + * A task is created at once, so the profiling page lists it right after; the none-stream DAO writes it into the + * table of the day it is written on. + */ + @Test + public void writesATaskRowAtOnceThroughTheNoneStreamDAO() throws Exception { + final long now = System.currentTimeMillis(); + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(IDManager.ServiceID.buildId("batch-it", true)); + task.setProcessLabelsJson(Const.EMPTY_STRING); + task.setStartTime(now); + task.setTriggerType(EBPFProfilingTriggerType.FIXED_TIME.value()); + task.setFixedTriggerDuration(60); + task.setTargetType(EBPFProfilingTargetType.OFF_CPU.value()); + task.setCreateTime(now); + task.setLastUpdateTime(now); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(now)); + task.generateLogicalId(); + task.setExtensionConfigJson(Const.EMPTY_STRING); + + storageDAO.newNoneStreamDao(new EBPFProfilingTaskRecord.Builder()).insert(taskModel, task); + + assertEquals( + Collections.singletonList(describe(task)), + new JDBCEBPFProfilingTaskDAO(support.getClient(), support.getTableHelper()) + .getTaskRecord(task.getLogicalId()).stream() + .map(JDBCBatchDAOITBase::describe) + .collect(Collectors.toList())); + } + + @Test + public void insertsAManagementRowOnlyWhenItsIdIsNew() throws Exception { + final UITemplate first = template("batch-it-template", "{\"version\":1}"); + final UITemplate second = template("batch-it-template", "{\"version\":2}"); + final UITemplate other = template("batch-it-other-template", "{\"version\":1}"); + for (final UITemplate template : new UITemplate[] {first, second, other}) { + storageDAO.newManagementDao(new UITemplate.Builder()).insert(templateModel, template); + } + + final JDBCUITemplateManagementDAO dao = + new JDBCUITemplateManagementDAO(support.getClient(), support.getTableHelper()); + assertEquals("batch-it-template|{\"version\":1}|false", describe(dao.getTemplate("batch-it-template"))); + assertEquals("batch-it-other-template|{\"version\":1}|false", + describe(dao.getTemplate("batch-it-other-template"))); + } + + /** + * The metrics worker caches each metric of a round once its insert is answered, reads back by id the metrics it + * does not hold, and writes a metric combined with what came in since as an update of the stored one. + */ + @Test + public void writesTheMetricsOfAPersistenceRoundAndReadsThemBackById() throws Exception { + final IMetricsDAO dao = storageDAO.newMetricsDao(respTimeBuilder); + final MetricsSessionCache sessionCache = new MetricsSessionCache(TimeUnit.MINUTES.toMillis(1), true); + final List inserted = new ArrayList<>(); + final List round = new ArrayList<>(); + for (final int minute : MINUTES) { + for (final String entityId : Arrays.asList(SERVICE_ID, OTHER_SERVICE_ID)) { + final Metrics metrics = respTime(RESP_TIME, entityId, minute, 100L + minute); + inserted.add(metrics); + round.add(dao.prepareBatchInsert(respTime, metrics, new SessionCacheCallback(sessionCache, metrics))); + } + } + batchDAO.flush(round).get(30, TimeUnit.SECONDS); + + assertEquals(columns(inserted), columns(inserted.stream().map(sessionCache::remove) + .collect(Collectors.toList())), + "every metric of the round is cached once the database has answered"); + final List asked = new ArrayList<>(inserted); + asked.add(respTime(RESP_TIME, IDManager.ServiceID.buildId("never-written", true), 0, 1L)); + assertEquals(columns(inserted), columns(dao.multiGet(respTime, asked))); + + final Metrics combined = respTime(RESP_TIME, SERVICE_ID, 0, 100L, 300L); + batchDAO.flush(Collections.singletonList(dao.prepareBatchUpdate(respTime, combined, null))) + .get(30, TimeUnit.SECONDS); + final List afterUpdate = inserted.stream() + .map(metrics -> metrics.id().build().equals(combined.id().build()) ? combined : metrics) + .collect(Collectors.toList()); + assertEquals(columns(afterUpdate), columns(dao.multiGet(respTime, inserted))); + } + + /** + * A metadata metric has one row for all time. The metrics worker asks for it with the minute it sees it in again, + * not the minute it was stored with. + */ + @Test + @SuppressWarnings({"unchecked", "rawtypes"}) + public void readsAMetadataMetricBackWhicheverDayItIsStoredIn() throws Exception { + final StorageBuilder builder = (StorageBuilder) new ServiceTraffic.Builder(); + final ServiceTraffic stored = service("checkout", Layer.GENERAL, -1); + support.writeMetrics(traffic, builder, Arrays.asList(stored, service("checkout", Layer.MESH, -1))); + + final List read = storageDAO.newMetricsDao(builder).multiGet(traffic, Arrays.asList( + service("checkout", Layer.GENERAL, 1), service("never-seen", Layer.GENERAL, 1))); + + assertEquals(columns(builder, Collections.singletonList(stored)), columns(builder, read)); + } + + /** + * The TTL drops every day table before the first one it keeps, so a metric of the day before is expired however + * late in that day it is, and a metric of the first kept day is not, however early. + */ + @Test + public void expiresAMetricWithTheDayTableTheTTLDrops() throws Exception { + final IMetricsDAO dao = storageDAO.newMetricsDao(respTimeBuilder); + final long now = new DateTime(MIDNIGHT).withHourOfDay(10).getMillis(); + // The first kept day as JDBCHistoryDeleteDAO counts it: from elapsed time, not calendar days. + final LocalDate firstKeptDay = + new LocalDate(now - TimeUnit.DAYS.toMillis(JDBCQueryITSupport.METRICS_TTL_DAYS - 1)); + final long firstKeptDayStart = firstKeptDay.toDateTimeAtStartOfDay().getMillis(); + assertEquals( + Arrays.asList(true, true, false, false), + Arrays.asList( + expired(dao, firstKeptDay.minusDays(1).toDateTimeAtStartOfDay().getMillis(), now), + expired(dao, firstKeptDayStart - TimeUnit.MINUTES.toMillis(1), now), + expired(dao, firstKeptDayStart, now), + expired(dao, now, now))); + } + + private boolean expired(final IMetricsDAO dao, final long metricTime, final long now) throws Exception { + final Metrics metrics = OALTestMetrics.newMetrics( + OALTestMetrics.metricsClass(RESP_TIME), "entityId", SERVICE_ID, + "timeBucket", TimeBucket.getMinuteTimeBucket(metricTime)); + return dao.isExpiredCache(respTime, metrics, now, JDBCQueryITSupport.METRICS_TTL_DAYS); + } + + private List columns(final List metrics) { + return columns(respTimeBuilder, metrics); + } + + /** + * Every column of each metric, as the metrics workers store and read them, in no order. + */ + private static List columns(final StorageBuilder builder, final List metrics) { + return metrics.stream().map(metric -> { + final HashMapConverter.ToStorage columns = new HashMapConverter.ToStorage(); + builder.entity2Storage(metric, columns); + return new TreeMap<>(columns.obtain()).toString(); + }).sorted().collect(Collectors.toList()); + } + + /** + * The response time of a minute of an entity, one call of each latency. + */ + private static Metrics respTime(final String metric, final String entityId, final int minute, + final long... latencies) throws Exception { + final LongAvgMetrics metrics = (LongAvgMetrics) OALTestMetrics.newMetrics( + OALTestMetrics.metricsClass(metric), "entityId", entityId, + "timeBucket", TimeBucket.getMinuteTimeBucket(MIDNIGHT + TimeUnit.MINUTES.toMillis(minute))); + for (final long latency : latencies) { + metrics.combine(latency, 1); + } + metrics.calculate(); + return metrics; + } + + private static ServiceTraffic service(final String name, final Layer layer, final int minute) { + final ServiceTraffic service = new ServiceTraffic(); + service.setName(name); + service.setLayer(layer); + service.setTimeBucket(TimeBucket.getMinuteTimeBucket(MIDNIGHT + TimeUnit.MINUTES.toMillis(minute))); + return service; + } + + private List read(final String scheduleId) throws Exception { + return describe(dataDAO.queryData( + Collections.singletonList(scheduleId), MIDNIGHT - 10_000L, MIDNIGHT + 10_000L)); + } + + /** + * Uploads of a schedule a second apart, half of them before midnight. + */ + private static List rows(final String scheduleId) { + final List rows = new ArrayList<>(); + for (int i = 0; i < ROWS; i++) { + final long uploadTime = MIDNIGHT + (i - ROWS / 2) * 1_000L; + final EBPFProfilingDataRecord row = new EBPFProfilingDataRecord(); + row.setScheduleId(scheduleId); + row.setTaskId("batch-it-task"); + row.setStackIdList("stack-" + i); + row.setTargetType(EBPFProfilingTargetType.ON_CPU.value()); + row.setDataBinary(("stack " + i).getBytes()); + row.setUploadTime(uploadTime); + row.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime)); + rows.add(row); + } + return rows; + } + + private static List describe(final List rows) { + return rows.stream() + .map(row -> String.join("|", row.getScheduleId(), row.getTaskId(), row.getStackIdList(), + String.valueOf(row.getTargetType()), + Base64.getEncoder().encodeToString(row.getDataBinary()), + String.valueOf(row.getUploadTime()))) + .sorted() + .collect(Collectors.toList()); + } + + private static String describe(final EBPFProfilingTaskRecord task) { + final HashMapConverter.ToStorage columns = new HashMapConverter.ToStorage(); + new EBPFProfilingTaskRecord.Builder().entity2Storage(task, columns); + return new TreeMap<>(columns.obtain()).toString(); + } + + private static String describe(final DashboardConfiguration template) { + return template.getId() + "|" + template.getConfiguration() + "|" + template.isDisabled(); + } + + private static UITemplate template(final String id, final String configuration) { + final DashboardSetting setting = new DashboardSetting(); + setting.setId(id); + setting.setConfiguration(configuration); + return setting.toEntity(); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOMariaDBIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOMariaDBIT.java new file mode 100644 index 000000000000..d6f38cd9668d --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOMariaDBIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCBatchDAOMariaDBIT extends JDBCBatchDAOITBase { + @Container + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); + + public JDBCBatchDAOMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOPostgreSQLIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOPostgreSQLIT.java new file mode 100644 index 000000000000..741fe5433323 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBatchDAOPostgreSQLIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCBatchDAOPostgreSQLIT extends JDBCBatchDAOITBase { + @Container + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); + + public JDBCBatchDAOPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBrowserLogQueryDAOTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBrowserLogQueryDAOTest.java index dffae1870552..abf7bea9de09 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBrowserLogQueryDAOTest.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCBrowserLogQueryDAOTest.java @@ -71,7 +71,7 @@ void buildSQL_withNoOptionalConditions_shouldProduceMinimalSQL() { assertThat(result.sql()).doesNotContain(BrowserErrorLogRecord.SERVICE_VERSION_ID); assertThat(result.sql()).doesNotContain(BrowserErrorLogRecord.PAGE_PATH_ID); assertThat(result.sql()).doesNotContain(BrowserErrorLogRecord.ERROR_CATEGORY); - assertThat(result.sql()).doesNotContain(BrowserErrorLogRecord.TIME_BUCKET); + assertThat(result.sql()).doesNotContain(BrowserErrorLogRecord.TIMESTAMP + " >= ?"); } @Test @@ -108,7 +108,7 @@ void buildSQL_withCategory_shouldIncludeCategoryCondition() { } @Test - void buildSQL_withDuration_shouldIncludeTimeBucketRange() { + void buildSQL_withDuration_shouldIncludeTheTimestampsThroughTheEndInstant() { final Duration duration = new Duration(); duration.setStart(new DateTime(2023, 1, 1, 0, 0).toString("yyyy-MM-dd HHmm")); duration.setEnd(new DateTime(2023, 1, 2, 0, 0).toString("yyyy-MM-dd HHmm")); @@ -116,8 +116,9 @@ void buildSQL_withDuration_shouldIncludeTimeBucketRange() { final SQLAndParameters result = dao.buildSQL(null, null, null, null, duration, 10, 0, TABLE); - assertThat(result.sql()).contains(BrowserErrorLogRecord.TIME_BUCKET + " >= ?"); - assertThat(result.sql()).contains(BrowserErrorLogRecord.TIME_BUCKET + " <= ?"); + assertThat(result.sql()).contains(BrowserErrorLogRecord.TIMESTAMP + " >= ?"); + assertThat(result.sql()).contains(BrowserErrorLogRecord.TIMESTAMP + " <= ?"); + assertThat(result.parameters()).contains(duration.getStartTimestamp(), duration.getEndTimestamp()); } @Test diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryITBase.java new file mode 100644 index 000000000000..9a66c8d68867 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryITBase.java @@ -0,0 +1,478 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import com.google.gson.Gson; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingDataRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingScheduleRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTargetType; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTriggerType; +import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingSchedule; +import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingTaskContinuousProfiling; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.type.HashMapConverter; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.JdbcDatabaseContainer; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads eBPF profiling tasks, schedules and data the way the eBPF receiver, the profiling query and mutation + * services and the profiling analysis do, from tables OAP installed and rows OAP's write paths stored. A subclass + * names the server. + * + *

The tasks are created on both sides of midnight, so in two day tables: fixed-time tasks of two services, a + * network task of an instance with the copy that extends it after midnight (a second row with the same logical id), + * and continuous-profiling tasks of two instances. Each filter has a near miss, and the start and update times lie + * on and just outside the bounds asked for. The schedules of a task start on both sides of midnight beside a + * schedule of another task. The eBPF data of a schedule is uploaded every half second through a window that spans + * midnight, with uploads just before the window and on its end, beside the uploads of other schedules. + * + *

The task and schedule DAOs promise no order, and their callers combine and sort the rows themselves, so rows + * are compared regardless of order; the schedules a task's page lists are latest start first. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCEBPFProfilingQueryITBase { + private static final Gson GSON = new Gson(); + private static final String SERVICE_ID = IDManager.ServiceID.buildId("checkout", true); + private static final String OTHER_SERVICE_ID = IDManager.ServiceID.buildId("payment", true); + private static final String INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-1"); + private static final String OTHER_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-2"); + + private static final EBPFProfilingTaskRecord ON_CPU = fixedTimeTask( + SERVICE_ID, Collections.emptyList(), EBPFProfilingTargetType.ON_CPU, minutes(-30), minutes(-30)); + /** Created five minutes after midnight to start five minutes later, for processes with a label. */ + private static final EBPFProfilingTaskRecord OFF_CPU = fixedTimeTask( + SERVICE_ID, Collections.singletonList("env:prod"), EBPFProfilingTargetType.OFF_CPU, minutes(5), minutes(10)); + private static final EBPFProfilingTaskRecord OTHER_SERVICE_ON_CPU = fixedTimeTask( + OTHER_SERVICE_ID, Collections.emptyList(), EBPFProfilingTargetType.ON_CPU, minutes(-10), minutes(-10)); + private static final EBPFProfilingTaskRecord NETWORK = networkTask(INSTANCE_ID, minutes(-20)); + private static final EBPFProfilingTaskRecord NETWORK_EXTENDED = extended(NETWORK, minutes(1)); + private static final EBPFProfilingTaskRecord CONTINUOUS_ON_CPU = continuousProfilingTask( + INSTANCE_ID, EBPFProfilingTargetType.ON_CPU, minutes(-5)); + private static final EBPFProfilingTaskRecord CONTINUOUS_OFF_CPU = continuousProfilingTask( + OTHER_INSTANCE_ID, EBPFProfilingTargetType.OFF_CPU, minutes(3)); + private static final List TASKS = Arrays.asList( + ON_CPU, OFF_CPU, OTHER_SERVICE_ON_CPU, NETWORK, NETWORK_EXTENDED, CONTINUOUS_ON_CPU, CONTINUOUS_OFF_CPU); + private static final List ALL_TARGETS = Arrays.asList(EBPFProfilingTargetType.values()); + private static final List CPU = + Arrays.asList(EBPFProfilingTargetType.ON_CPU, EBPFProfilingTargetType.OFF_CPU); + + private static final EBPFProfilingScheduleRecord FIRST_SCHEDULE = + schedule(NETWORK.getLogicalId(), INSTANCE_ID, "checkout", minutes(-15), minutes(-14)); + /** Started before midnight and still running after it, so stored in today's table. */ + private static final EBPFProfilingScheduleRecord ACROSS_MIDNIGHT_SCHEDULE = + schedule(NETWORK.getLogicalId(), INSTANCE_ID, "checkout-worker", minutes(-3), minutes(2)); + private static final EBPFProfilingScheduleRecord LAST_SCHEDULE = + schedule(NETWORK.getLogicalId(), INSTANCE_ID, "checkout", minutes(5), minutes(6)); + private static final EBPFProfilingScheduleRecord OTHER_TASK_SCHEDULE = + schedule(ON_CPU.getLogicalId(), INSTANCE_ID, "checkout", minutes(-1), minutes(0)); + + private static final String SCHEDULE = "schedule-read"; + private static final String OTHER_SCHEDULE = "schedule-also-read"; + private static final String UNASKED_SCHEDULE = "schedule-not-read"; + private static final long WINDOW_BEGIN = MIDNIGHT - 5_000L; + private static final long WINDOW_END = MIDNIGHT + 5_000L; + private static final int UPLOADS = 20; + private static final int STACKS_PER_UPLOAD = 3; + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCEBPFProfilingTaskDAO taskDAO; + private JDBCEBPFProfilingScheduleDAO scheduleDAO; + private JDBCEBPFProfilingDataDAO dataDAO; + private final List windowOfSchedule = new ArrayList<>(); + private final List windowOfOtherSchedule = new ArrayList<>(); + private final List beforeWindow = new ArrayList<>(); + private final List onWindowEnd = new ArrayList<>(); + + JDBCEBPFProfilingQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void writeProfiling() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + + final Model tasks = support.install( + EBPFProfilingTaskRecord.class, DefaultScopeDefine.EBPF_PROFILING_TASK, EBPFProfilingTaskRecord.INDEX_NAME, + DownSampling.Minute); + for (final EBPFProfilingTaskRecord task : TASKS) { + support.writeNoneStream(tasks, new EBPFProfilingTaskRecord.Builder(), task, task.getCreateTime()); + } + + final Model schedules = support.install( + EBPFProfilingScheduleRecord.class, DefaultScopeDefine.EBPF_PROFILING_SCHEDULE, + EBPFProfilingScheduleRecord.INDEX_NAME, DownSampling.Minute); + support.writeMetrics(schedules, new EBPFProfilingScheduleRecord.Builder(), Arrays.asList( + ACROSS_MIDNIGHT_SCHEDULE, OTHER_TASK_SCHEDULE, FIRST_SCHEDULE, LAST_SCHEDULE)); + + for (int upload = 0; upload < UPLOADS; upload++) { + for (int stack = 0; stack < STACKS_PER_UPLOAD; stack++) { + windowOfSchedule.add(data(SCHEDULE, upload + "-" + stack, WINDOW_BEGIN + upload * 500L)); + } + } + for (int stack = 0; stack < STACKS_PER_UPLOAD; stack++) { + beforeWindow.add(data(SCHEDULE, "before-" + stack, WINDOW_BEGIN - 1)); + onWindowEnd.add(data(SCHEDULE, "end-" + stack, WINDOW_END)); + windowOfOtherSchedule.add(data(OTHER_SCHEDULE, "other-" + stack, MIDNIGHT - 1_000L)); + windowOfOtherSchedule.add(data(OTHER_SCHEDULE, "other-later-" + stack, MIDNIGHT + 1_000L)); + } + final List data = new ArrayList<>(windowOfSchedule); + data.addAll(windowOfOtherSchedule); + data.addAll(beforeWindow); + data.addAll(onWindowEnd); + data.add(data(UNASKED_SCHEDULE, "unasked", MIDNIGHT)); + final Model dataModel = support.install( + EBPFProfilingDataRecord.class, DefaultScopeDefine.EBPF_PROFILING_DATA, EBPFProfilingDataRecord.INDEX_NAME, + DownSampling.Second); + support.writeRecords(dataModel, new EBPFProfilingDataRecord.Builder(), data); + + taskDAO = new JDBCEBPFProfilingTaskDAO(support.getClient(), support.getTableHelper()); + scheduleDAO = new JDBCEBPFProfilingScheduleDAO(support.getClient(), support.getTableHelper()); + dataDAO = new JDBCEBPFProfilingDataDAO(support.getClient(), support.getTableHelper()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + /** + * The rover asks for the fixed-time tasks of the services of its processes: neither the tasks of another + * service nor the continuous-profiling tasks, which the rover triggers itself. + */ + @Test + public void readsTheFixedTimeTasksOfTheRoversServices() throws Exception { + assertEquals(describe(ON_CPU, OFF_CPU, NETWORK, NETWORK_EXTENDED), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, 0, 0))); + assertEquals(describe(TASKS), describe(taskDAO.queryTasksByServices( + Arrays.asList(SERVICE_ID, OTHER_SERVICE_ID), null, 0, 0))); + } + + /** + * The rover asks for the tasks updated after its last poll: a task updated at that time is not read again, and + * the copy extending the network task after midnight is. + */ + @Test + public void readsTheTasksUpdatedAfterTheRoversLastPoll() throws Exception { + assertEquals(describe(OFF_CPU, NETWORK_EXTENDED), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, 0, + NETWORK.getLastUpdateTime()))); + assertEquals(describe(OFF_CPU, NETWORK_EXTENDED), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, 0, + NETWORK_EXTENDED.getLastUpdateTime() - 1))); + assertEquals(describe(OFF_CPU), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, 0, + NETWORK_EXTENDED.getLastUpdateTime()))); + } + + @Test + public void readsTheTasksStartingFromATime() throws Exception { + assertEquals(describe(OFF_CPU, NETWORK, NETWORK_EXTENDED), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, NETWORK.getStartTime(), 0))); + assertEquals(describe(OFF_CPU), describe(taskDAO.queryTasksByServices( + Collections.singletonList(SERVICE_ID), EBPFProfilingTriggerType.FIXED_TIME, NETWORK.getStartTime() + 1, + 0))); + } + + /** + * The task list of a service, as the profiling page asks for it, by target type. + */ + @Test + public void readsTheTasksOfTheTargetTypesAsked() throws Exception { + assertEquals(describe(ON_CPU, OFF_CPU, NETWORK, NETWORK_EXTENDED, CONTINUOUS_ON_CPU, CONTINUOUS_OFF_CPU), + describe(taskDAO.queryTasksByTargets(SERVICE_ID, null, ALL_TARGETS, null, 0, 0))); + assertEquals(describe(ON_CPU, CONTINUOUS_ON_CPU), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, Collections.singletonList(EBPFProfilingTargetType.ON_CPU), null, 0, 0))); + assertEquals(describe(ON_CPU, OFF_CPU, CONTINUOUS_ON_CPU, CONTINUOUS_OFF_CPU), + describe(taskDAO.queryTasksByTargets(SERVICE_ID, null, CPU, null, 0, 0))); + assertEquals(describe(ON_CPU, OTHER_SERVICE_ON_CPU, CONTINUOUS_ON_CPU), describe(taskDAO.queryTasksByTargets( + null, null, Collections.singletonList(EBPFProfilingTargetType.ON_CPU), null, 0, 0))); + } + + @Test + public void readsTheTasksOfAnInstance() throws Exception { + assertEquals(describe(NETWORK, NETWORK_EXTENDED, CONTINUOUS_ON_CPU), + describe(taskDAO.queryTasksByTargets(SERVICE_ID, INSTANCE_ID, ALL_TARGETS, null, 0, 0))); + assertEquals(describe(CONTINUOUS_OFF_CPU), + describe(taskDAO.queryTasksByTargets(null, OTHER_INSTANCE_ID, ALL_TARGETS, null, 0, 0))); + } + + /** + * The continuous-profiling page counts the tasks a policy triggered since a time, creating a fixed-time task + * looks for the tasks of its target starting from its own start, and the task list of a duration asks for the + * tasks updated after its end. + */ + @Test + public void readsTheTasksOfATriggerTypeByTheirStartAndUpdateTimes() throws Exception { + final EBPFProfilingTriggerType continuous = EBPFProfilingTriggerType.CONTINUOUS_PROFILING; + assertEquals(describe(CONTINUOUS_ON_CPU, CONTINUOUS_OFF_CPU), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, CPU, continuous, CONTINUOUS_ON_CPU.getStartTime(), 0))); + assertEquals(describe(CONTINUOUS_OFF_CPU), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, CPU, continuous, CONTINUOUS_ON_CPU.getStartTime() + 1, 0))); + assertEquals(describe(NETWORK, NETWORK_EXTENDED), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, Collections.singletonList(EBPFProfilingTargetType.NETWORK), + EBPFProfilingTriggerType.FIXED_TIME, NETWORK.getStartTime(), 0))); + assertEquals(describe(OFF_CPU), describe(taskDAO.queryTasksByTargets( + SERVICE_ID, null, CPU, EBPFProfilingTriggerType.FIXED_TIME, 0, NETWORK_EXTENDED.getLastUpdateTime()))); + } + + /** + * Keeping a network task alive reads every row of the task, the one the task was created with and the copies + * that extend it, whichever day table each is in. + */ + @Test + public void readsEveryRowOfATaskById() throws Exception { + assertEquals(describe(NETWORK, NETWORK_EXTENDED), describe(taskDAO.getTaskRecord(NETWORK.getLogicalId()))); + assertEquals(describe(OFF_CPU), describe(taskDAO.getTaskRecord(OFF_CPU.getLogicalId()))); + assertEquals(Collections.emptyList(), taskDAO.getTaskRecord("missing-task")); + } + + @Test + public void readsTheSchedulesOfOneTaskLatestStartFirst() throws Exception { + assertEquals(describeSchedules(LAST_SCHEDULE, ACROSS_MIDNIGHT_SCHEDULE, FIRST_SCHEDULE), + describeSchedules(scheduleDAO.querySchedules(NETWORK.getLogicalId()))); + assertEquals(describeSchedules(OTHER_TASK_SCHEDULE), + describeSchedules(scheduleDAO.querySchedules(ON_CPU.getLogicalId()))); + assertEquals(Collections.emptyList(), scheduleDAO.querySchedules("missing-task")); + } + + /** + * The analysis reads every stack a schedule uploaded in a window, from both day tables, and none uploaded just + * before the window or on its end. + */ + @Test + public void readsEveryStackOfAScheduleInTheWindow() throws Exception { + assertEquals(describeData(windowOfSchedule), + describeData(dataDAO.queryData(Collections.singletonList(SCHEDULE), WINDOW_BEGIN, WINDOW_END))); + } + + @Test + public void readsTheStacksOfTheSchedulesAskedOnly() throws Exception { + final List both = new ArrayList<>(windowOfSchedule); + both.addAll(windowOfOtherSchedule); + assertEquals(describeData(both), describeData( + dataDAO.queryData(Arrays.asList(SCHEDULE, OTHER_SCHEDULE), WINDOW_BEGIN, WINDOW_END))); + } + + /** + * The analysis asks for windows that meet at their ends: an upload on a window's end belongs to the next one. + */ + @Test + public void readsTheUploadsOnAWindowsBoundsInOneWindowOnly() throws Exception { + assertEquals(describeData(onWindowEnd), describeData(dataDAO.queryData( + Collections.singletonList(SCHEDULE), WINDOW_END, WINDOW_END + 10_000L))); + assertEquals(describeData(beforeWindow), describeData(dataDAO.queryData( + Collections.singletonList(SCHEDULE), WINDOW_BEGIN - 10_000L, WINDOW_BEGIN))); + } + + private static long minutes(final int fromMidnight) { + return MIDNIGHT + TimeUnit.MINUTES.toMillis(fromMidnight); + } + + /** + * Every column of each task, as the task workers stored it, in no order. + */ + private static List describe(final List tasks) { + final EBPFProfilingTaskRecord.Builder builder = new EBPFProfilingTaskRecord.Builder(); + return tasks.stream().map(task -> { + final HashMapConverter.ToStorage columns = new HashMapConverter.ToStorage(); + builder.entity2Storage(task, columns); + return new TreeMap<>(columns.obtain()).toString(); + }).sorted().collect(Collectors.toList()); + } + + private static List describe(final EBPFProfilingTaskRecord... tasks) { + return describe(Arrays.asList(tasks)); + } + + private static List describeSchedules(final List schedules) { + return schedules.stream() + .map(schedule -> String.join("|", schedule.getScheduleId(), schedule.getTaskId(), + schedule.getProcessId(), String.valueOf(schedule.getStartTime()), + String.valueOf(schedule.getEndTime()))) + .collect(Collectors.toList()); + } + + private static List describeSchedules(final EBPFProfilingScheduleRecord... records) { + return describeSchedules(Arrays.stream(records).map(record -> { + final EBPFProfilingSchedule schedule = new EBPFProfilingSchedule(); + schedule.setScheduleId(record.getScheduleId()); + schedule.setTaskId(record.getTaskId()); + schedule.setProcessId(record.getProcessId()); + schedule.setStartTime(record.getStartTime()); + schedule.setEndTime(record.getEndTime()); + return schedule; + }).collect(Collectors.toList())); + } + + /** + * Every field of each stack the analysis uses, in no order. + */ + private static List describeData(final List data) { + return data.stream() + .map(row -> String.join("|", row.getScheduleId(), row.getTaskId(), row.getStackIdList(), + String.valueOf(row.getTargetType()), + Base64.getEncoder().encodeToString(row.getDataBinary()), + String.valueOf(row.getUploadTime()))) + .sorted() + .collect(Collectors.toList()); + } + + /** + * A fixed-time task as the profiling page creates it ({@code EBPFProfilingMutationService#createTask}). + */ + private static EBPFProfilingTaskRecord fixedTimeTask(final String serviceId, final List processLabels, + final EBPFProfilingTargetType targetType, + final long createTime, final long startTime) { + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(serviceId); + task.setProcessLabelsJson(processLabels.isEmpty() ? Const.EMPTY_STRING : GSON.toJson(processLabels)); + task.setStartTime(startTime); + task.setTriggerType(EBPFProfilingTriggerType.FIXED_TIME.value()); + task.setFixedTriggerDuration(60); + task.setTargetType(targetType.value()); + task.setCreateTime(createTime); + task.setLastUpdateTime(createTime); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(createTime)); + task.generateLogicalId(); + task.setExtensionConfigJson(Const.EMPTY_STRING); + return task; + } + + /** + * A network-profiling task of an instance, created to start at once. + */ + private static EBPFProfilingTaskRecord networkTask(final String instanceId, final long createTime) { + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(IDManager.ServiceInstanceID.analysisId(instanceId).getServiceId()); + task.setProcessLabelsJson(Const.EMPTY_STRING); + task.setInstanceId(instanceId); + task.setStartTime(createTime); + task.setTriggerType(EBPFProfilingTriggerType.FIXED_TIME.value()); + task.setFixedTriggerDuration(600); + task.setTargetType(EBPFProfilingTargetType.NETWORK.value()); + task.setCreateTime(createTime); + task.setLastUpdateTime(createTime); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(createTime)); + task.setExtensionConfigJson("{\"networkSamplings\":[{\"uriRegex\":\"/checkout\",\"minDuration\":0}]}"); + task.generateLogicalId(); + return task; + } + + /** + * The copy {@code EBPFProfilingMutationService#keepEBPFNetworkProfiling} writes to keep a network task alive: + * the same logical id, ten more minutes, created and updated at the time it is kept alive. + */ + private static EBPFProfilingTaskRecord extended(final EBPFProfilingTaskRecord task, final long keptAt) { + final EBPFProfilingTaskRecord copy = new EBPFProfilingTaskRecord(); + copy.setLogicalId(task.getLogicalId()); + copy.setServiceId(task.getServiceId()); + copy.setProcessLabelsJson(Const.EMPTY_STRING); + copy.setInstanceId(task.getInstanceId()); + copy.setStartTime(task.getStartTime()); + copy.setTriggerType(task.getTriggerType()); + copy.setFixedTriggerDuration(task.getFixedTriggerDuration() + 600); + copy.setTargetType(EBPFProfilingTargetType.NETWORK.value()); + copy.setCreateTime(keptAt); + copy.setLastUpdateTime(keptAt); + copy.setTimeBucket(TimeBucket.getMinuteTimeBucket(keptAt)); + copy.setExtensionConfigJson(Const.EMPTY_STRING); + return copy; + } + + /** + * A task as the eBPF agent reports one when a continuous-profiling policy triggers on a process of an instance. + */ + private static EBPFProfilingTaskRecord continuousProfilingTask(final String instanceId, + final EBPFProfilingTargetType targetType, + final long time) { + final EBPFProfilingTaskContinuousProfiling continuousProfiling = new EBPFProfilingTaskContinuousProfiling(); + continuousProfiling.setProcessId(IDManager.ProcessID.buildId(instanceId, "checkout")); + continuousProfiling.setProcessName("checkout"); + final EBPFProfilingTaskRecord task = new EBPFProfilingTaskRecord(); + task.setServiceId(IDManager.ServiceInstanceID.analysisId(instanceId).getServiceId()); + task.setProcessLabelsJson(Const.EMPTY_STRING); + task.setInstanceId(instanceId); + task.setStartTime(time); + task.setTriggerType(EBPFProfilingTriggerType.CONTINUOUS_PROFILING.value()); + task.setFixedTriggerDuration(60); + task.setTargetType(targetType.value()); + task.setCreateTime(time); + task.setLastUpdateTime(time); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(time)); + task.setContinuousProfilingJson(GSON.toJson(continuousProfiling)); + task.generateLogicalId(); + return task; + } + + /** + * A schedule as the eBPF receiver's dispatcher stores it, at the minute of its latest report. + */ + private static EBPFProfilingScheduleRecord schedule(final String taskId, final String instanceId, + final String processName, final long startTime, + final long endTime) { + final EBPFProfilingScheduleRecord schedule = new EBPFProfilingScheduleRecord(); + schedule.setTaskId(taskId); + schedule.setProcessId(IDManager.ProcessID.buildId(instanceId, processName)); + schedule.setStartTime(startTime); + schedule.setEndTime(endTime); + schedule.setScheduleId(taskId + "-" + processName + "-" + startTime); + schedule.setTimeBucket(TimeBucket.getMinuteTimeBucket(endTime)); + return schedule; + } + + /** + * A stack of an upload, as the eBPF profiling data dispatcher writes it. + */ + private static EBPFProfilingDataRecord data(final String scheduleId, final String stack, final long uploadTime) { + final EBPFProfilingDataRecord data = new EBPFProfilingDataRecord(); + data.setScheduleId(scheduleId); + data.setTaskId(NETWORK.getLogicalId()); + data.setStackIdList("stack-" + stack); + data.setTargetType(EBPFProfilingTargetType.ON_CPU.value()); + data.setDataBinary(("stack " + stack).getBytes()); + data.setUploadTime(uploadTime); + data.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime)); + return data; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryMariaDBIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryMariaDBIT.java new file mode 100644 index 000000000000..4a7cc287fb6f --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryMariaDBIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCEBPFProfilingQueryMariaDBIT extends JDBCEBPFProfilingQueryITBase { + @Container + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); + + public JDBCEBPFProfilingQueryMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryPostgreSQLIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryPostgreSQLIT.java new file mode 100644 index 000000000000..5b985b9aedf6 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCEBPFProfilingQueryPostgreSQLIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCEBPFProfilingQueryPostgreSQLIT extends JDBCEBPFProfilingQueryITBase { + @Container + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); + + public JDBCEBPFProfilingQueryPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryITBase.java new file mode 100644 index 000000000000..ac9c5052fe3b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryITBase.java @@ -0,0 +1,436 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.genai.GenAIEvaluationRecord; +import org.apache.skywalking.oap.server.core.query.enumeration.GenAIEvaluationRecordSortBy; +import org.apache.skywalking.oap.server.core.query.enumeration.GenAIEvaluationValueType; +import org.apache.skywalking.oap.server.core.query.enumeration.GenAITraceRefType; +import org.apache.skywalking.oap.server.core.query.enumeration.Order; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.GenAITraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.GenAITraceRef; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.JdbcDatabaseContainer; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads GenAI evaluation records the way the GenAI evaluation query service does on a JDBC storage, through + * {@link JDBCGenAIEvaluationRecordQueryDAO}: by service, provider, model, value type, score bounds, boolean value, + * task, evaluation level, judge model and the trace, native segment and span index, or OTLP and Zipkin span it was + * produced from, newest, oldest, highest or lowest score first, page by page. A subclass names the server. + * + *

The records are written as the span evaluation writes them, on both sides of midnight, so a read crosses two + * day tables. For each filter a record matches every other condition but that one, and the score bounds lie on a + * record's score. Records of their own lie on and just outside the bounds of a duration: on its first millisecond and + * the one before, on its last millisecond, on the instant it ends at ({@link Duration#getEndTimestamp()}, which a + * read includes) and the millisecond after. + * One more lies on the end instant of a duration ending in a day's last minute: the next midnight, in the next + * day's table. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCGenAIEvaluationQueryITBase { + private static final long MINUTE = 60_000L; + private static final Duration AROUND_MIDNIGHT = minutes(MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE); + private static final Duration BOUNDED = minutes(MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE); + private static final Duration ENDING = minutes(MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE); + /** Yesterday's midnight. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = minutes(PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE); + + private static final String CHAT = "chat-app"; + private static final String OPENAI = "openai"; + private static final String TRACE_1 = "trace-1"; + private static final String TRACE_2 = "trace-2"; + private static final String OTLP_SPAN = "a1b2c3d4e5f60718"; + + private static final List EVALUATIONS = Arrays.asList( + new Evaluation("g1", MIDNIGHT - 150_000L, CHAT, OPENAI, "gpt-4o", "relevance", "judge-a") + .score(800_000L, "SPAN").nativeSpan(TRACE_1, "segment-1", 0), + new Evaluation("g2", MIDNIGHT - 110_000L, CHAT, OPENAI, "gpt-4o", "safety", "judge-a") + .bool(true, "TRACE").nativeSpan(TRACE_1, "segment-1", 1), + new Evaluation("g3", MIDNIGHT - 70_000L, CHAT, OPENAI, "gpt-4o-mini", "relevance", "judge-b") + .score(300_000L, "SPAN").nativeSpan(TRACE_1, "segment-2", 0), + new Evaluation("g4", MIDNIGHT - 30_000L, "search-app", "anthropic", "claude", "relevance", "judge-a") + .score(950_000L, "SPAN").span(GenAITraceRefType.OTLP, TRACE_2, OTLP_SPAN), + new Evaluation("g5", MIDNIGHT + 30_000L, CHAT, "anthropic", "claude", "safety", "judge-b") + .bool(false, "SPAN").span(GenAITraceRefType.ZIPKIN, "trace-3", "0000000000000001"), + new Evaluation("g6", MIDNIGHT + 70_000L, CHAT, OPENAI, "gpt-4o", "tone", "judge-a") + .string("polite", "SPAN").span(GenAITraceRefType.OTLP, TRACE_2, "a1b2c3d4e5f60719"), + new Evaluation("g7", MIDNIGHT + 110_000L, CHAT, OPENAI, "gpt-4o", "relevance", "judge-a") + .score(600_000L, "SPAN").nativeSpan("trace-4", "segment-4", 0), + bound("before-start", BOUNDED.getStartTimestamp() - 1), + bound("on-start", BOUNDED.getStartTimestamp()), + bound("last-millisecond", BOUNDED.getEndTimestamp() - 1), + bound("after-end", BOUNDED.getEndTimestamp() + 1), + bound("on-end", ENDING.getEndTimestamp()), + bound("on-midnight", ENDING_AT_MIDNIGHT.getEndTimestamp()) + ); + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCGenAIEvaluationRecordQueryDAO dao; + + JDBCGenAIEvaluationQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void writeEvaluations() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + final Model model = support.install( + GenAIEvaluationRecord.class, DefaultScopeDefine.GEN_AI_EVALUATION_RECORD, GenAIEvaluationRecord.INDEX_NAME, + DownSampling.Second); + support.writeRecords(model, new GenAIEvaluationRecord.Builder(), EVALUATIONS.stream() + .map(Evaluation::record).collect(Collectors.toList())); + dao = new JDBCGenAIEvaluationRecordQueryDAO(support.getClient(), support.getTableHelper()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheEvaluationsOfAServiceNewestOrOldestFirst() { + assertEquals(Arrays.asList("g7", "g6", "g5", "g3", "g2", "g1"), ids(read().service(CHAT))); + assertEquals(Arrays.asList("g1", "g2", "g3", "g5", "g6", "g7"), ids(read().service(CHAT).order(Order.ASC))); + } + + @Test + public void readsTheEvaluationsOfAProviderAndOfAModel() { + assertEquals(Arrays.asList("g7", "g6", "g3", "g2", "g1"), ids(read().provider(OPENAI))); + assertEquals(Arrays.asList("g7", "g6", "g2", "g1"), ids(read().model(OPENAI, "gpt-4o"))); + } + + @Test + public void readsTheEvaluationsOfAValueType() { + assertEquals(Arrays.asList("g7", "g4", "g3", "g1"), ids(read().valueType(GenAIEvaluationValueType.SCORE))); + assertEquals(Collections.singletonList("g6"), ids(read().valueType(GenAIEvaluationValueType.STRING))); + } + + /** + * The query service reads score bounds and the score order of scores only, and a boolean value of booleans + * only. A bound lies on a score. + */ + @Test + public void readsTheScoresWithinTheBoundsAndTheBooleansOfAValue() { + assertEquals(Arrays.asList("g7", "g4", "g1"), ids(read().scores(600_000L, null))); + assertEquals(Arrays.asList("g7", "g3", "g1"), ids(read().scores(null, 800_000L))); + assertEquals(Arrays.asList("g7", "g1"), ids(read().scores(500_000L, 900_000L))); + assertEquals(Collections.singletonList("g2"), ids(read().bool(true))); + assertEquals(Collections.singletonList("g5"), ids(read().bool(false))); + } + + @Test + public void readsTheScoresHighestOrLowestFirst() { + assertEquals(Arrays.asList("g4", "g1", "g7", "g3"), ids(read().scores(null, null).byScore())); + assertEquals(Arrays.asList("g3", "g7", "g1", "g4"), ids(read().scores(null, null).byScore() + .order(Order.ASC))); + } + + @Test + public void readsTheEvaluationsOfATaskALevelAndAJudgeModel() { + assertEquals(Arrays.asList("g7", "g4", "g3", "g1"), ids(read().task("relevance"))); + assertEquals(Collections.singletonList("g2"), ids(read().level("TRACE"))); + assertEquals(Arrays.asList("g5", "g3"), ids(read().judge("judge-b"))); + } + + @Test + public void readsTheEvaluationsOfANativeTraceSegmentAndSpan() { + assertEquals(Arrays.asList("g3", "g2", "g1"), ids(read().trace(GenAITraceRefType.SKYWALKING_NATIVE, TRACE_1, + null, null, null))); + assertEquals(Arrays.asList("g2", "g1"), ids(read().trace(GenAITraceRefType.SKYWALKING_NATIVE, TRACE_1, + "segment-1", null, null))); + assertEquals(Collections.singletonList("g2"), ids(read().trace(GenAITraceRefType.SKYWALKING_NATIVE, TRACE_1, + "segment-1", 1, null))); + } + + /** + * The trace's store is part of the reference: a Zipkin trace is not found as an OTLP one. + */ + @Test + public void readsTheEvaluationsOfAnOtlpOrZipkinTraceAndSpan() { + assertEquals(Arrays.asList("g6", "g4"), ids(read().trace(GenAITraceRefType.OTLP, TRACE_2, null, null, null))); + assertEquals(Collections.singletonList("g4"), ids(read().trace(GenAITraceRefType.OTLP, TRACE_2, null, null, + OTLP_SPAN))); + assertEquals(Collections.singletonList("g5"), ids(read().trace(GenAITraceRefType.ZIPKIN, "trace-3", null, + null, null))); + assertEquals(Collections.emptyList(), ids(read().trace(GenAITraceRefType.OTLP, "trace-3", null, null, null))); + } + + @Test + public void pagesThroughTheEvaluationsOfAService() { + assertEquals(Arrays.asList("g7", "g6"), ids(read().service(CHAT).page(0, 2))); + assertEquals(Arrays.asList("g5", "g3"), ids(read().service(CHAT).page(2, 2))); + assertEquals(Arrays.asList("g2", "g1"), ids(read().service(CHAT).page(4, 2))); + assertEquals(Collections.emptyList(), ids(read().service(CHAT).page(6, 2))); + } + + @Test + public void readsEachEvaluationWhole() { + final List expected = new ArrayList<>(); + for (int i = 6; i >= 0; i--) { + expected.add(render(EVALUATIONS.get(i).record())); + } + assertEquals(expected, read().run(dao).stream().map(JDBCGenAIEvaluationQueryITBase::render) + .collect(Collectors.toList())); + } + + @Test + public void readsTheEvaluationsOfEveryDayWithoutADuration() { + assertEquals(Arrays.asList("g3", "g2", "g1"), ids(read().trace(GenAITraceRefType.SKYWALKING_NATIVE, TRACE_1, + null, null, null).during(null))); + } + + @Test + public void keepsTheEvaluationsWithinTheDuration() { + assertEquals(Arrays.asList("last-millisecond", "on-start"), ids(read().service("bounds-app").during(BOUNDED))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's table. + */ + @Test + public void readsAnEvaluationOnTheInstantTheDurationEndsAt() { + assertEquals(Collections.singletonList("on-end"), ids(read().service("bounds-app").during(ENDING))); + assertEquals(Collections.singletonList("on-midnight"), ids(read().service("bounds-app") + .during(ENDING_AT_MIDNIGHT))); + } + + private List ids(final EvaluationRead read) { + return read.run(dao).stream().map(GenAIEvaluationRecord::getUniqueId).collect(Collectors.toList()); + } + + private static EvaluationRead read() { + return new EvaluationRead(); + } + + private static String render(final GenAIEvaluationRecord record) { + final GenAITraceRef ref = record.getTraceRef(); + return String.join("|", record.getUniqueId(), record.getServiceId(), record.getProviderId(), + record.getModelId(), record.getOperationName(), String.valueOf(record.getValueType()), + String.valueOf(record.getEvalNumberValue()), record.getEvalStringValue(), + record.getTaskName(), record.getEvaluationLevel(), record.getReason(), + record.getJudgeModel(), String.valueOf(record.getEvaluationTime()), + String.valueOf(ref.getType()), ref.getTraceId(), ref.getSegmentId(), + String.valueOf(ref.getSpanIndex()), ref.getSpanId()); + } + + private static Duration minutes(final long start, final long end) { + return JDBCQueryITSupport.duration(start, end, Step.MINUTE); + } + + private static Evaluation bound(final String id, final long time) { + return new Evaluation(id, time, "bounds-app", OPENAI, "gpt-4o", "relevance", "judge-a") + .score(500_000L, "SPAN").nativeSpan("trace-" + id, "segment-" + id, 0); + } + + /** + * The arguments of one read, as the GenAI evaluation query service passes them after its checks: score bounds + * and the score order ask for scores, a boolean value for booleans. + */ + private static final class EvaluationRead { + private String serviceId; + private String providerId; + private String modelId; + private GenAIEvaluationValueType valueType; + private Long minScore; + private Long maxScore; + private Boolean booleanValue; + private GenAIEvaluationRecordSortBy sortBy = GenAIEvaluationRecordSortBy.EVALUATION_TIME; + private String taskName; + private String evaluationLevel; + private String judgeModel; + private GenAITraceScopeCondition trace; + private Order order = Order.DES; + private int from = 0; + private int limit = 100; + private Duration duration = AROUND_MIDNIGHT; + + EvaluationRead service(final String service) { + serviceId = GenAIEvaluationRecord.toServiceId(service); + return this; + } + + EvaluationRead provider(final String provider) { + providerId = GenAIEvaluationRecord.toEntityId(provider); + return this; + } + + EvaluationRead model(final String provider, final String model) { + modelId = IDManager.ServiceInstanceID.buildId(GenAIEvaluationRecord.toEntityId(provider), model); + return this; + } + + EvaluationRead valueType(final GenAIEvaluationValueType valueType) { + this.valueType = valueType; + return this; + } + + EvaluationRead scores(final Long min, final Long max) { + valueType = GenAIEvaluationValueType.SCORE; + minScore = min; + maxScore = max; + return this; + } + + EvaluationRead bool(final boolean value) { + valueType = GenAIEvaluationValueType.BOOLEAN; + booleanValue = value; + return this; + } + + EvaluationRead byScore() { + sortBy = GenAIEvaluationRecordSortBy.SCORE_VALUE; + return this; + } + + EvaluationRead task(final String taskName) { + this.taskName = taskName; + return this; + } + + EvaluationRead level(final String evaluationLevel) { + this.evaluationLevel = evaluationLevel; + return this; + } + + EvaluationRead judge(final String judgeModel) { + this.judgeModel = judgeModel; + return this; + } + + EvaluationRead trace(final GenAITraceRefType type, final String traceId, final String segmentId, + final Integer spanIndex, final String spanId) { + trace = new GenAITraceScopeCondition(); + trace.setType(type); + trace.setTraceId(traceId); + trace.setSegmentId(segmentId); + trace.setSpanIndex(spanIndex); + trace.setSpanId(spanId); + return this; + } + + EvaluationRead order(final Order order) { + this.order = order; + return this; + } + + EvaluationRead page(final int from, final int limit) { + this.from = from; + this.limit = limit; + return this; + } + + EvaluationRead during(final Duration duration) { + this.duration = duration; + return this; + } + + List run(final JDBCGenAIEvaluationRecordQueryDAO dao) { + return dao.queryGenAIEvaluationRecord(serviceId, providerId, modelId, valueType, minScore, maxScore, + booleanValue, sortBy, taskName, evaluationLevel, judgeModel, trace, + order, from, limit, duration) + .getGenAIEvaluationRecordList(); + } + } + + /** + * A GenAI evaluation as the span evaluation hands it to the record stream. + */ + private static final class Evaluation { + private final GenAIEvaluationRecord record; + + Evaluation(final String id, final long time, final String service, final String provider, final String model, + final String task, final String judge) { + record = GenAIEvaluationRecord.create(provider, model); + record.setUniqueId(id); + record.setServiceName(service); + record.setOperationName("chat"); + record.setTaskName(task); + record.setJudgeModel(judge); + record.setReason("reason of " + id); + record.setEvaluationTime(time); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + } + + Evaluation score(final long scorePpm, final String level) { + record.setValueType(GenAIEvaluationValueType.SCORE.name()); + record.setEvalNumberValue(scorePpm); + record.setEvaluationLevel(level); + return this; + } + + Evaluation bool(final boolean value, final String level) { + record.setValueType(GenAIEvaluationValueType.BOOLEAN.name()); + record.setEvalNumberValue(value ? GenAIEvaluationRecord.SCORE_SCALE : 0L); + record.setEvaluationLevel(level); + return this; + } + + Evaluation string(final String value, final String level) { + record.setValueType(GenAIEvaluationValueType.STRING.name()); + record.setEvalStringValue(value); + record.setEvaluationLevel(level); + return this; + } + + Evaluation nativeSpan(final String traceId, final String segmentId, final int spanIndex) { + record.setRefType(GenAITraceRefType.SKYWALKING_NATIVE.name()); + record.setTraceId(traceId); + record.setSegmentId(segmentId); + record.setSpanIndex(spanIndex); + return this; + } + + Evaluation span(final GenAITraceRefType type, final String traceId, final String spanId) { + record.setRefType(type.name()); + record.setTraceId(traceId); + record.setSpanId(spanId); + return this; + } + + GenAIEvaluationRecord record() { + return record; + } + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryMariaDBIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryMariaDBIT.java new file mode 100644 index 000000000000..bbda0f0d2e30 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryMariaDBIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCGenAIEvaluationQueryMariaDBIT extends JDBCGenAIEvaluationQueryITBase { + @Container + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); + + public JDBCGenAIEvaluationQueryMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryPostgreSQLIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryPostgreSQLIT.java new file mode 100644 index 000000000000..e97b6883f31a --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCGenAIEvaluationQueryPostgreSQLIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCGenAIEvaluationQueryPostgreSQLIT extends JDBCGenAIEvaluationQueryITBase { + @Container + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); + + public JDBCGenAIEvaluationQueryPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryITBase.java new file mode 100644 index 000000000000..4da5b3a7d2fa --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryITBase.java @@ -0,0 +1,259 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.hierarchy.instance.InstanceHierarchyRelationTraffic; +import org.apache.skywalking.oap.server.core.hierarchy.service.ServiceHierarchyRelationTraffic; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.JdbcDatabaseContainer; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Reads service and instance hierarchy relations the way the hierarchy query service does, through + * {@link JDBCHierarchyQueryDAO}: every service relation, and the relations of an instance in the layer it is read in, + * found from either side of the relation. A subclass names the server. + * + *

A hierarchy relation is traffic: it stays in the day table it was first stored in, and the relations here were + * first stored on both sides of midnight, so every read crosses two day tables. One instance id is in three layers, + * as a service of one name is in each layer it reports to, so it is the upper instance of relations in two layers and + * the lower instance of a relation in a third. The near misses are an instance of the same service with relations of + * its own, and reads in a layer the instance has relations in on the other side only. One pod lies below instances in + * twelve layers. Neither read promises an order, so the relations are compared sorted. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCHierarchyQueryITBase { + private static final int METADATA_QUERY_MAX_SIZE = 5000; + /** Readable names of the ids the fixture builds, to print the relations read. */ + private static final Map NAMES = new HashMap<>(); + + private static final String CHECKOUT = service("checkout"); + private static final String CHECKOUT_POD = service("checkout.default"); + private static final String MYSQL = service("mysql::orders-db"); + private static final String MYSQL_POD = service("orders-db.default"); + private static final String ORDERS_DB = service("orders-db.default:3306"); + + private static final String CHECKOUT_0 = instance(CHECKOUT, "pod-0"); + private static final String CHECKOUT_1 = instance(CHECKOUT, "pod-1"); + private static final String CHECKOUT_POD_0 = instance(CHECKOUT_POD, "pod-0"); + private static final String CHECKOUT_POD_1 = instance(CHECKOUT_POD, "pod-1"); + private static final String EDGE_0 = instance(service("edge"), "pod-0"); + private static final String EDGE_POD_0 = instance(service("edge.default"), "pod-0"); + /** Every layer the edge service reports to, each above K8S_SERVICE. */ + private static final List EDGE_LAYERS = Arrays.asList( + Layer.GENERAL, Layer.MESH, Layer.MESH_DP, Layer.MYSQL, Layer.POSTGRESQL, Layer.APISIX, Layer.NGINX, + Layer.SO11Y_OAP, Layer.ROCKETMQ, Layer.RABBITMQ, Layer.KAFKA, Layer.CLICKHOUSE); + + private static final List SERVICE_RELATIONS = sorted( + "checkout GENERAL -> checkout.default K8S_SERVICE", + "checkout MESH -> checkout MESH_DP", + "checkout MESH -> checkout.default K8S_SERVICE", + "mysql::orders-db MYSQL -> orders-db.default K8S_SERVICE", + "orders-db.default:3306 VIRTUAL_DATABASE -> mysql::orders-db MYSQL"); + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCHierarchyQueryDAO dao; + + JDBCHierarchyQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void writeRelations() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + final long yesterday = MIDNIGHT - 2 * 3_600_000L; + final long today = MIDNIGHT + 20 * 60_000L; + + support.writeMetrics( + support.install(ServiceHierarchyRelationTraffic.class, DefaultScopeDefine.SERVICE_HIERARCHY_RELATION, + ServiceHierarchyRelationTraffic.INDEX_NAME, DownSampling.Minute), + new ServiceHierarchyRelationTraffic.Builder(), + Arrays.asList( + services(CHECKOUT, Layer.GENERAL, CHECKOUT_POD, Layer.K8S_SERVICE, yesterday), + services(CHECKOUT, Layer.MESH, CHECKOUT_POD, Layer.K8S_SERVICE, today), + services(CHECKOUT, Layer.MESH, CHECKOUT, Layer.MESH_DP, today), + services(MYSQL, Layer.MYSQL, MYSQL_POD, Layer.K8S_SERVICE, today), + services(ORDERS_DB, Layer.VIRTUAL_DATABASE, MYSQL, Layer.MYSQL, yesterday))); + + final Model instanceModel = support.install( + InstanceHierarchyRelationTraffic.class, DefaultScopeDefine.INSTANCE_HIERARCHY_RELATION, + InstanceHierarchyRelationTraffic.INDEX_NAME, DownSampling.Minute); + support.writeMetrics( + instanceModel, + new InstanceHierarchyRelationTraffic.Builder(), + Arrays.asList( + instances(CHECKOUT_0, Layer.GENERAL, CHECKOUT_POD_0, Layer.K8S_SERVICE, yesterday), + instances(CHECKOUT_0, Layer.MESH, CHECKOUT_POD_0, Layer.K8S_SERVICE, today), + instances(CHECKOUT_0, Layer.MESH, CHECKOUT_0, Layer.MESH_DP, today), + instances(CHECKOUT_1, Layer.GENERAL, CHECKOUT_POD_1, Layer.K8S_SERVICE, today))); + final List edge = new ArrayList<>(); + for (int i = 0; i < EDGE_LAYERS.size(); i++) { + edge.add(instances( + EDGE_0, EDGE_LAYERS.get(i), EDGE_POD_0, Layer.K8S_SERVICE, i % 2 == 0 ? yesterday : today)); + } + support.writeMetrics(instanceModel, new InstanceHierarchyRelationTraffic.Builder(), edge); + + dao = new JDBCHierarchyQueryDAO(support.getClient(), METADATA_QUERY_MAX_SIZE, support.getTableHelper()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsEveryServiceRelationOfEveryDayTable() throws Exception { + assertEquals(SERVICE_RELATIONS, serviceRelations(dao)); + } + + /** + * The JDBC DAO reads up to twice the {@code metadataQueryMaxSize}. + */ + @Test + public void readsNoMoreServiceRelationsThanTheLimit() throws Exception { + final List limited = serviceRelations( + new JDBCHierarchyQueryDAO(support.getClient(), 2, support.getTableHelper())); + assertEquals(4, limited.size(), limited.toString()); + assertEquals(4, limited.stream().distinct().count(), limited.toString()); + assertTrue(SERVICE_RELATIONS.containsAll(limited), limited.toString()); + } + + @Test + public void readsTheRelationsOfAnInstanceFromEitherSideInItsLayer() throws Exception { + assertEquals( + sorted("checkout/pod-0 GENERAL -> checkout.default/pod-0 K8S_SERVICE"), + instanceRelations(CHECKOUT_0, Layer.GENERAL)); + assertEquals( + sorted( + "checkout/pod-0 MESH -> checkout.default/pod-0 K8S_SERVICE", + "checkout/pod-0 MESH -> checkout/pod-0 MESH_DP"), + instanceRelations(CHECKOUT_0, Layer.MESH)); + assertEquals( + sorted("checkout/pod-0 MESH -> checkout/pod-0 MESH_DP"), + instanceRelations(CHECKOUT_0, Layer.MESH_DP)); + assertEquals( + sorted( + "checkout/pod-0 GENERAL -> checkout.default/pod-0 K8S_SERVICE", + "checkout/pod-0 MESH -> checkout.default/pod-0 K8S_SERVICE"), + instanceRelations(CHECKOUT_POD_0, Layer.K8S_SERVICE)); + } + + @Test + public void readsEveryRelationOfAnInstanceBelowManyLayers() throws Exception { + assertEquals(edgeRelations(), instanceRelations(EDGE_POD_0, Layer.K8S_SERVICE)); + assertEquals( + sorted("edge/pod-0 KAFKA -> edge.default/pod-0 K8S_SERVICE"), instanceRelations(EDGE_0, Layer.KAFKA)); + } + + @Test + public void readsNoRelationOfAnInstanceInTheLayerOfItsOtherSide() throws Exception { + assertEquals(Collections.emptyList(), instanceRelations(CHECKOUT_0, Layer.K8S_SERVICE)); + assertEquals(Collections.emptyList(), instanceRelations(CHECKOUT_POD_0, Layer.GENERAL)); + } + + private List instanceRelations(final String instanceId, final Layer layer) throws Exception { + return dao.readInstanceHierarchyRelations(instanceId, layer.name()).stream() + .map(r -> name(r.getInstanceId()) + " " + r.getServiceLayer() + " -> " + + name(r.getRelatedInstanceId()) + " " + r.getRelatedServiceLayer()) + .sorted() + .collect(Collectors.toList()); + } + + private static List serviceRelations(final JDBCHierarchyQueryDAO dao) throws Exception { + return dao.readAllServiceHierarchyRelations().stream() + .map(r -> name(r.getServiceId()) + " " + r.getServiceLayer() + " -> " + + name(r.getRelatedServiceId()) + " " + r.getRelatedServiceLayer()) + .sorted() + .collect(Collectors.toList()); + } + + private static List edgeRelations() { + return EDGE_LAYERS.stream() + .map(layer -> "edge/pod-0 " + layer + " -> edge.default/pod-0 K8S_SERVICE") + .sorted() + .collect(Collectors.toList()); + } + + private static List sorted(final String... relations) { + return Arrays.stream(relations).sorted().collect(Collectors.toList()); + } + + private static String name(final String id) { + return NAMES.getOrDefault(id, id); + } + + private static String service(final String name) { + final String id = IDManager.ServiceID.buildId(name, true); + NAMES.put(id, name); + return id; + } + + private static String instance(final String serviceId, final String name) { + final String id = IDManager.ServiceInstanceID.buildId(serviceId, name); + NAMES.put(id, NAMES.get(serviceId) + "/" + name); + return id; + } + + private static ServiceHierarchyRelationTraffic services(final String upper, final Layer upperLayer, + final String lower, final Layer lowerLayer, + final long firstStored) { + final ServiceHierarchyRelationTraffic relation = new ServiceHierarchyRelationTraffic(); + relation.setServiceId(upper); + relation.setServiceLayer(upperLayer); + relation.setRelatedServiceId(lower); + relation.setRelatedServiceLayer(lowerLayer); + relation.setTimeBucket(TimeBucket.getMinuteTimeBucket(firstStored)); + return relation; + } + + private static InstanceHierarchyRelationTraffic instances(final String upper, final Layer upperLayer, + final String lower, final Layer lowerLayer, + final long firstStored) { + final InstanceHierarchyRelationTraffic relation = new InstanceHierarchyRelationTraffic(); + relation.setInstanceId(upper); + relation.setServiceLayer(upperLayer); + relation.setRelatedInstanceId(lower); + relation.setRelatedServiceLayer(lowerLayer); + relation.setTimeBucket(TimeBucket.getMinuteTimeBucket(firstStored)); + return relation; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryMariaDBIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryMariaDBIT.java new file mode 100644 index 000000000000..988c7148d4e2 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryMariaDBIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCHierarchyQueryMariaDBIT extends JDBCHierarchyQueryITBase { + @Container + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); + + public JDBCHierarchyQueryMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryPostgreSQLIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryPostgreSQLIT.java new file mode 100644 index 000000000000..595feb56e7e3 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCHierarchyQueryPostgreSQLIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCHierarchyQueryPostgreSQLIT extends JDBCHierarchyQueryITBase { + @Container + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); + + public JDBCHierarchyQueryPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryITBase.java index 0ddb87c62c26..5504aa0d1e74 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCLogQueryITBase.java @@ -21,25 +21,38 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; import org.apache.skywalking.apm.network.language.agent.v3.BrowserErrorLog; import org.apache.skywalking.apm.network.language.agent.v3.ErrorCategory; +import org.apache.skywalking.apm.network.logging.v3.LogTags; +import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.log.LogRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; import org.apache.skywalking.oap.server.core.analysis.record.Event; +import org.apache.skywalking.oap.server.core.analysis.record.LongText; import org.apache.skywalking.oap.server.core.browser.manual.errorlog.BrowserErrorLogRecord; import org.apache.skywalking.oap.server.core.browser.source.BrowserErrorCategory; +import org.apache.skywalking.oap.server.core.config.ConfigService; import org.apache.skywalking.oap.server.core.query.enumeration.Order; import org.apache.skywalking.oap.server.core.query.enumeration.Step; import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.TraceScopeCondition; +import org.apache.skywalking.oap.server.core.query.type.ContentType; +import org.apache.skywalking.oap.server.core.query.type.Log; import org.apache.skywalking.oap.server.core.query.type.Pagination; import org.apache.skywalking.oap.server.core.query.type.event.EventQueryCondition; import org.apache.skywalking.oap.server.core.query.type.event.EventType; import org.apache.skywalking.oap.server.core.query.type.event.Events; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.joda.time.DateTime; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -48,10 +61,19 @@ import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; /** - * Reads events the way the event list and the alarm list do, and browser error logs the way the browser log query - * does, on rows written on both sides of midnight, so in two day tables. A subclass names the server. + * Reads events the way the event list and the alarm list do, browser error logs the way the browser log query does, + * and logs the way the log query service does, on rows written on both sides of midnight, so in two day tables. A + * subclass names the server. + * + *

Logs are read by service, instance, endpoint, trace, native span, OTLP span and searchable tag, newest or oldest + * first, page by page. For each filter a line matches every other condition but that one; a tag kept in the raw tags + * only is not searchable. Log lines and browser errors of their own lie on and just outside the bounds of a + * duration: on its first millisecond and the one before, on its last millisecond, on the instant it ends at + * ({@link Duration#getEndTimestamp()}, which a read includes) and the millisecond after. One more lies on the end + * instant of a duration ending in a day's last minute: the next midnight, in the next day's table. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) abstract class JDBCLogQueryITBase { @@ -76,12 +98,70 @@ abstract class JDBCLogQueryITBase { {"b3", 10}, {"b4", 20}, }; + private static final long MINUTE = 60_000L; + /** Every log line of the filter reads, which lie within three minutes of midnight. */ + private static final Duration AROUND_MIDNIGHT = JDBCQueryITSupport.duration( + MIDNIGHT - 5 * MINUTE, MIDNIGHT + 4 * MINUTE, Step.MINUTE); + private static final Duration BOUNDED = JDBCQueryITSupport.duration( + MIDNIGHT + 10 * MINUTE, MIDNIGHT + 11 * MINUTE, Step.MINUTE); + private static final Duration ENDING = JDBCQueryITSupport.duration( + MIDNIGHT + 20 * MINUTE, MIDNIGHT + 20 * MINUTE, Step.MINUTE); + /** Yesterday's midnight. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + /** Ends in the last minute of the day before, so it ends at {@link #PREVIOUS_MIDNIGHT}. */ + private static final Duration ENDING_AT_MIDNIGHT = JDBCQueryITSupport.duration( + PREVIOUS_MIDNIGHT - MINUTE, PREVIOUS_MIDNIGHT - MINUTE, Step.MINUTE); + /** OAP's default {@code searchableLogsTags}, the tag keys a log line is searchable by. */ + private static final String SEARCHABLE_LOG_TAGS = "level,http.status_code"; + private static final String PAYMENT = "payment"; + private static final String BOUNDS = "bounds"; + private static final String POST_ORDERS = "POST /orders"; + private static final String TRACE_A = "trace-a"; + private static final String TRACE_B = "trace-b"; + private static final String OTLP_SPAN = "00f067aa0ba902b7"; + private static final List LINES = Arrays.asList( + new Line("l1", MIDNIGHT - 150_000L, SERVICE, "checkout-1", POST_ORDERS) + .nativeSpan(TRACE_A, "segment-a1", 1).text("order created").tags("level=INFO", "thread=main"), + new Line("l2", MIDNIGHT - 90_000L, SERVICE, "checkout-1", POST_ORDERS) + .nativeSpan(TRACE_A, "segment-a1", 2).text("payment timeout for order") + .tags("level=ERROR", "http.status_code=500"), + new Line("l3", MIDNIGHT - 30_000L, SERVICE, "checkout-2", POST_ORDERS) + .nativeSpan(TRACE_A, "segment-a2", 1).text("order was created").tags("level=ERROR"), + new Line("l4", MIDNIGHT + 30_000L, SERVICE, "checkout-1", "GET /orders") + .otlpSpan(TRACE_B, OTLP_SPAN).text("order created again").tags("level=ERROR", "http.status_code=500"), + new Line("l5", MIDNIGHT + 90_000L, SERVICE, "checkout-1", null) + .otlpSpan(TRACE_B, "00f067aa0ba902b8").json("{\"event\":\"order created\"}"), + new Line("l6", MIDNIGHT + 150_000L, PAYMENT, "payment-1", "POST /pay") + .nativeSpan("trace-c", "segment-c1", 1).text("order created").tags("level=ERROR", "http.status_code=500"), + new Line("before-start", BOUNDED.getStartTimestamp() - 1, BOUNDS, "bounds-1", null).text("health check"), + new Line("on-start", BOUNDED.getStartTimestamp(), BOUNDS, "bounds-1", null).text("health check"), + new Line("last-millisecond", BOUNDED.getEndTimestamp() - 1, BOUNDS, "bounds-1", null).text("health check"), + new Line("after-end", BOUNDED.getEndTimestamp() + 1, BOUNDS, "bounds-1", null).text("health check"), + new Line("on-end", ENDING.getEndTimestamp(), BOUNDS, "bounds-1", null).text("health check"), + new Line("on-midnight", ENDING_AT_MIDNIGHT.getEndTimestamp(), BOUNDS, "bounds-1", null).text("health check") + ); + private static final Map LINE_BY_TIME = new HashMap<>(); + private static final String BOUNDS_BROWSER_SERVICE = "bounds-web"; + /** Browser errors on and just outside the bounds, by their unique id. */ + private static final Object[][] BOUND_BROWSER_ERRORS = { + {"before-start", BOUNDED.getStartTimestamp() - 1}, + {"on-start", BOUNDED.getStartTimestamp()}, + {"last-millisecond", BOUNDED.getEndTimestamp() - 1}, + {"after-end", BOUNDED.getEndTimestamp() + 1}, + {"on-end", ENDING.getEndTimestamp()}, + {"on-midnight", ENDING_AT_MIDNIGHT.getEndTimestamp()}, + }; + + static { + LINES.forEach(line -> LINE_BY_TIME.put(line.time, line.id)); + } private final JdbcDatabaseContainer server; private final JDBCQueryITSupport.Target target; private JDBCQueryITSupport support; private JDBCEventQueryDAO eventDAO; private JDBCBrowserLogQueryDAO browserLogDAO; + private JDBCLogQueryDAO logDAO; JDBCLogQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { this.server = server; @@ -89,7 +169,7 @@ abstract class JDBCLogQueryITBase { } @BeforeAll - public void writeEventsAndBrowserErrors() throws Exception { + public void writeEventsBrowserErrorsAndLogs() throws Exception { support = JDBCQueryITSupport.connect(server, target); final var eventModel = support.install( Event.class, DefaultScopeDefine.EVENT, Event.INDEX_NAME, DownSampling.Second); @@ -106,10 +186,22 @@ public void writeEventsAndBrowserErrors() throws Exception { for (final Object[] error : BROWSER_ERRORS) { errors.add(browserError((String) error[0], MIDNIGHT + (int) error[1] * 1000L)); } + for (final Object[] error : BOUND_BROWSER_ERRORS) { + errors.add(browserError(BOUNDS_BROWSER_SERVICE, (String) error[0], (long) error[1])); + } support.writeRecords(browserErrorModel, new BrowserErrorLogRecord.Builder(), errors); eventDAO = new JDBCEventQueryDAO(support.getClient(), support.getTableHelper()); browserLogDAO = new JDBCBrowserLogQueryDAO(support.getClient(), support.getTableHelper()); + + final var logModel = support.install( + LogRecord.class, DefaultScopeDefine.LOG, LogRecord.INDEX_NAME, DownSampling.Second); + support.writeRecords(logModel, new LogRecord.Builder(), LINES.stream() + .map(Line::record).collect(Collectors.toList())); + final ConfigService config = support.getModuleManager().find(CoreModule.NAME).provider() + .getService(ConfigService.class); + when(config.getSearchableLogsTags()).thenReturn(SEARCHABLE_LOG_TAGS); + logDAO = new JDBCLogQueryDAO(support.getClient(), support.getModuleManager(), support.getTableHelper()); } @AfterAll @@ -132,13 +224,13 @@ public void readsTheEventsOfALayerOnBothDays() throws Exception { } /** - * A minute window holds the events from its first millisecond up to, not including, the next minute's first, so - * an event on a minute's first millisecond belongs to that minute alone. + * A minute window holds the events from its first millisecond up to and including the next minute's first, its + * end, so an event on a minute's first millisecond is read by that minute and by the minute before it. */ @Test - public void readsAnEventOnItsMinutesFirstMillisecond() throws Exception { - assertEquals(Collections.singletonList("e2"), uuids(eventDAO.queryEvents(minute(MIDNIGHT - 60_000L)))); - assertEquals(Arrays.asList("e4", "e3"), uuids(eventDAO.queryEvents(minute(MIDNIGHT)))); + public void readsAnEventOnAMinutesFirstMillisecondInBothWindowsItBounds() throws Exception { + assertEquals(Arrays.asList("e3", "e2"), uuids(eventDAO.queryEvents(minute(MIDNIGHT - 60_000L)))); + assertEquals(Arrays.asList("e5", "e4", "e3"), uuids(eventDAO.queryEvents(minute(MIDNIGHT)))); assertEquals(Collections.singletonList("e5"), uuids(eventDAO.queryEvents(minute(MIDNIGHT + 60_000L)))); } @@ -198,18 +290,23 @@ private static Event event(final String uuid, final long startTime, final Layer } private static BrowserErrorLogRecord browserError(final String uniqueId, final long time) { - final String version = IDManager.ServiceInstanceID.buildId(BROWSER_SERVICE_ID, "v1.0.0"); + return browserError(BROWSER_SERVICE, uniqueId, time); + } + + private static BrowserErrorLogRecord browserError(final String service, final String uniqueId, final long time) { + final String serviceId = IDManager.ServiceID.buildId(service, true); + final String version = IDManager.ServiceInstanceID.buildId(serviceId, "v1.0.0"); final BrowserErrorLogRecord record = new BrowserErrorLogRecord(); record.setUniqueId(uniqueId); - record.setServiceId(BROWSER_SERVICE_ID); + record.setServiceId(serviceId); record.setServiceVersionId(version); - record.setPagePathId(IDManager.EndpointID.buildId(BROWSER_SERVICE_ID, "/cart")); + record.setPagePathId(IDManager.EndpointID.buildId(serviceId, "/cart")); record.setErrorCategory(BrowserErrorCategory.JS.getValue()); record.setTimestamp(time); record.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); record.setDataBinary(BrowserErrorLog.newBuilder() .setUniqueId(uniqueId) - .setService(BROWSER_SERVICE) + .setService(service) .setServiceVersion("v1.0.0") .setPagePath("/cart") .setCategory(ErrorCategory.js) @@ -219,4 +316,283 @@ private static BrowserErrorLogRecord browserError(final String uniqueId, final l .toByteArray()); return record; } + + @Test + public void readsTheLogsOfAServiceNewestOrOldestFirst() throws Exception { + assertEquals(Arrays.asList("l5", "l4", "l3", "l2", "l1"), lines(read().service(SERVICE))); + assertEquals(Arrays.asList("l1", "l2", "l3", "l4", "l5"), lines(read().service(SERVICE).order(Order.ASC))); + assertEquals(Collections.singletonList("l6"), lines(read().service(PAYMENT))); + } + + @Test + public void readsTheLogsOfAnInstance() throws Exception { + assertEquals(Arrays.asList("l5", "l4", "l2", "l1"), lines(read().instance(SERVICE, "checkout-1"))); + } + + @Test + public void readsTheLogsOfAnEndpoint() throws Exception { + assertEquals(Arrays.asList("l3", "l2", "l1"), lines(read().endpoint(SERVICE, POST_ORDERS))); + } + + @Test + public void readsTheLogsOfATrace() throws Exception { + assertEquals(Arrays.asList("l3", "l2", "l1"), lines(read().trace(TRACE_A, null, null, null))); + assertEquals(Arrays.asList("l5", "l4"), lines(read().trace(TRACE_B, null, null, null))); + } + + /** + * A native span is its segment and its number within the segment: the trace's other span of the segment, and + * the span of the same number in its other segment, stay out. + */ + @Test + public void readsTheLogsOfANativeSpan() throws Exception { + assertEquals(Collections.singletonList("l1"), lines(read().trace(TRACE_A, "segment-a1", 1, null))); + assertEquals(Collections.singletonList("l2"), lines(read().trace(TRACE_A, "segment-a1", 2, null))); + } + + @Test + public void readsTheLogsOfAnOtlpSpan() throws Exception { + assertEquals(Collections.singletonList("l4"), lines(read().trace(TRACE_B, null, null, OTLP_SPAN))); + } + + @Test + public void readsTheLogsHavingEverySearchableTag() throws Exception { + assertEquals(Arrays.asList("l6", "l4", "l3", "l2"), lines(read().tags("level=ERROR"))); + assertEquals(Arrays.asList("l6", "l4", "l2"), lines(read().tags("level=ERROR", "http.status_code=500"))); + assertEquals(Collections.emptyList(), lines(read().tags("thread=main"))); + } + + @Test + public void pagesThroughTheLogsOfAService() throws Exception { + assertEquals(Arrays.asList("l5", "l4"), lines(read().service(SERVICE).page(0, 2))); + assertEquals(Arrays.asList("l3", "l2"), lines(read().service(SERVICE).page(2, 2))); + assertEquals(Collections.singletonList("l1"), lines(read().service(SERVICE).page(4, 2))); + assertEquals(Collections.emptyList(), lines(read().service(SERVICE).page(6, 2))); + } + + @Test + public void readsEachLogWhole() throws Exception { + final List expected = new ArrayList<>(); + for (int i = 5; i >= 0; i--) { + expected.add(LINES.get(i).render()); + } + assertEquals(expected, read().run(logDAO).stream().map(JDBCLogQueryITBase::render) + .collect(Collectors.toList())); + } + + @Test + public void readsTheLogsOfEveryDayWithoutADuration() throws Exception { + assertEquals(Arrays.asList("l3", "l2", "l1"), lines(read().trace(TRACE_A, null, null, null).during(null))); + } + + @Test + public void keepsTheLogsWithinTheDuration() throws Exception { + assertEquals(Arrays.asList("last-millisecond", "on-start"), lines(read().service(BOUNDS).during(BOUNDED))); + } + + /** + * A duration ending in a day's last minute ends at the next midnight, which is in the next day's table. + */ + @Test + public void readsALogOnTheInstantTheDurationEndsAt() throws Exception { + assertEquals(Collections.singletonList("on-end"), lines(read().service(BOUNDS).during(ENDING))); + assertEquals(Collections.singletonList("on-midnight"), + lines(read().service(BOUNDS).during(ENDING_AT_MIDNIGHT))); + } + + @Test + public void keepsTheBrowserErrorsWithinTheDuration() { + assertEquals(Arrays.asList("last-millisecond", "on-start"), boundBrowserErrors(BOUNDED)); + } + + @Test + public void readsABrowserErrorOnTheInstantTheDurationEndsAt() { + assertEquals(Collections.singletonList("on-end"), boundBrowserErrors(ENDING)); + assertEquals(Collections.singletonList("on-midnight"), boundBrowserErrors(ENDING_AT_MIDNIGHT)); + } + + private List boundBrowserErrors(final Duration duration) { + return browserLogDAO.queryBrowserErrorLogs(IDManager.ServiceID.buildId(BOUNDS_BROWSER_SERVICE, true), null, + null, null, duration, 10, 0) + .getLogs() + .stream() + .map(log -> log.getMessage()) + .collect(Collectors.toList()); + } + + private List lines(final LogRead read) { + return read.run(logDAO).stream().map(log -> LINE_BY_TIME.get(log.getTimestamp())).collect(Collectors.toList()); + } + + private static LogRead read() { + return new LogRead(); + } + + private static String render(final Log log) { + return String.join("|", log.getServiceId(), log.getServiceInstanceId(), log.getEndpointId(), + log.getEndpointName(), log.getTraceId(), String.valueOf(log.getTimestamp()), + log.getContentType().name(), log.getContent(), log.getTags().stream() + .map(tag -> tag.getKey() + "=" + tag.getValue()).collect(Collectors.joining(","))); + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + /** + * The arguments of one log read, as the log query service passes them. + */ + private static final class LogRead { + private String serviceId; + private String instanceId; + private String endpointId; + private TraceScopeCondition trace; + private Order order = Order.DES; + private int from = 0; + private int limit = 100; + private Duration duration = AROUND_MIDNIGHT; + private List tags; + + LogRead service(final String service) { + serviceId = serviceId(service); + return this; + } + + LogRead instance(final String service, final String instance) { + instanceId = IDManager.ServiceInstanceID.buildId(serviceId(service), instance); + return this; + } + + LogRead endpoint(final String service, final String endpoint) { + endpointId = IDManager.EndpointID.buildId(serviceId(service), endpoint); + return this; + } + + LogRead trace(final String traceId, final String segmentId, final Integer spanId, final String otlpSpanId) { + trace = new TraceScopeCondition(); + trace.setTraceId(traceId); + trace.setSegmentId(segmentId); + trace.setSpanId(spanId); + trace.setOtlpSpanId(otlpSpanId); + return this; + } + + LogRead order(final Order order) { + this.order = order; + return this; + } + + LogRead page(final int from, final int limit) { + this.from = from; + this.limit = limit; + return this; + } + + LogRead during(final Duration duration) { + this.duration = duration; + return this; + } + + LogRead tags(final String... tags) { + this.tags = Arrays.stream(tags).map(tag -> tag.split("=")).map(kv -> new Tag(kv[0], kv[1])) + .collect(Collectors.toList()); + return this; + } + + List run(final JDBCLogQueryDAO dao) { + return dao.queryLogs(serviceId, instanceId, endpointId, trace, order, from, limit, duration, tags, + null, null).getLogs(); + } + } + + /** + * A log line as the log analyzer hands it to the record stream: every tag in the raw tags, the searchable ones + * indexed. + */ + private static final class Line { + private final String id; + private final long time; + private final String serviceId; + private final String instanceId; + private final String endpointId; + private final String endpoint; + private String traceId; + private String segmentId; + private int spanId; + private String otlpSpanId; + private ContentType contentType; + private String content; + private List tags = Collections.emptyList(); + + Line(final String id, final long time, final String service, final String instance, final String endpoint) { + this.id = id; + this.time = time; + this.serviceId = serviceId(service); + this.instanceId = IDManager.ServiceInstanceID.buildId(serviceId, instance); + this.endpointId = endpoint == null ? null : IDManager.EndpointID.buildId(serviceId, endpoint); + this.endpoint = endpoint; + } + + Line nativeSpan(final String traceId, final String segmentId, final int spanId) { + this.traceId = traceId; + this.segmentId = segmentId; + this.spanId = spanId; + return this; + } + + Line otlpSpan(final String traceId, final String otlpSpanId) { + this.traceId = traceId; + this.otlpSpanId = otlpSpanId; + return this; + } + + Line text(final String content) { + this.contentType = ContentType.TEXT; + this.content = content; + return this; + } + + Line json(final String content) { + this.contentType = ContentType.JSON; + this.content = content; + return this; + } + + Line tags(final String... tags) { + this.tags = Arrays.asList(tags); + return this; + } + + LogRecord record() { + final LogRecord record = new LogRecord(); + record.setUniqueId(id); + record.setServiceId(serviceId); + record.setServiceInstanceId(instanceId); + record.setEndpointId(endpointId); + record.setTraceId(traceId); + record.setTraceSegmentId(segmentId); + record.setSpanId(spanId); + record.setOtlpSpanId(otlpSpanId); + record.setContentType(contentType.value()); + record.setContent(new LongText(content)); + record.setTimestamp(time); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + if (!tags.isEmpty()) { + final LogTags.Builder raw = LogTags.newBuilder(); + tags.forEach(tag -> raw.addData(KeyStringValuePair.newBuilder() + .setKey(tag.split("=")[0]) + .setValue(tag.split("=")[1]))); + record.setTagsRawData(raw.build().toByteArray()); + } + final List searchable = Arrays.asList(SEARCHABLE_LOG_TAGS.split(",")); + record.setTagsInString(tags.stream().filter(tag -> searchable.contains(tag.split("=")[0])) + .collect(Collectors.toList())); + return record; + } + + String render() { + return String.join("|", serviceId, instanceId, endpointId, endpoint, traceId, String.valueOf(time), + contentType.name(), content, String.join(",", tags)); + } + } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCManagementITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCManagementITBase.java index ebd59483829f..b1bfb57b86d1 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCManagementITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCManagementITBase.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -30,12 +31,15 @@ import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics; import org.apache.skywalking.oap.server.core.analysis.metrics.IntList; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.management.runtimerule.RuntimeRule; import org.apache.skywalking.oap.server.core.management.ui.template.UITemplate; +import org.apache.skywalking.oap.server.core.profiling.continuous.storage.ContinuousProfilingPolicy; import org.apache.skywalking.oap.server.core.query.input.DashboardSetting; import org.apache.skywalking.oap.server.core.query.type.DashboardConfiguration; import org.apache.skywalking.oap.server.core.query.type.TemplateChangeStatus; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.SessionCacheCallback; +import org.apache.skywalking.oap.server.core.storage.management.RuntimeRuleManagementDAO; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.core.storage.model.StorageManipulationOpt; import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; @@ -57,9 +61,10 @@ import static org.mockito.Mockito.verify; /** - * Manages UI templates the way the UI management API does, persists management rows and updates metrics the way - * OAP's workers do, and inspects installed tables the way the boot and the runtime-rule reconciler do. A subclass - * names the server. + * Manages UI templates the way the UI management API does, continuous-profiling policies the way the policy page and + * the eBPF agents do, and runtime rules the way the runtime-rule API and its reconciler do; persists management rows + * and updates metrics the way OAP's workers do, and inspects installed tables the way the boot and the runtime-rule + * reconciler do. A subclass names the server. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) abstract class JDBCManagementITBase { @@ -101,6 +106,11 @@ public void writeTemplatesAndRelation() throws Exception { support.writeMetrics( relationModel, new ServiceRelationServerSideMetrics.Builder(), Collections.singletonList(relation(CHECKOUT_ID, PAYMENT_ID, SPRING_MVC))); + + support.install( + ContinuousProfilingPolicy.class, DefaultScopeDefine.CONTINUOUS_PROFILING_POLICY, + ContinuousProfilingPolicy.INDEX_NAME, DownSampling.None); + support.install(RuntimeRule.class, DefaultScopeDefine.RUNTIME_RULE, RuntimeRule.INDEX_NAME, DownSampling.None); } @AfterAll @@ -217,6 +227,76 @@ public void reportsATableMissingAColumn() throws Exception { info.buildInstallInfoMsg()); } + /** + * A service has one policy: saving it again, as the policy page does on every change, replaces the whole policy. + * The eBPF agents ask for the policies of their services, some of which have none. + */ + @Test + public void savesAPolicyPerServiceAndReplacesItOnTheNextSave() throws Exception { + final JDBCContinuousProfilingPolicyDAO dao = + new JDBCContinuousProfilingPolicyDAO(support.getClient(), support.getTableHelper()); + final String withoutPolicy = IDManager.ServiceID.buildId("policy-less", true); + final ContinuousProfilingPolicy checkout = policy(CHECKOUT_ID, "{\"targets\":[\"ON_CPU\"]}", "uuid-1"); + final ContinuousProfilingPolicy payment = policy(PAYMENT_ID, "{\"targets\":[\"OFF_CPU\"]}", "uuid-2"); + dao.savePolicy(checkout); + dao.savePolicy(payment); + assertEquals(policies(checkout), policies(dao.queryPolicies(Collections.singletonList(CHECKOUT_ID)))); + + final ContinuousProfilingPolicy changed = policy(CHECKOUT_ID, "{\"targets\":[\"NETWORK\"]}", "uuid-3"); + dao.savePolicy(changed); + assertEquals(policies(changed, payment), + policies(dao.queryPolicies(Arrays.asList(CHECKOUT_ID, PAYMENT_ID, withoutPolicy)))); + assertEquals(policies(payment), policies(dao.queryPolicies(Collections.singletonList(PAYMENT_ID)))); + assertEquals(Collections.emptyList(), dao.queryPolicies(Collections.singletonList(withoutPolicy))); + } + + /** + * Runtime rules are kept by catalog and name, the same name in two catalogs being two rules. A save replaces the + * whole rule, as {@code /addOrUpdate} and {@code /inactivate} do, and a delete removes the rule of one catalog, + * silently when there is none, as a retried {@code /delete} does. The reconciler reads every rule, so each read + * is compared whole, with the rules already kept. + */ + @Test + public void savesReplacesAndDeletesRulesByCatalogAndName() throws Exception { + final JDBCRuntimeRuleManagementDAO dao = ruleDAO(); + final List kept = rules(dao.getAll()); + final String active = RuntimeRule.STATUS_ACTIVE; + final RuntimeRule vm = rule("otel-rules", "vm", "metricPrefix: vm", active, 1_000L); + final RuntimeRule logVm = rule("log-mal-rules", "vm", "metricPrefix: log_vm", active, 1_001L); + final RuntimeRule mysql = rule("otel-rules", "mysql", "metricPrefix: mysql", active, 1_002L); + for (final RuntimeRule rule : Arrays.asList(vm, logVm, mysql)) { + dao.save(rule); + } + assertEquals(rules(kept, vm, logVm, mysql), rules(dao.getAll())); + + final RuntimeRule inactiveVm = + rule("otel-rules", "vm", "metricPrefix: vm_v2", RuntimeRule.STATUS_INACTIVE, 2_000L); + dao.save(inactiveVm); + assertEquals(rules(kept, inactiveVm, logVm, mysql), rules(dao.getAll())); + + dao.delete("log-mal-rules", "vm"); + assertEquals(rules(kept, inactiveVm, mysql), rules(dao.getAll())); + dao.delete("log-mal-rules", "vm"); + dao.delete("otel-rules", "mysql"); + assertEquals(rules(kept, inactiveVm), rules(dao.getAll())); + } + + /** + * A rule's name is its path under the catalog, so it may hold a '/'. + */ + @Test + public void savesAndDeletesARuleNamedByAPath() throws Exception { + final JDBCRuntimeRuleManagementDAO dao = ruleDAO(); + final List kept = rules(dao.getAll()); + final RuntimeRule gateway = rule("otel-rules", "aws-gateway/gateway-service", "metricPrefix: aws_gateway", + RuntimeRule.STATUS_ACTIVE, 3_000L); + dao.save(gateway); + assertEquals(rules(kept, gateway), rules(dao.getAll())); + + dao.delete("otel-rules", "aws-gateway/gateway-service"); + assertEquals(kept, rules(dao.getAll())); + } + private List templateIds(final boolean includingDisabled) { return templateDAO.getAllTemplates(includingDisabled) .stream() @@ -253,4 +333,63 @@ private static ServiceRelationServerSideMetrics relation(final String sourceServ relation.setTimeBucket(TimeBucket.getMinuteTimeBucket(MIDNIGHT)); return relation; } + + private JDBCRuntimeRuleManagementDAO ruleDAO() { + return new JDBCRuntimeRuleManagementDAO(support.getClient(), support.getTableHelper()); + } + + private static ContinuousProfilingPolicy policy(final String serviceId, final String configuration, + final String uuid) { + final ContinuousProfilingPolicy policy = new ContinuousProfilingPolicy(); + policy.setServiceId(serviceId); + policy.setConfigurationJson(configuration); + policy.setUuid(uuid); + return policy; + } + + private static List policies(final ContinuousProfilingPolicy... policies) { + return policies(Arrays.asList(policies)); + } + + private static List policies(final List policies) { + return policies.stream() + .map(policy -> String.join("|", policy.getServiceId(), policy.getConfigurationJson(), + policy.getUuid())) + .sorted() + .collect(Collectors.toList()); + } + + private static RuntimeRule rule(final String catalog, final String name, final String content, + final String status, final long updateTime) { + final RuntimeRule rule = new RuntimeRule(); + rule.setCatalog(catalog); + rule.setName(name); + rule.setContent(content); + rule.setStatus(status); + rule.setUpdateTime(updateTime); + return rule; + } + + private static List rules(final List kept, final RuntimeRule... rules) { + final List all = new ArrayList<>(kept); + all.addAll(rules(rules)); + Collections.sort(all); + return all; + } + + private static List rules(final RuntimeRule... rules) { + return Arrays.stream(rules) + .map(rule -> String.join("|", rule.getCatalog(), rule.getName(), rule.getContent(), + rule.getStatus(), String.valueOf(rule.getUpdateTime()))) + .sorted() + .collect(Collectors.toList()); + } + + private static List rules(final List files) { + return files.stream() + .map(file -> String.join("|", file.getCatalog(), file.getName(), file.getContent(), + file.getStatus(), String.valueOf(file.getUpdateTime()))) + .sorted() + .collect(Collectors.toList()); + } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryITBase.java index 475360a0b2d3..bd2660e7b46b 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetadataQueryITBase.java @@ -18,29 +18,42 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +import com.google.gson.Gson; import com.google.gson.JsonObject; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; import java.util.stream.Collectors; import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; import org.apache.skywalking.oap.server.core.analysis.manual.instance.InstanceTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.networkalias.NetworkAddressAlias; import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessDetectType; import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.process.ServiceLabelRecord; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagAutocompleteData; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.TagType; +import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; import org.apache.skywalking.oap.server.core.query.enumeration.ProfilingSupportStatus; import org.apache.skywalking.oap.server.core.query.enumeration.Step; import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.Attribute; +import org.apache.skywalking.oap.server.core.query.type.Endpoint; import org.apache.skywalking.oap.server.core.query.type.Process; +import org.apache.skywalking.oap.server.core.query.type.Service; import org.apache.skywalking.oap.server.core.query.type.ServiceInstance; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -49,16 +62,25 @@ import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Reads instances, the processes an eBPF agent reports and searchable tags the way the metadata query, the eBPF - * receiver and tag autocompletion do, on traffic first stored on both sides of midnight, so in two day tables: a - * traffic row stays in the table of the day it was first stored, however long it keeps pinging. A subclass names the - * server. + * Reads services, instances, endpoints, the processes an eBPF agent reports, the labels of a service's processes, + * network address aliases and searchable tags the way the metadata query, the eBPF receiver and profiling, the alias + * cache and tag autocompletion do, on traffic first stored on both sides of midnight, so in two day tables: a traffic + * row stays in the table of the day it was first stored, however long it keeps pinging, and a row written again + * (an instance that pings, an alias that changes) is updated in that table. A subclass names the server. + * + *

A row is created at one minute and last seen at another, so a read by a window applies both bounds: created no + * later than the window's end, last seen no earlier than its start. The window most reads use crosses midnight; rows + * sit on each of its bounds and one minute outside, and rows of another service, instance or agent match every other + * condition. Processes are read by an instance at minute, hour and day steps, as the query protocol allows any step. + * Endpoints are found newest first, across the day tables. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) abstract class JDBCMetadataQueryITBase { + private static final Gson GSON = new Gson(); private static final int METADATA_QUERY_MAX_SIZE = 5000; private static final String SERVICE_ID = IDManager.ServiceID.buildId("checkout", true); private static final String INSTANCE_FIRST_SEEN_YESTERDAY = "checkout-1"; @@ -86,13 +108,31 @@ abstract class JDBCMetadataQueryITBase { {"http.status_code", "503"}, }; + private static final String ORDERS = "orders"; + private static final String ORDERS_ID = IDManager.ServiceID.buildId(ORDERS, true); + private static final String PAYMENT = "finance::payment"; + private static final String PAYMENT_ID = IDManager.ServiceID.buildId(PAYMENT, true); + private static final String DATABASE = "mysql.orders:3306"; + private static final String ORDERS_INSTANCE = "orders-1"; + private static final String ORDERS_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(ORDERS_ID, ORDERS_INSTANCE); + private static final String PAYMENT_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(PAYMENT_ID, "payment-1"); + private static final String ORDERS_AGENT = "rover-node-2"; + /** The first and the last minute, counted from midnight, of the window most reads use. */ + private static final int WINDOW_START = -10; + private static final int WINDOW_END = 10; + private final JdbcDatabaseContainer server; private final JDBCQueryITSupport.Target target; + private final Map instanceRows = new HashMap<>(); + private final Map processRows = new HashMap<>(); private JDBCQueryITSupport support; private JDBCMetadataQueryDAO metadataDAO; private JDBCTagAutoCompleteQueryDAO tagDAO; + private JDBCNetworkAddressAliasDAO aliasDAO; + private JDBCServiceLabelQueryDAO labelDAO; private long now; private Duration aroundMidnight; + private Duration window; JDBCMetadataQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { this.server = server; @@ -108,13 +148,52 @@ public void writeTraffic() throws Exception { final long yesterday = MIDNIGHT - 2 * 3_600_000L; support.writeMetrics( - support.install(InstanceTraffic.class, DefaultScopeDefine.SERVICE_INSTANCE, InstanceTraffic.INDEX_NAME, + support.install(ServiceTraffic.class, DefaultScopeDefine.SERVICE, ServiceTraffic.INDEX_NAME, DownSampling.Minute), + new ServiceTraffic.Builder(), + Arrays.asList( + service(ORDERS, Layer.GENERAL, -60), + service(ORDERS, Layer.MESH, 5), + service(PAYMENT, Layer.GENERAL, 5), + service(DATABASE, Layer.VIRTUAL_DATABASE, -60))); + + final Model instanceModel = support.install( + InstanceTraffic.class, DefaultScopeDefine.SERVICE_INSTANCE, InstanceTraffic.INDEX_NAME, + DownSampling.Minute); + support.writeMetrics( + instanceModel, new InstanceTraffic.Builder(), Arrays.asList( instance(INSTANCE_FIRST_SEEN_YESTERDAY, yesterday, now), - instance(INSTANCE_FIRST_SEEN_TODAY, now, now))); + instance(INSTANCE_FIRST_SEEN_TODAY, now, now), + ordersInstance("seen-across-midnight", -60, 30, "java", "hostname", "orders-host-0"), + ordersInstance("pinged-at-start", -60, WINDOW_START, "java"), + ordersInstance("pinged-before-start", -60, WINDOW_START - 1, "java"), + ordersInstance("created-at-end", WINDOW_END, 30, null), + ordersInstance("created-after-end", WINDOW_END + 1, 30, null), + ordersInstance("updated", -60, -30, "java"), + instance(PAYMENT_ID, "payment-1", -60, 30, properties("python")))); + // The metrics worker writes an instance again when it pings, with its first minute kept. + support.updateMetrics(instanceModel, new InstanceTraffic.Builder(), + Collections.singletonList(ordersInstance("updated", -60, 5, "java"))); + + support.writeMetrics( + support.install(EndpointTraffic.class, DefaultScopeDefine.ENDPOINT, EndpointTraffic.INDEX_NAME, + DownSampling.Minute), + new EndpointTraffic.Builder(), + Arrays.asList( + endpoint(ORDERS_ID, "GET:/api/orders", -60, 30), + endpoint(ORDERS_ID, "GET:/api/orders/archive", -61, WINDOW_START - 1), + endpoint(ORDERS_ID, "PATCH:/api/orders", -59, WINDOW_START), + endpoint(ORDERS_ID, "POST:/api/orders", -5, 30), + endpoint(ORDERS_ID, "GET:/api/users", 5, 30), + endpoint(ORDERS_ID, "DELETE:/api/orders", WINDOW_END, WINDOW_END), + endpoint(ORDERS_ID, "PUT:/api/orders", WINDOW_END + 1, 30), + endpoint(PAYMENT_ID, "GET:/api/orders", -60, 30))); + final JsonObject host = new JsonObject(); + host.addProperty("host_name", "node-0"); + host.addProperty("pid", "1001"); support.writeMetrics( support.install(ProcessTraffic.class, DefaultScopeDefine.PROCESS, ProcessTraffic.INDEX_NAME, DownSampling.Minute), @@ -122,7 +201,25 @@ public void writeTraffic() throws Exception { Arrays.asList( process(INSTANCE_FIRST_SEEN_YESTERDAY_ID, "java-app", yesterday, now), process(INSTANCE_FIRST_SEEN_TODAY_ID, "envoy", now, now), - process(INSTANCE_FIRST_SEEN_YESTERDAY_ID, "batch-job", yesterday, now - 30 * 60_000L))); + process(INSTANCE_FIRST_SEEN_YESTERDAY_ID, "batch-job", yesterday, now - 30 * 60_000L), + ordersProcess("first-seen-yesterday", ProcessDetectType.VM, + ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, -60, 30, host, "env:prod", "team:a"), + ordersProcess("first-seen-today", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + 5, 30, null), + ordersProcess("virtual", ProcessDetectType.VIRTUAL, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + -60, 30, null), + ordersProcess("not-supported", ProcessDetectType.KUBERNETES, ProfilingSupportStatus.NOT_SUPPORT, + -60, 30, null), + ordersProcess("pinged-at-start", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + -60, WINDOW_START, null), + ordersProcess("pinged-before-start", ProcessDetectType.VM, + ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, -60, WINDOW_START - 1, null), + ordersProcess("created-at-end", ProcessDetectType.VM, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, + WINDOW_END, WINDOW_END, null), + ordersProcess("created-after-end", ProcessDetectType.VM, + ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, WINDOW_END + 1, 30, null), + process(PAYMENT_ID, PAYMENT_INSTANCE_ID, "rover-node-3", "payment-process", ProcessDetectType.VM, + ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, -60, 30, null))); final List tags = new ArrayList<>(); for (final String[] tag : TAGS_YESTERDAY) { @@ -136,9 +233,32 @@ public void writeTraffic() throws Exception { TagAutocompleteData.INDEX_NAME, DownSampling.Minute), new TagAutocompleteData.Builder(), tags); - metadataDAO = new JDBCMetadataQueryDAO(support.getClient(), METADATA_QUERY_MAX_SIZE, support.getModuleManager()); + final Model aliasModel = support.install( + NetworkAddressAlias.class, DefaultScopeDefine.NETWORK_ADDRESS_ALIAS, NetworkAddressAlias.INDEX_NAME, + DownSampling.Minute); + support.writeMetrics(aliasModel, new NetworkAddressAlias.Builder(), Arrays.asList( + alias("10.0.0.1:80", ORDERS_INSTANCE_ID, -60, -30), + alias("10.0.0.2:80", ORDERS_INSTANCE_ID, -60, WINDOW_START), + alias("10.0.0.3:80", ORDERS_INSTANCE_ID, -60, WINDOW_START - 1), + alias("10.0.0.4:80", PAYMENT_INSTANCE_ID, 5, 5))); + // An alias set up again by another instance is updated in place, with its first minute kept. + support.updateMetrics(aliasModel, new NetworkAddressAlias.Builder(), + Collections.singletonList(alias("10.0.0.1:80", PAYMENT_INSTANCE_ID, -60, 5))); + + support.writeMetrics( + support.install(ServiceLabelRecord.class, DefaultScopeDefine.SERVICE_LABEL, ServiceLabelRecord.INDEX_NAME, + DownSampling.Minute), + new ServiceLabelRecord.Builder(), + Arrays.asList(label(ORDERS_ID, "env:prod", -60), label(ORDERS_ID, "team:a", 5), + label(PAYMENT_ID, "env:test", -60))); + + metadataDAO = new JDBCMetadataQueryDAO( + support.getClient(), METADATA_QUERY_MAX_SIZE, support.getModuleManager()); tagDAO = new JDBCTagAutoCompleteQueryDAO(support.getClient(), support.getTableHelper()); + aliasDAO = new JDBCNetworkAddressAliasDAO(support.getClient(), support.getModuleManager()); + labelDAO = new JDBCServiceLabelQueryDAO(support.getClient(), support.getTableHelper()); aroundMidnight = JDBCQueryITSupport.duration(MIDNIGHT - 30 * 60_000L, MIDNIGHT + 30 * 60_000L, Step.MINUTE); + window = JDBCQueryITSupport.duration(minute(WINDOW_START), minute(WINDOW_END), Step.MINUTE); } @AfterAll @@ -148,6 +268,27 @@ public void close() { } } + @Test + public void listsEveryServiceOfEveryDayTable() throws Exception { + assertEquals( + new TreeSet<>(Arrays.asList( + ORDERS_ID + "|orders|orders|null|GENERAL|true", + ORDERS_ID + "|orders|orders|null|MESH|true", + PAYMENT_ID + "|finance::payment|payment|finance|GENERAL|true", + IDManager.ServiceID.buildId(DATABASE, false) + "|" + DATABASE + "|" + DATABASE + + "|null|VIRTUAL_DATABASE|false")), + serviceRows(metadataDAO.listServices())); + } + + @Test + public void limitsTheServicesToTheMetadataQueryMaxSize() throws Exception { + final Set all = serviceRows(metadataDAO.listServices()); + final Set limited = serviceRows( + new JDBCMetadataQueryDAO(support.getClient(), 2, support.getModuleManager()).listServices()); + assertEquals(2, limited.size()); + assertTrue(all.containsAll(limited)); + } + @Test public void readsTheInstancesOfEveryDayTable() throws Exception { assertEquals(new TreeSet<>(Arrays.asList(INSTANCE_FIRST_SEEN_YESTERDAY, INSTANCE_FIRST_SEEN_TODAY)), @@ -156,6 +297,97 @@ public void readsTheInstancesOfEveryDayTable() throws Exception { instanceNames(INSTANCE_FIRST_SEEN_YESTERDAY_ID)); } + @Test + public void listsTheInstancesOfAServiceAliveInTheWindow() throws Exception { + assertEquals(expectedInstances("seen-across-midnight", "pinged-at-start", "created-at-end", "updated"), + instanceRows(metadataDAO.listInstances(window, ORDERS_ID))); + } + + @Test + public void listsEveryInstanceOfAServiceWithoutAWindow() throws Exception { + assertEquals(expectedInstances("seen-across-midnight", "pinged-at-start", "pinged-before-start", + "created-at-end", "created-after-end", "updated"), + instanceRows(metadataDAO.listInstances(null, ORDERS_ID))); + } + + @Test + public void readsAnInstanceByIdFromEitherDayTable() throws Exception { + assertEquals(instanceRows.get("seen-across-midnight"), + instanceRow(metadataDAO.getInstance(ordersInstanceId("seen-across-midnight")))); + assertEquals(instanceRows.get("created-at-end"), + instanceRow(metadataDAO.getInstance(ordersInstanceId("created-at-end")))); + assertNull(metadataDAO.getInstance(ordersInstanceId("missing"))); + } + + @Test + public void findsTheEndpointsMatchingAKeywordNewestFirst() throws Exception { + assertEquals(endpointRows("DELETE:/api/orders", "POST:/api/orders", "PATCH:/api/orders", "GET:/api/orders"), + endpointRows(metadataDAO.findEndpoint("orders", ORDERS_ID, 10, window))); + assertEquals(endpointRows("PUT:/api/orders", "DELETE:/api/orders", "POST:/api/orders", "PATCH:/api/orders", + "GET:/api/orders", "GET:/api/orders/archive"), + endpointRows(metadataDAO.findEndpoint("orders", ORDERS_ID, 10, null))); + // JDBC matches the keyword anywhere in the name, not by words. + assertEquals(endpointRows("GET:/api/orders/archive"), + endpointRows(metadataDAO.findEndpoint("orders/arch", ORDERS_ID, 10, null))); + } + + @Test + public void findsTheNewestEndpointsUpToTheLimit() throws Exception { + assertEquals(endpointRows("DELETE:/api/orders", "GET:/api/users"), + endpointRows(metadataDAO.findEndpoint(null, ORDERS_ID, 2, window))); + assertEquals(endpointRows("DELETE:/api/orders", "GET:/api/users", "POST:/api/orders"), + endpointRows(metadataDAO.findEndpoint(null, ORDERS_ID, 3, window))); + } + + /** + * As the eBPF profiling and continuous profiling read them: profiling-supported or every non-virtual process + * alive in the window, or, without an end, every one pinging since the start. + */ + @Test + public void listsTheProcessesOfAServiceAliveInTheWindow() throws Exception { + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "pinged-at-start", "created-at-end"), + processRows(metadataDAO.listProcesses( + ORDERS_ID, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, bucket(WINDOW_START), + bucket(WINDOW_END)))); + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "not-supported", "pinged-at-start", + "created-at-end"), + processRows(metadataDAO.listProcesses(ORDERS_ID, null, bucket(WINDOW_START), bucket(WINDOW_END)))); + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "not-supported", "pinged-at-start", + "created-at-end", "created-after-end"), + processRows(metadataDAO.listProcesses(ORDERS_ID, null, bucket(WINDOW_START), 0))); + } + + @Test + public void listsTheProcessesOfAnInstanceAliveInTheWindow() throws Exception { + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "not-supported", "pinged-at-start", + "created-at-end"), + processRows(metadataDAO.listProcesses(ORDERS_INSTANCE_ID, window, false))); + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "virtual", "not-supported", + "pinged-at-start", "created-at-end"), + processRows(metadataDAO.listProcesses(ORDERS_INSTANCE_ID, window, true))); + } + + /** + * The process list of the query protocol takes a duration of any step; the rows' minutes are compared with the + * minutes the duration covers, the first hour after midnight or the last before it, and today or yesterday. + */ + @Test + public void listsTheProcessesOfAnInstanceInHourAndDayWindows() throws Exception { + final Set today = expectedProcesses( + "first-seen-yesterday", "first-seen-today", "virtual", "not-supported", "created-at-end", + "created-after-end"); + final Set yesterday = expectedProcesses( + "first-seen-yesterday", "virtual", "not-supported", "pinged-at-start", "pinged-before-start"); + assertEquals(today, processRows(metadataDAO.listProcesses( + ORDERS_INSTANCE_ID, JDBCQueryITSupport.duration(MIDNIGHT, MIDNIGHT, Step.HOUR), true))); + assertEquals(yesterday, processRows(metadataDAO.listProcesses( + ORDERS_INSTANCE_ID, JDBCQueryITSupport.duration(minute(-1), minute(-1), Step.HOUR), true))); + assertEquals(today, processRows(metadataDAO.listProcesses( + ORDERS_INSTANCE_ID, JDBCQueryITSupport.duration(MIDNIGHT, MIDNIGHT, Step.DAY), true))); + assertEquals(yesterday, processRows(metadataDAO.listProcesses( + ORDERS_INSTANCE_ID, JDBCQueryITSupport.duration(minute(-1), minute(-1), Step.DAY), true))); + } + /** * The eBPF receiver asks for the processes of an agent that pinged in the last five minutes; batch-job stopped * pinging half an hour ago. @@ -168,6 +400,60 @@ public void listsTheProcessesOfAnAgentThatPingedInTheRange() throws Exception { processes.stream().map(Process::getName).collect(Collectors.toCollection(TreeSet::new))); } + @Test + public void listsTheProcessesOfAnAgentAliveInTheWindow() throws Exception { + assertEquals(expectedProcesses("first-seen-yesterday", "first-seen-today", "not-supported", "pinged-at-start", + "created-at-end"), + processRows(metadataDAO.listProcesses(ORDERS_AGENT, bucket(WINDOW_START), bucket(WINDOW_END)))); + } + + @Test + public void countsTheProcessesOfAServiceAndOfAnInstance() throws Exception { + assertEquals(4, metadataDAO.getProcessCount( + ORDERS_ID, ProfilingSupportStatus.SUPPORT_EBPF_PROFILING, bucket(WINDOW_START), bucket(WINDOW_END))); + // Every non-virtual process of the instance, whenever it pinged. + assertEquals(7, metadataDAO.getProcessCount(ORDERS_INSTANCE_ID)); + } + + @Test + public void readsAProcessByIdFromEitherDayTable() throws Exception { + assertEquals( + String.join("|", IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, "first-seen-yesterday"), + "first-seen-yesterday", ORDERS_ID, ORDERS, ORDERS_INSTANCE_ID, ORDERS_INSTANCE, ORDERS_AGENT, + "VM", "SUPPORT_EBPF_PROFILING", "host_name=node-0,pid=1001", "env:prod,team:a"), + processRow(metadataDAO.getProcess( + IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, "first-seen-yesterday")))); + assertEquals(processRows.get("first-seen-today"), + processRow(metadataDAO.getProcess(IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, + "first-seen-today")))); + assertNull(metadataDAO.getProcess(IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, "missing"))); + } + + /** + * As each node's alias cache reloads them: every alias whose last update is on or after a minute, from the table + * of the day it was first set up. + */ + @Test + public void loadsTheAliasesUpdatedSinceAMinute() { + assertEquals( + Arrays.asList( + "10.0.0.1:80|" + PAYMENT_ID + "|" + PAYMENT_INSTANCE_ID + "|" + bucket(5) + "|" + bucket(-60), + "10.0.0.2:80|" + ORDERS_ID + "|" + ORDERS_INSTANCE_ID + "|" + bucket(WINDOW_START) + "|" + bucket(-60), + "10.0.0.4:80|" + PAYMENT_ID + "|" + PAYMENT_INSTANCE_ID + "|" + bucket(5) + "|" + bucket(5)), + aliasRows(aliasDAO.loadLastUpdate(bucket(WINDOW_START)))); + assertEquals(Arrays.asList("10.0.0.1:80", "10.0.0.4:80"), + aliasDAO.loadLastUpdate(bucket(5)).stream().map(NetworkAddressAlias::getAddress).sorted() + .collect(Collectors.toList())); + assertEquals(Collections.emptyList(), aliasDAO.loadLastUpdate(bucket(6))); + } + + @Test + public void readsTheLabelsOfAServiceFromEveryDayTable() { + assertEquals(Arrays.asList("env:prod", "team:a"), + labelDAO.queryAllLabels(ORDERS_ID).stream().sorted().collect(Collectors.toList())); + assertEquals(Collections.emptyList(), labelDAO.queryAllLabels(IDManager.ServiceID.buildId("missing", true))); + } + @Test public void limitsTheTagKeysOfBothDays() throws Exception { final Set keys = new TreeSet<>(); @@ -184,12 +470,41 @@ public void limitsTheTagKeysOfBothDays() throws Exception { assertTrue(keys.containsAll(limited)); } + /** + * A tag is kept for the whole day it is seen in, so a range ends with the day of its last step, though its end + * instant, the next midnight, opens the next day: a range ending in the last minute before midnight, and a range + * of one day, read none of the next day's tags. + */ + @Test + public void readsNoTagOfTheDayAfterTheLastStep() throws Exception { + final Set yesterdayKeys = new TreeSet<>(); + for (final String[] tag : TAGS_YESTERDAY) { + yesterdayKeys.add(tag[0]); + } + for (final Duration yesterday : Arrays.asList( + JDBCQueryITSupport.duration(MIDNIGHT - 30 * 60_000L, MIDNIGHT - 60_000L, Step.MINUTE), + JDBCQueryITSupport.duration(MIDNIGHT - 30 * 60_000L, MIDNIGHT - 30 * 60_000L, Step.DAY))) { + assertEquals(yesterdayKeys, tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 100, yesterday)); + assertEquals(new TreeSet<>(Arrays.asList("200", "404")), + tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 100, yesterday)); + } + final Set todayKeys = new TreeSet<>(); + for (final String[] tag : TAGS_TODAY) { + todayKeys.add(tag[0]); + } + final Duration today = JDBCQueryITSupport.duration(now, now, Step.DAY); + assertEquals(todayKeys, tagDAO.queryTagAutocompleteKeys(TagType.TRACE, 100, today)); + assertEquals(new TreeSet<>(Arrays.asList("500", "503")), + tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 100, today)); + } + @Test public void limitsTheTagValuesOfBothDays() throws Exception { final Set values = new TreeSet<>(Arrays.asList("200", "404", "500", "503")); assertEquals(values, tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 100, aroundMidnight)); - final Set limited = tagDAO.queryTagAutocompleteValues(TagType.TRACE, "http.status_code", 3, aroundMidnight); + final Set limited = tagDAO.queryTagAutocompleteValues( + TagType.TRACE, "http.status_code", 3, aroundMidnight); assertEquals(3, limited.size()); assertTrue(values.containsAll(limited)); } @@ -201,6 +516,89 @@ private Set instanceNames(final String... ids) throws Exception { .collect(Collectors.toCollection(TreeSet::new)); } + private Set expectedInstances(final String... names) { + return Arrays.stream(names).map(instanceRows::get).collect(Collectors.toCollection(TreeSet::new)); + } + + private Set expectedProcesses(final String... names) { + return Arrays.stream(names).map(processRows::get).collect(Collectors.toCollection(TreeSet::new)); + } + + private static Set serviceRows(final List services) { + return services.stream() + .map(s -> String.join("|", s.getId(), s.getName(), s.getShortName(), + String.valueOf(s.getGroup()), + String.join(",", new TreeSet<>(s.getLayers())), + String.valueOf(s.isNormal()))) + .collect(Collectors.toCollection(TreeSet::new)); + } + + private static Set instanceRows(final List instances) { + return instances.stream().map(JDBCMetadataQueryITBase::instanceRow) + .collect(Collectors.toCollection(TreeSet::new)); + } + + private static String instanceRow(final ServiceInstance instance) { + return String.join("|", instance.getId(), instance.getName(), instance.getInstanceUUID(), + instance.getLanguage().name(), attributes(instance.getAttributes())); + } + + private static List endpointRows(final String... names) { + return Arrays.stream(names) + .map(name -> IDManager.EndpointID.buildId(ORDERS_ID, name) + "|" + name) + .collect(Collectors.toList()); + } + + private static List endpointRows(final List endpoints) { + return endpoints.stream().map(e -> e.getId() + "|" + e.getName()).collect(Collectors.toList()); + } + + private static Set processRows(final List processes) { + return processes.stream().map(JDBCMetadataQueryITBase::processRow) + .collect(Collectors.toCollection(TreeSet::new)); + } + + private static String processRow(final Process process) { + return String.join("|", process.getId(), process.getName(), process.getServiceId(), process.getServiceName(), + process.getInstanceId(), process.getInstanceName(), process.getAgentId(), + process.getDetectType(), process.getProfilingSupportStatus(), + attributes(process.getAttributes()), String.join(",", process.getLabels())); + } + + private static List aliasRows(final List aliases) { + return aliases.stream() + .map(a -> String.join("|", a.getAddress(), a.getRepresentServiceId(), + a.getRepresentServiceInstanceId(), + String.valueOf(a.getLastUpdateTimeBucket()), + String.valueOf(a.getTimeBucket()))) + .sorted() + .collect(Collectors.toList()); + } + + private static String attributes(final Collection attributes) { + return attributes.stream().map(a -> a.getName() + "=" + a.getValue()).collect(Collectors.joining(",")); + } + + private static long minute(final int minutesFromMidnight) { + return MIDNIGHT + minutesFromMidnight * 60_000L; + } + + private static long bucket(final int minutesFromMidnight) { + return TimeBucket.getMinuteTimeBucket(minute(minutesFromMidnight)); + } + + private static String ordersInstanceId(final String name) { + return IDManager.ServiceInstanceID.buildId(ORDERS_ID, name); + } + + private static ServiceTraffic service(final String name, final Layer layer, final int created) { + final ServiceTraffic service = new ServiceTraffic(); + service.setName(name); + service.setLayer(layer); + service.setTimeBucket(bucket(created)); + return service; + } + private static InstanceTraffic instance(final String name, final long firstSeen, final long lastPing) { final JsonObject properties = new JsonObject(); properties.addProperty(InstanceTraffic.PropertyUtil.LANGUAGE, "java"); @@ -213,6 +611,54 @@ private static InstanceTraffic instance(final String name, final long firstSeen, return instance; } + private static InstanceTraffic instance(final String serviceId, final String name, final int created, + final int lastPing, final JsonObject properties) { + final InstanceTraffic instance = new InstanceTraffic(); + instance.setServiceId(serviceId); + instance.setName(name); + instance.setProperties(properties); + instance.setTimeBucket(bucket(created)); + instance.setLastPingTimestamp(bucket(lastPing)); + return instance; + } + + /** + * An instance of the orders service, and the row the metadata query reads for it. + * + * @param language the language the agent reports, or null for an instance that reports no properties + * @param attributes the other properties, as key and value + */ + private InstanceTraffic ordersInstance(final String name, final int created, final int lastPing, + final String language, final String... attributes) { + final List expected = new ArrayList<>(); + for (int i = 0; i < attributes.length; i += 2) { + expected.add(new Attribute(attributes[i], attributes[i + 1])); + } + final String id = ordersInstanceId(name); + instanceRows.put(name, String.join("|", id, name, id, language == null ? "UNKNOWN" : language.toUpperCase(), + attributes(expected))); + return instance(ORDERS_ID, name, created, lastPing, language == null ? null : properties(language, attributes)); + } + + private static JsonObject properties(final String language, final String... attributes) { + final JsonObject properties = new JsonObject(); + properties.addProperty(InstanceTraffic.PropertyUtil.LANGUAGE, language); + for (int i = 0; i < attributes.length; i += 2) { + properties.addProperty(attributes[i], attributes[i + 1]); + } + return properties; + } + + private static EndpointTraffic endpoint(final String serviceId, final String name, final int created, + final int lastPing) { + final EndpointTraffic endpoint = new EndpointTraffic(); + endpoint.setServiceId(serviceId); + endpoint.setName(name); + endpoint.setTimeBucket(bucket(created)); + endpoint.setLastPingTimestamp(bucket(lastPing)); + return endpoint; + } + private static ProcessTraffic process(final String instanceId, final String name, final long firstSeen, final long lastPing) { final ProcessTraffic process = new ProcessTraffic(); @@ -228,6 +674,41 @@ private static ProcessTraffic process(final String instanceId, final String name return process; } + private static ProcessTraffic process(final String serviceId, final String instanceId, final String agentId, + final String name, final ProcessDetectType detectType, + final ProfilingSupportStatus status, final int created, final int lastPing, + final JsonObject properties, final String... labels) { + final ProcessTraffic process = new ProcessTraffic(); + process.setServiceId(serviceId); + process.setInstanceId(instanceId); + process.setName(name); + process.setAgentId(agentId); + process.setDetectType(detectType.value()); + process.setProfilingSupportStatus(status.value()); + process.setProperties(properties); + process.setLabelsJson(labels.length == 0 ? Const.EMPTY_STRING : GSON.toJson(labels)); + process.setTimeBucket(bucket(created)); + process.setLastPingTimestamp(bucket(lastPing)); + return process; + } + + /** + * A process of the orders instance, reported by its agent, and the row the metadata query reads for it. + */ + private ProcessTraffic ordersProcess(final String name, final ProcessDetectType detectType, + final ProfilingSupportStatus status, final int created, final int lastPing, + final JsonObject properties, final String... labels) { + final List expected = new ArrayList<>(); + if (properties != null) { + properties.entrySet().forEach(p -> expected.add(new Attribute(p.getKey(), p.getValue().getAsString()))); + } + processRows.put(name, String.join("|", IDManager.ProcessID.buildId(ORDERS_INSTANCE_ID, name), name, ORDERS_ID, + ORDERS, ORDERS_INSTANCE_ID, ORDERS_INSTANCE, ORDERS_AGENT, detectType.name(), + status.name(), attributes(expected), String.join(",", labels))); + return process(ORDERS_ID, ORDERS_INSTANCE_ID, ORDERS_AGENT, name, detectType, status, created, lastPing, + properties, labels); + } + private static TagAutocompleteData tag(final String[] tag, final long firstSeen) { final TagAutocompleteData data = new TagAutocompleteData(); data.setTagKey(tag[0]); @@ -236,4 +717,23 @@ private static TagAutocompleteData tag(final String[] tag, final long firstSeen) data.setTimeBucket(TimeBucket.retainToDay4MinuteBucket(TimeBucket.getMinuteTimeBucket(firstSeen))); return data; } + + private static NetworkAddressAlias alias(final String address, final String instanceId, final int created, + final int lastUpdate) { + final NetworkAddressAlias alias = new NetworkAddressAlias(); + alias.setAddress(address); + alias.setRepresentServiceId(IDManager.ServiceInstanceID.analysisId(instanceId).getServiceId()); + alias.setRepresentServiceInstanceId(instanceId); + alias.setLastUpdateTimeBucket(bucket(lastUpdate)); + alias.setTimeBucket(bucket(created)); + return alias; + } + + private static ServiceLabelRecord label(final String serviceId, final String label, final int created) { + final ServiceLabelRecord record = new ServiceLabelRecord(); + record.setServiceId(serviceId); + record.setLabel(label); + record.setTimeBucket(bucket(created)); + return record; + } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsDAOTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsDAOTest.java new file mode 100644 index 000000000000..bcdb48656d06 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsDAOTest.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.service.ServiceTraffic; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class JDBCMetricsDAOTest { + private static final DateTimeFormatter DAY = DateTimeFormatter.BASIC_ISO_DATE; + private static final int[] TTLS = {1, 3, 7}; + private static final List DOWNSAMPLINGS = + Arrays.asList(DownSampling.Minute, DownSampling.Hour, DownSampling.Day); + + /** + * A metric is expired exactly when its day is before the first day table {@link JDBCHistoryDeleteDAO} keeps, the + * day of {@code ttl - 1} days of elapsed time before now, at every instant: as now moves forward and back, and as + * the TTL changes. The checked metrics are in the first and the last minute of each day around the boundary. + */ + @Test + void expiresAMetricWithTheDayTableTheTTLDropsAtEveryInstant() { + final String zoneId = ZoneId.systemDefault().getId(); + final long start = LocalDate.of(2027, 3, 10).atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli(); + final List instants = new ArrayList<>(); + for (long now = start; now < start + TimeUnit.DAYS.toMillis(10); now += TimeUnit.MINUTES.toMillis(13)) { + instants.add(now); + } + final JDBCMetricsDAO dao = new JDBCMetricsDAO(null, null); + for (final DownSampling downSampling : DOWNSAMPLINGS) { + final Model model = mock(Model.class); + when(model.getDownsampling()).thenReturn(downSampling); + for (final int ttl : TTLS) { + for (final long now : instants) { + check(dao, model, now, ttl, zoneId); + } + for (int i = instants.size() - 1; i >= 0; i--) { + check(dao, model, instants.get(i), ttl, zoneId); + } + } + } + } + + private static void check(final JDBCMetricsDAO dao, final Model model, final long now, final int ttl, + final String zoneId) { + final long firstKeptDay = TimeBucket.getTimeBucket(now - TimeUnit.DAYS.toMillis(ttl - 1), DownSampling.Day); + final LocalDate today = LocalDate.parse(String.valueOf(TimeBucket.getTimeBucket(now, DownSampling.Day)), DAY); + for (LocalDate day = today.minusDays(ttl + 1); !day.isAfter(today); day = day.plusDays(1)) { + final long dayBucket = Long.parseLong(day.format(DAY)); + for (final long bucket : buckets(dayBucket, model.getDownsampling())) { + final ServiceTraffic metric = new ServiceTraffic(); + metric.setTimeBucket(bucket); + assertEquals(dayBucket < firstKeptDay, dao.isExpiredCache(model, metric, now, ttl), + zoneId + " now=" + now + " ttl=" + ttl + " bucket=" + bucket); + } + } + } + + /** + * The first and the last bucket of the day at the downsampling. + */ + private static long[] buckets(final long dayBucket, final DownSampling downSampling) { + switch (downSampling) { + case Minute: + return new long[] {dayBucket * 10_000, dayBucket * 10_000 + 2359}; + case Hour: + return new long[] {dayBucket * 100, dayBucket * 100 + 23}; + default: + return new long[] {dayBucket}; + } + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsQueryITBase.java index b484bbed919d..ade89255183e 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCMetricsQueryITBase.java @@ -18,39 +18,56 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +import com.google.gson.Gson; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.stream.Collectors; import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.Layer; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.cache.TopNCacheReadCommand; import org.apache.skywalking.oap.server.core.analysis.manual.database.TopNDatabaseStatement; import org.apache.skywalking.oap.server.core.analysis.meter.MeterEntity; import org.apache.skywalking.oap.server.core.analysis.meter.function.AcceptableValue; import org.apache.skywalking.oap.server.core.analysis.meter.function.BucketedValues; import org.apache.skywalking.oap.server.core.analysis.meter.function.avg.AvgFunction; import org.apache.skywalking.oap.server.core.analysis.meter.function.avg.AvgHistogramFunction; +import org.apache.skywalking.oap.server.core.analysis.meter.function.avg.AvgLabeledFunction; +import org.apache.skywalking.oap.server.core.analysis.metrics.ApdexMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.CPMMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.DataTable; +import org.apache.skywalking.oap.server.core.analysis.metrics.LongAvgMetrics; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.PercentileMetrics2; import org.apache.skywalking.oap.server.core.analysis.topn.TopN; import org.apache.skywalking.oap.server.core.config.NamingControl; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; import org.apache.skywalking.oap.server.core.query.enumeration.Order; import org.apache.skywalking.oap.server.core.query.enumeration.Scope; import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.AttrCondition; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.query.input.Entity; import org.apache.skywalking.oap.server.core.query.input.MetricsCondition; import org.apache.skywalking.oap.server.core.query.input.RecordCondition; import org.apache.skywalking.oap.server.core.query.input.TopNCondition; import org.apache.skywalking.oap.server.core.query.type.HeatMap; +import org.apache.skywalking.oap.server.core.query.type.KVInt; +import org.apache.skywalking.oap.server.core.query.type.KeyValue; +import org.apache.skywalking.oap.server.core.query.type.MetricsValues; import org.apache.skywalking.oap.server.core.query.type.Record; import org.apache.skywalking.oap.server.core.query.type.SelectedRecord; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata; import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -62,15 +79,38 @@ /** * Reads metrics the way the metrics query service, the top-N ranking and the top-N records do, over ranges that - * cross midnight, so over two day tables: a heat map of a MAL histogram, MAL {@code vm.yaml} metrics ranked across - * the hosts, and the slow statements of a database. A subclass names the server. + * cross midnight, so over two day tables. The metrics are the classes OAP generates from the shipped {@code core.oal} + * and MAL meters of their function classes, so each value column is read under the name its function stores it by: + * a long ({@code service_instance_resp_time}, minute and hour), an int ({@code service_apdex}), the percentile labels + * ({@code service_percentile}), a MAL histogram, MAL labeled meters, the call rate whose entities are listed, and MAL + * {@code vm.yaml} meters ranked across hosts. A subclass names the server. + * + *

The edge cases: a minute and an hour without a point, read as empty, beside another instance that has every + * point; a labeled row without one of the labels; points just outside each range that is filtered by time; metrics of + * the same function, which share the day tables and are told apart by {@code table_name}, on the same entities and + * minutes; more labeled rows than the read of every entity returns; entities ranked by the mean of the whole range, + * which no single minute or day ranks the same way; endpoints of another service; statements on and just outside the + * bounds of the range, its end instant included, on the end instant of a range ending at midnight, of another + * database, and cache commands of the same entity. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) abstract class JDBCMetricsQueryITBase { private static final String LATENCY_HISTOGRAM = "meter_checkout_latency_histogram"; private static final String CPU = "meter_vm_cpu_total_percentage"; private static final String MEMORY = "meter_vm_memory_used"; + private static final String LOAD = "meter_vm_load"; + private static final String INSTANCE_RESP_TIME = "service_instance_resp_time"; + private static final String RESP_TIME = "service_resp_time"; + private static final String MQ_LATENCY = "service_mq_consume_latency"; + private static final String ENDPOINT_RESP_TIME = "endpoint_resp_time"; + private static final String APDEX = "service_apdex"; + private static final String PERCENTILE = "service_percentile"; + private static final String CPM = "service_cpm"; + private static final String ENDPOINT_CPM = "endpoint_cpm"; + private static final String POOL_SIZE = "meter_pool_size"; + private static final String POOL_WAIT = "meter_pool_wait"; private static final String SERVICE = "checkout"; + private static final String OTHER_SERVICE = "payment"; private static final long[] HISTOGRAM_BUCKETS = {0, 50, 100}; /** Minute from midnight, then the count of each bucket. Minute -1 has no histogram. */ private static final long[][] HISTOGRAMS = { @@ -88,26 +128,99 @@ abstract class JDBCMetricsQueryITBase { {"batch-1", new long[] {60, 60, 60, 5, -1, -1}}, {"backup-1", new long[] {-1, -1, 5, 62, 62, 62}}, }; + /** + * Hosts on each side of midnight that top web-1 on their own day, a dozen each, so a ranking that merges a few + * tops of each day table loses web-1. web-1 is steady at 50 from minute -3 to 2; each day host is 60 until + * midnight and 5 at minute 0, each night host 5 at minute -1 and 62 from midnight, so they average 46 and 47. + */ + private static final int HOSTS_PER_DAY = 12; private static final long GIB = 1L << 30; /** Host, then the used memory in bytes of each minute from -2 to 0: db-1 averages 26 GiB, web-1 4 GiB. */ private static final Object[][] MEMORY_MINUTES = { {"db-1", new long[] {30 * GIB, 30 * GIB, 18 * GIB}}, {"web-1", new long[] {-1, 3 * GIB, 5 * GIB}}, }; + /** Minute from midnight, then checkout-1's response time. Minute -1 has none; checkout-2 has one every minute. */ + private static final long[][] INSTANCE_MINUTES = {{-3, 110}, {-2, 120}, {0, 140}, {1, 150}, {2, 160}}; + /** Hour from midnight, then checkout-1's response time. Hour -1 has none; checkout-2 has one every hour. */ + private static final long[][] INSTANCE_HOURS = {{-2, 1000}, {0, 3000}, {1, 4000}}; + /** Service, then its apdex at minutes -1, 0 and 1; -1 is no point. Over minutes -1 and 0: 10000, 5000, 2500. */ + private static final Object[][] APDEX_MINUTES = { + {SERVICE, new long[] {10000, -1, 5000}}, + {OTHER_SERVICE, new long[] {7500, 2500, -1}}, + {"auth", new long[] {-1, 2500, 10000}}, + }; + /** + * Service, its layer (attr0), then its response time at minutes -1 and 0. The means, 50, 60 and 10, rank them + * unlike either minute does. resp-c also has 1000 at minutes -2 and 1, just outside the range ranked, and resp-a + * has 1 at both minutes in {@code service_mq_consume_latency}, a metric of the same function. + */ + private static final Object[][] RANKED_SERVICES = { + {"resp-a", "GENERAL", new long[] {10, 90}}, + {"resp-b", "GENERAL", new long[] {60, 60}}, + {"resp-c", "MESH", new long[] {5, 15}}, + }; + /** Endpoint of checkout, then its response time at minutes -1 and 0; payment's /order is 500 at both. */ + private static final Object[][] RANKED_ENDPOINTS = { + {"/order", new long[] {10, 90}}, + {"/pay", new long[] {60, 60}}, + {"/cart", new long[] {5, 15}}, + }; + /** + * Service, then the minutes it has a call rate at. The range listed is minutes -3 to 2: cpm-e and cpm-f are just + * outside it, cpm-g and cpm-h on its bounds, and an endpoint of checkout has {@code endpoint_cpm} at minute 2. + */ + private static final Object[][] CPM_MINUTES = { + {"cpm-a", new long[] {-3, -2}}, + {"cpm-b", new long[] {-1}}, + {"cpm-c", new long[] {-3, 0}}, + {"cpm-d", new long[] {1}}, + {"cpm-e", new long[] {3}}, + {"cpm-f", new long[] {-4}}, + {"cpm-g", new long[] {2}}, + {"cpm-h", new long[] {-3}}, + }; + /** + * Pool, minute, then its active and idle connections; -1 is a label the row does not have. The range read is + * minutes -1 and 0: pool-3 at minute -2 and pool-0 at minute 1 are just outside it. + */ + private static final long[][] POOLS = { + {0, -1, 10, 100}, + {1, -1, 20, 200}, + {2, -1, 30, -1}, + {0, 0, 11, 101}, + {1, 0, 21, 201}, + {3, -2, 98, 998}, + {0, 1, 99, 999}, + }; + /** Waiting pools at each of minutes -1 and 0: more rows than a read of every entity returns. */ + private static final int WAITING_POOLS = 6; private static final String DATABASE = "orders-db:3306"; - /** Second from midnight, latency in ms. Three before midnight, three after. */ + /** + * Second from midnight, latency in ms. The range read, minutes -2 to 1, is read from second -120 through its end + * instant, second 120, so the first and last statements sit on its bounds. + */ private static final long[][] STATEMENTS = { + {-120, 600}, {-110, 1200}, {-80, 300}, {-40, 900}, {15, 1500}, {45, 100}, {90, 700}, + {119, 50}, + {120, 20}, }; + /** The slowest and the fastest statements of the database, each a millisecond outside the range. */ + private static final long[][] MILLIS_OUTSIDE = {{-120_001, 5000}, {120_001, 10}}; + /** Its range ends in the last minute before midnight, so at midnight, the first instant of the next day. */ + private static final String MIDNIGHT_DATABASE = "midnight-db:3306"; private final JdbcDatabaseContainer server; private final JDBCQueryITSupport.Target target; private JDBCQueryITSupport support; + private final Map models = new HashMap<>(); + private final Map waiting = new TreeMap<>(); JDBCMetricsQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { this.server = server; @@ -118,9 +231,8 @@ abstract class JDBCMetricsQueryITBase { public void writeMetrics() throws Exception { support = JDBCQueryITSupport.connect(server, target); MeterEntity.setNamingControl(new NamingControl(70, 100, 150, new EndpointNameGrouping())); + installModels(); - final Model histogram = support.install( - HistogramMeter.class, DefaultScopeDefine.SERVICE, LATENCY_HISTOGRAM, DownSampling.Minute); final List histograms = new ArrayList<>(); for (final long[] point : HISTOGRAMS) { final HistogramMeter meter = new HistogramMeter(); @@ -130,19 +242,25 @@ public void writeMetrics() throws Exception { meter.calculate(); histograms.add(meter); } - support.writeMetrics(histogram, new AvgHistogramFunction.AvgHistogramFunctionBuilder(), histograms); + support.writeMetrics(model(LATENCY_HISTOGRAM, DownSampling.Minute), + new AvgHistogramFunction.AvgHistogramFunctionBuilder(), histograms); writeHostMetric(CPU, CPU_MINUTES, -3); writeHostMetric(MEMORY, MEMORY_MINUTES, -2); - - final Model statementModel = support.install( - TopNDatabaseStatement.class, DefaultScopeDefine.DATABASE_SLOW_STATEMENT, TopNDatabaseStatement.INDEX_NAME, - DownSampling.Second); - final List statements = new ArrayList<>(); - for (int i = 0; i < STATEMENTS.length; i++) { - statements.add(statement(DATABASE, "segment-" + i, STATEMENTS[i][0], STATEMENTS[i][1])); + final List loads = new ArrayList<>(); + loads.add(new Object[] {"web-1", new long[] {50, 50, 50, 50, 50, 50}}); + for (int i = 0; i < HOSTS_PER_DAY; i++) { + loads.add(new Object[] {"day-" + i, new long[] {60, 60, 60, 5, -1, -1}}); + loads.add(new Object[] {"night-" + i, new long[] {-1, -1, 5, 62, 62, 62}}); } - support.writeRecords(statementModel, new TopNDatabaseStatement.Builder(), statements); + writeHostMetric(LOAD, loads.toArray(new Object[0][]), -3); + writeInstanceResponseTimes(); + writeApdex(); + writePercentiles(); + writeRankedResponseTimes(); + writeCallRates(); + writePools(); + writeStatements(); } @AfterAll @@ -152,6 +270,52 @@ public void close() { } } + /** + * Every minute of the range in minute order, each from whichever day table holds it, and the minute none holds + * as empty. + */ + @Test + public void readsEveryMinuteOfARangeAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("110", "120", "empty", "140", "150", "160"), + values(INSTANCE_RESP_TIME, instance("checkout-1"), minutes(-3, 2))); + assertEquals(Arrays.asList("empty", "empty"), + values(INSTANCE_RESP_TIME, instance("checkout-9"), minutes(-1, 0))); + } + + @Test + public void readsEveryHourOfARangeAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("1000", "empty", "3000", "4000"), + values(INSTANCE_RESP_TIME, instance("checkout-1"), hours(-2, 1))); + } + + @Test + public void readsTheIntValuesOfAMetric() throws Exception { + assertEquals(Arrays.asList("10000", "empty", "5000"), values(APDEX, service(SERVICE), minutes(-1, 1))); + } + + /** + * Each percentile label over every minute of the range, the minute without a point as empty. + */ + @Test + public void readsEveryLabelOfALabeledMetricAcrossMidnight() throws Exception { + final Map> expected = new LinkedHashMap<>(); + expected.put("{p=50}", Arrays.asList("500", "empty", "1000")); + expected.put("{p=75}", Arrays.asList("750", "empty", "1500")); + expected.put("{p=90}", Arrays.asList("900", "empty", "1800")); + expected.put("{p=95}", Arrays.asList("950", "empty", "1900")); + expected.put("{p=99}", Arrays.asList("990", "empty", "1980")); + assertEquals(expected, labeled(Collections.emptyList())); + } + + @Test + public void readsTheLabelsAskedFor() throws Exception { + final Map> expected = new LinkedHashMap<>(); + expected.put("{p=50}", Arrays.asList("500", "empty", "1000")); + expected.put("{p=99}", Arrays.asList("990", "empty", "1980")); + assertEquals(expected, labeled(Collections.singletonList(new KeyValue("p", "50,99")))); + assertEquals(Collections.emptyMap(), labeled(Collections.singletonList(new KeyValue("p", "42")))); + } + /** * One column per minute of the range, in minute order, each from whichever day table holds it, and an empty * column for the minute none holds. @@ -159,12 +323,59 @@ public void close() { @Test public void readsOneHeatMapColumnPerMinuteAcrossMidnight() throws Exception { final HeatMap heatMap = support.metricsQueryDAO().readHeatMap( - serviceCondition(LATENCY_HISTOGRAM, SERVICE), ValueColumnMetadata.INSTANCE.getValueCName(LATENCY_HISTOGRAM), - minutes(-2, 1)); + condition(LATENCY_HISTOGRAM, service(SERVICE)), valueColumn(LATENCY_HISTOGRAM), minutes(-2, 1)); assertEquals( - Arrays.asList( - Arrays.asList(3L, 5L, 1L), Arrays.asList(0L, 0L, 0L), Arrays.asList(4L, 6L, 2L), Arrays.asList(5L, 7L, 3L)), + Arrays.asList(Arrays.asList(3L, 5L, 1L), Arrays.asList(0L, 0L, 0L), Arrays.asList(4L, 6L, 2L), + Arrays.asList(5L, 7L, 3L)), heatMap.getValues().stream().map(HeatMap.HeatMapColumn::getValues).collect(Collectors.toList())); + assertEquals("[{\"min\":\"0\",\"max\":\"50\"},{\"min\":\"50\",\"max\":\"100\"}," + + "{\"min\":\"100\",\"max\":\"infinite+\"}]", + new Gson().toJson(heatMap.getBuckets())); + } + + /** + * Every row of the metric in the range, from both day tables, each under every label; a row without a label + * reads as empty under it. + */ + @Test + public void readsTheLabeledValuesOfEveryEntityAcrossMidnight() throws Exception { + final Map> expected = new TreeMap<>(); + expected.put("{pool=active}", new TreeMap<>()); + expected.put("{pool=idle}", new TreeMap<>()); + for (final long[] pool : POOLS) { + if (pool[1] < -1 || pool[1] > 0) { + continue; + } + final String id = poolRowId(POOL_SIZE, pool[0], pool[1]); + expected.get("{pool=active}").put(id, String.valueOf(pool[2])); + expected.get("{pool=idle}").put(id, pool[3] < 0 ? "empty" : String.valueOf(pool[3])); + } + assertEquals(expected, everyEntity(POOL_SIZE, Collections.emptyList())); + + expected.remove("{pool=active}"); + assertEquals(expected, everyEntity(POOL_SIZE, Collections.singletonList(new KeyValue("pool", "idle")))); + } + + @Test + public void capsTheLabeledValuesOfEveryEntity() throws Exception { + final Map> read = everyEntity(POOL_WAIT, Collections.emptyList()); + assertEquals(Collections.singleton("{pool=waiting}"), read.keySet()); + final Map rows = read.get("{pool=waiting}"); + assertEquals(IMetricsQueryDAO.METRICS_VALUES_WITHOUT_ENTITY_LIMIT, rows.size()); + for (final Map.Entry row : rows.entrySet()) { + assertEquals(String.valueOf(waiting.get(row.getKey())), row.getValue(), row.getKey()); + } + } + + /** + * The entities with a point in the range, by their latest point, newest first: the latest of one is on the other + * day of the range, and the ones just outside the range or of another metric of the same function are not. + */ + @Test + public void listsTheEntitiesOfARangeNewestFirstAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("cpm-g", "cpm-d", "cpm-c", "cpm-b", "cpm-a", "cpm-h"), entities(minutes(-3, 2), 10)); + assertEquals(Arrays.asList("cpm-g", "cpm-d", "cpm-c"), entities(minutes(-3, 2), 3)); + assertEquals(Collections.emptyList(), entities(minutes(-6, -5), 10)); } /** @@ -172,7 +383,8 @@ public void readsOneHeatMapColumnPerMinuteAcrossMidnight() throws Exception { */ @Test public void ranksTheMeanOfTheWholeRangeAcrossMidnight() throws Exception { - assertEquals(Arrays.asList("web-1=50", "backup-1=47", "batch-1=46"), top(CPU, 3, minutes(-3, 2))); + assertEquals(Arrays.asList("web-1=50", "backup-1=47", "batch-1=46"), + rank(top(CPU, 3, Order.DES), minutes(-3, 2))); } /** @@ -180,28 +392,131 @@ public void ranksTheMeanOfTheWholeRangeAcrossMidnight() throws Exception { */ @Test public void keepsTheTopOfTheRangeThatTopsNoDay() throws Exception { - assertEquals(Collections.singletonList("web-1=50"), top(CPU, 1, minutes(-3, 2))); + assertEquals(Collections.singletonList("web-1=50"), rank(top(CPU, 1, Order.DES), minutes(-3, 2))); + } + + @Test + public void keepsTheTopOfTheRangeThatTopsNoDayAmongManyHosts() throws Exception { + assertEquals(Collections.singletonList("web-1=50"), rank(top(LOAD, 1, Order.DES), minutes(-3, 2))); } @Test public void ranksAMetricAboveTheIntRange() throws Exception { - assertEquals(Arrays.asList("db-1=" + 26 * GIB, "web-1=" + 4 * GIB), top(MEMORY, 2, minutes(-2, 0))); + assertEquals(Arrays.asList("db-1=" + 26 * GIB, "web-1=" + 4 * GIB), + rank(top(MEMORY, 2, Order.DES), minutes(-2, 0))); // Within one day the ranking is done in SQL. - assertEquals(Arrays.asList("db-1=" + 18 * GIB, "web-1=" + 5 * GIB), top(MEMORY, 2, minutes(0, 0))); + assertEquals(Arrays.asList("db-1=" + 18 * GIB, "web-1=" + 5 * GIB), + rank(top(MEMORY, 2, Order.DES), minutes(0, 0))); + } + + @Test + public void ranksTheServicesEitherWayAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("resp-b=60", "resp-a=50", "resp-c=10"), + rank(top(RESP_TIME, 10, Order.DES), minutes(-1, 0))); + assertEquals(Arrays.asList("resp-c=10", "resp-a=50", "resp-b=60"), + rank(top(RESP_TIME, 10, Order.ASC), minutes(-1, 0))); + assertEquals(Arrays.asList("resp-b=60", "resp-a=50"), rank(top(RESP_TIME, 2, Order.DES), minutes(-1, 0))); + assertEquals(Collections.singletonList("resp-c=10"), rank(top(RESP_TIME, 1, Order.ASC), minutes(-1, 0))); + assertEquals(Collections.emptyList(), rank(top(RESP_TIME, 10, Order.DES), minutes(-6, -5))); + } + + @Test + public void ranksTheServicesOfAnAttribute() throws Exception { + final TopNCondition general = top(RESP_TIME, 10, Order.DES); + general.setAttributes(Collections.singletonList(new AttrCondition("attr0", "GENERAL", true))); + assertEquals(Arrays.asList("resp-b=60", "resp-a=50"), rank(general, minutes(-1, 0))); + final TopNCondition notGeneral = top(RESP_TIME, 10, Order.DES); + notGeneral.setAttributes(Collections.singletonList(new AttrCondition("attr0", "GENERAL", false))); + assertEquals(Collections.singletonList("resp-c=10"), rank(notGeneral, minutes(-1, 0))); + } + + /** + * The endpoints of one service, as the top-N of a service's endpoints asks with its {@code service_id}. + */ + @Test + public void ranksTheEndpointsOfAService() throws Exception { + final List ofService = Collections.singletonList( + new KeyValue("service_id", IDManager.ServiceID.buildId(SERVICE, true))); + final TopNCondition descending = top(ENDPOINT_RESP_TIME, 10, Order.DES); + descending.setScope(Scope.Endpoint); + assertEquals(Arrays.asList("/pay=60", "/order=50", "/cart=10"), rank(descending, minutes(-1, 0), ofService)); + final TopNCondition ascending = top(ENDPOINT_RESP_TIME, 2, Order.ASC); + ascending.setScope(Scope.Endpoint); + assertEquals(Arrays.asList("/cart=10", "/order=50"), rank(ascending, minutes(-1, 0), ofService)); + } + + @Test + public void ranksAnIntMetricAcrossMidnight() throws Exception { + assertEquals(Arrays.asList("checkout=10000", "payment=5000", "auth=2500"), + rank(top(APDEX, 10, Order.DES), minutes(-1, 0))); + assertEquals(Arrays.asList("auth=2500", "payment=5000"), rank(top(APDEX, 2, Order.ASC), minutes(-1, 0))); } @Test public void readsTheSlowestStatementsAcrossMidnight() throws Exception { - assertEquals(Arrays.asList("1500", "1200", "900"), latencies(DATABASE, 3, Order.DES)); + assertEquals(Arrays.asList(statementRecord(4, 1500), statementRecord(1, 1200), statementRecord(3, 900)), + records(DATABASE, 3, Order.DES)); } @Test public void readsTheFastestStatementsAcrossMidnight() throws Exception { - assertEquals(Arrays.asList("100", "300"), latencies(DATABASE, 2, Order.ASC)); + assertEquals(Arrays.asList(statementRecord(8, 20), statementRecord(7, 50), statementRecord(5, 100)), + records(DATABASE, 3, Order.ASC)); + assertEquals(Collections.emptyList(), records("idle-db:3306", 3, Order.ASC)); + } + + /** + * A range ending in a day's last minute ends at the next midnight, which is in the next day's table. + */ + @Test + public void readsTheStatementOnTheEndInstantOfARangeEndingAtMidnight() throws Exception { + assertEquals(Collections.singletonList(statementRecord(40, 30)), + records(MIDNIGHT_DATABASE, 10, Order.DES, minutes(-2, -1))); + } + + /** + * Installs every model before the first write, as OAP does at boot. + */ + private void installModels() throws Exception { + installMeter(HistogramMeter.class, LATENCY_HISTOGRAM); + for (final String hostMetric : Arrays.asList(CPU, MEMORY, LOAD)) { + installMeter(AvgMeter.class, hostMetric); + } + installMeter(LabeledMeter.class, POOL_SIZE); + installMeter(LabeledMeter.class, POOL_WAIT); + for (final String metric : Arrays.asList(INSTANCE_RESP_TIME, APDEX, PERCENTILE, RESP_TIME, MQ_LATENCY, + ENDPOINT_RESP_TIME, CPM, ENDPOINT_CPM)) { + installOAL(metric, DownSampling.Minute); + } + installOAL(INSTANCE_RESP_TIME, DownSampling.Hour); + models.put(key(TopNDatabaseStatement.INDEX_NAME, DownSampling.Second), support.install( + TopNDatabaseStatement.class, DefaultScopeDefine.DATABASE_SLOW_STATEMENT, TopNDatabaseStatement.INDEX_NAME, + DownSampling.Second)); + models.put(key(TopNCacheReadCommand.INDEX_NAME, DownSampling.Second), support.install( + TopNCacheReadCommand.class, DefaultScopeDefine.CACHE_SLOW_ACCESS, TopNCacheReadCommand.INDEX_NAME, + DownSampling.Second)); + } + + private void installMeter(final Class meterClass, final String name) throws Exception { + models.put(key(name, DownSampling.Minute), + support.install(meterClass, DefaultScopeDefine.SERVICE, name, DownSampling.Minute)); + } + + private void installOAL(final String metric, final DownSampling downSampling) throws Exception { + final Class metricsClass = OALTestMetrics.metricsClass(metric); + models.put(key(metric, downSampling), + support.install(metricsClass, OALTestMetrics.scopeId(metricsClass), metric, downSampling)); + } + + private Model model(final String name, final DownSampling downSampling) { + return models.get(key(name, downSampling)); + } + + private static String key(final String name, final DownSampling downSampling) { + return name + "/" + downSampling; } private void writeHostMetric(final String name, final Object[][] hosts, final long firstMinute) throws Exception { - final Model model = support.install(AvgMeter.class, DefaultScopeDefine.SERVICE, name, DownSampling.Minute); final List metrics = new ArrayList<>(); for (final Object[] host : hosts) { final long[] values = (long[]) host[1]; @@ -216,24 +531,224 @@ private void writeHostMetric(final String name, final Object[][] hosts, final lo metrics.add(meter); } } - support.writeMetrics(model, new AvgFunction.AvgStorageBuilder(), metrics); + support.writeMetrics(model(name, DownSampling.Minute), new AvgFunction.AvgStorageBuilder(), metrics); } - /** The ranked hosts as {@code host=value}. */ - private List top(final String metric, final int topN, final Duration duration) throws Exception { - final TopNCondition condition = new TopNCondition(); - condition.setName(metric); - condition.setScope(Scope.Service); - condition.setTopN(topN); - condition.setOrder(Order.DES); + private void writeInstanceResponseTimes() throws Exception { + final List minutes = new ArrayList<>(); + for (final long[] point : INSTANCE_MINUTES) { + minutes.add(instanceResponseTime("checkout-1", minuteBucket(point[0]), point[1])); + } + for (long minute = -3; minute <= 2; minute++) { + minutes.add(instanceResponseTime("checkout-2", minuteBucket(minute), 7)); + } + writeOAL(INSTANCE_RESP_TIME, DownSampling.Minute, minutes); + + final List hours = new ArrayList<>(); + for (final long[] point : INSTANCE_HOURS) { + hours.add(instanceResponseTime("checkout-1", hourBucket(point[0]), point[1])); + } + for (long hour = -2; hour <= 1; hour++) { + hours.add(instanceResponseTime("checkout-2", hourBucket(hour), 7)); + } + writeOAL(INSTANCE_RESP_TIME, DownSampling.Hour, hours); + } + + private void writeApdex() throws Exception { + final List metrics = new ArrayList<>(); + for (final Object[] service : APDEX_MINUTES) { + final long[] values = (long[]) service[1]; + for (int i = 0; i < values.length; i++) { + if (values[i] < 0) { + continue; + } + // An apdex of n x 2500 is n satisfied calls of four. + final ApdexMetrics apdex = (ApdexMetrics) oal( + APDEX, serviceId((String) service[0]), minuteBucket(i - 1), + "totalNum", 4L, "sNum", values[i] / 2500, "tNum", 0L); + apdex.calculate(); + metrics.add(apdex); + } + } + writeOAL(APDEX, DownSampling.Minute, metrics); + } + + private void writePercentiles() throws Exception { + final List metrics = new ArrayList<>(); + // checkout's latencies are 10 to 1000 ms at minute -1 and twice that at minute 1. + metrics.add(percentile(SERVICE, -1, 1)); + metrics.add(percentile(SERVICE, 1, 2)); + for (long minute = -1; minute <= 1; minute++) { + metrics.add(percentile(OTHER_SERVICE, minute, 3)); + } + writeOAL(PERCENTILE, DownSampling.Minute, metrics); + } + + private void writeRankedResponseTimes() throws Exception { + final List services = new ArrayList<>(); + for (final Object[] service : RANKED_SERVICES) { + final long[] values = (long[]) service[2]; + for (int i = 0; i < values.length; i++) { + services.add(responseTime(RESP_TIME, serviceId((String) service[0]), minuteBucket(i - 1), values[i], + "attr0", service[1])); + } + } + services.add(responseTime(RESP_TIME, serviceId("resp-c"), minuteBucket(-2), 1000, "attr0", "MESH")); + services.add(responseTime(RESP_TIME, serviceId("resp-c"), minuteBucket(1), 1000, "attr0", "MESH")); + writeOAL(RESP_TIME, DownSampling.Minute, services); + + final List mq = new ArrayList<>(); + for (long minute = -1; minute <= 0; minute++) { + mq.add(responseTime(MQ_LATENCY, serviceId("resp-a"), minuteBucket(minute), 1, "attr0", "GENERAL")); + } + writeOAL(MQ_LATENCY, DownSampling.Minute, mq); + + final List endpoints = new ArrayList<>(); + for (final Object[] endpoint : RANKED_ENDPOINTS) { + final long[] values = (long[]) endpoint[1]; + for (int i = 0; i < values.length; i++) { + endpoints.add(responseTime( + ENDPOINT_RESP_TIME, endpointId(SERVICE, (String) endpoint[0]), minuteBucket(i - 1), values[i], + "serviceId", serviceId(SERVICE))); + } + } + for (long minute = -1; minute <= 0; minute++) { + endpoints.add(responseTime(ENDPOINT_RESP_TIME, endpointId(OTHER_SERVICE, "/order"), minuteBucket(minute), + 500, "serviceId", serviceId(OTHER_SERVICE))); + } + writeOAL(ENDPOINT_RESP_TIME, DownSampling.Minute, endpoints); + } + + private void writeCallRates() throws Exception { + final List services = new ArrayList<>(); + for (final Object[] service : CPM_MINUTES) { + for (final long minute : (long[]) service[1]) { + services.add(callRate(CPM, serviceId((String) service[0]), minute)); + } + } + writeOAL(CPM, DownSampling.Minute, services); + writeOAL(ENDPOINT_CPM, DownSampling.Minute, + Collections.singletonList(callRate(ENDPOINT_CPM, endpointId(SERVICE, "/order"), 2, "serviceId", + serviceId(SERVICE)))); + } + + private void writePools() throws Exception { + final List sizes = new ArrayList<>(); + for (final long[] pool : POOLS) { + final DataTable value = new DataTable(); + value.put("{pool=active}", pool[2]); + if (pool[3] >= 0) { + value.put("{pool=idle}", pool[3]); + } + sizes.add(labeledMeter("pool-" + pool[0], pool[1], value)); + } + support.writeMetrics(model(POOL_SIZE, DownSampling.Minute), new AvgLabeledFunction.AvgLabeledStorageBuilder(), + sizes); + + final List waits = new ArrayList<>(); + for (long minute = -1; minute <= 0; minute++) { + for (int pool = 0; pool < WAITING_POOLS; pool++) { + final DataTable value = new DataTable(); + final long waiters = pool * 10L + minute + 2; + value.put("{pool=waiting}", waiters); + waits.add(labeledMeter("pool-" + pool, minute, value)); + waiting.put(poolRowId(POOL_WAIT, pool, minute), waiters); + } + } + support.writeMetrics(model(POOL_WAIT, DownSampling.Minute), new AvgLabeledFunction.AvgLabeledStorageBuilder(), + waits); + } + + private void writeStatements() throws Exception { + final List statements = new ArrayList<>(); + for (int i = 0; i < STATEMENTS.length; i++) { + statements.add(statement(DATABASE, i, STATEMENTS[i][0], STATEMENTS[i][1])); + } + for (int i = 0; i < MILLIS_OUTSIDE.length; i++) { + statements.add(statementAt(DATABASE, 10 + i, MIDNIGHT + MILLIS_OUTSIDE[i][0], MILLIS_OUTSIDE[i][1])); + } + statements.add(statementAt(MIDNIGHT_DATABASE, 40, MIDNIGHT, 30)); + statements.add(statementAt(MIDNIGHT_DATABASE, 41, MIDNIGHT + 1, 40)); + statements.add(statement("inventory-db:3306", 20, 0, 9000)); + statements.add(statement("inventory-db:3306", 21, 10, 1)); + support.writeRecords(model(TopNDatabaseStatement.INDEX_NAME, DownSampling.Second), + new TopNDatabaseStatement.Builder(), statements); + support.writeRecords(model(TopNCacheReadCommand.INDEX_NAME, DownSampling.Second), + new TopNCacheReadCommand.Builder(), + Arrays.asList(cacheCommand(DATABASE, 30, 5, 8000), cacheCommand(DATABASE, 31, 6, 2))); + } + + private void writeOAL(final String metric, final DownSampling downSampling, + final List metrics) throws Exception { + support.writeMetrics(model(metric, downSampling), + OALTestMetrics.builder(OALTestMetrics.metricsClass(metric)), metrics); + } + + private List values(final String metric, final Entity entity, final Duration duration) throws Exception { + return support.metricsQueryDAO().readMetricsValues(condition(metric, entity), valueColumn(metric), duration) + .getValues().getValues().stream() + .map(JDBCMetricsQueryITBase::value) + .collect(Collectors.toList()); + } + + /** checkout's percentiles over minutes -1 to 1, as label to values. */ + private Map> labeled(final List labels) throws Exception { + final Map> byLabel = new LinkedHashMap<>(); + for (final MetricsValues values : support.metricsQueryDAO().readLabeledMetricsValues( + condition(PERCENTILE, service(SERVICE)), valueColumn(PERCENTILE), labels, minutes(-1, 1))) { + byLabel.put(values.getLabel(), values.getValues().getValues().stream() + .map(JDBCMetricsQueryITBase::value) + .collect(Collectors.toList())); + } + return byLabel; + } + + /** The labeled values of every entity over minutes -1 and 0, as label to row id to value. */ + private Map> everyEntity(final String metric, + final List labels) throws Exception { + final Map> byLabel = new TreeMap<>(); + for (final MetricsValues values : support.metricsQueryDAO().readLabeledMetricsValuesWithoutEntity( + metric, valueColumn(metric), labels, minutes(-1, 0))) { + final Map rows = new TreeMap<>(); + for (final KVInt kv : values.getValues().getValues()) { + rows.put(kv.getId(), value(kv)); + } + byLabel.put(values.getLabel(), rows); + } + return byLabel; + } + + /** The services listed, by name. */ + private List entities(final Duration duration, final int limit) throws Exception { + return support.metricsQueryDAO().listEntityIdsInRange(CPM, valueColumn(CPM), null, duration, limit) + .stream() + .map(id -> IDManager.ServiceID.analysisId(id).getName()) + .collect(Collectors.toList()); + } + + private List rank(final TopNCondition condition, final Duration duration) throws Exception { + return rank(condition, duration, null); + } + + /** The ranked entities as {@code name=value}, an endpoint by its own name. */ + private List rank(final TopNCondition condition, final Duration duration, + final List additionalConditions) throws Exception { final List ranked = support.aggregationQueryDAO().sortMetrics( - condition, ValueColumnMetadata.INSTANCE.getValueCName(metric), duration, null); + condition, valueColumn(condition.getName()), duration, additionalConditions); return ranked.stream() - .map(record -> IDManager.ServiceID.analysisId(record.getId()).getName() + "=" + record.getValue()) + .map(record -> (condition.getScope() == Scope.Endpoint + ? IDManager.EndpointID.analysisId(record.getId()).getEndpointName() + : IDManager.ServiceID.analysisId(record.getId()).getName()) + "=" + record.getValue()) .collect(Collectors.toList()); } - private List latencies(final String database, final int topN, final Order order) throws Exception { + /** The records as {@code name|refId|id|value}. */ + private List records(final String database, final int topN, final Order order) throws Exception { + return records(database, topN, order, minutes(-2, 1)); + } + + private List records(final String database, final int topN, final Order order, + final Duration duration) throws Exception { final Entity entity = new Entity(); entity.setScope(Scope.Service); entity.setServiceName(database); @@ -244,15 +759,50 @@ private List latencies(final String database, final int topN, final Orde condition.setTopN(topN); condition.setOrder(order); final List records = new JDBCRecordsQueryDAO(support.getClient(), support.getTableHelper()) - .readRecords(condition, TopN.LATENCY, minutes(-2, 1)); - return records.stream().map(Record::getValue).collect(Collectors.toList()); + .readRecords(condition, TopN.LATENCY, duration); + return records.stream() + .map(record -> record.getName() + "|" + record.getRefId() + "|" + record.getId() + "|" + + record.getValue()) + .collect(Collectors.toList()); + } + + private static String statementRecord(final int i, final long latency) { + return "select * from orders where id = " + i + "|trace-" + i + "|trace-" + i + "|" + latency; + } + + private static TopNCondition top(final String metric, final int topN, final Order order) { + final TopNCondition condition = new TopNCondition(); + condition.setName(metric); + condition.setScope(Scope.Service); + condition.setTopN(topN); + condition.setOrder(order); + return condition; + } + + private static String value(final KVInt kv) { + return kv.isEmptyValue() ? "empty" : String.valueOf(kv.getValue()); } - private static MetricsCondition serviceCondition(final String metric, final String service) { + private static String valueColumn(final String metric) { + return ValueColumnMetadata.INSTANCE.getValueCName(metric); + } + + private static Entity service(final String name) { final Entity entity = new Entity(); entity.setScope(Scope.Service); - entity.setServiceName(service); + entity.setServiceName(name); entity.setNormal(true); + return entity; + } + + private static Entity instance(final String name) { + final Entity entity = service(SERVICE); + entity.setScope(Scope.ServiceInstance); + entity.setServiceInstanceName(name); + return entity; + } + + private static MetricsCondition condition(final String metric, final Entity entity) { final MetricsCondition condition = new MetricsCondition(); condition.setName(metric); condition.setEntity(entity); @@ -263,24 +813,116 @@ private static Duration minutes(final long first, final long last) { return JDBCQueryITSupport.duration(MIDNIGHT + first * 60_000L, MIDNIGHT + last * 60_000L, Step.MINUTE); } + private static Duration hours(final long first, final long last) { + return JDBCQueryITSupport.duration(MIDNIGHT + first * 3_600_000L, MIDNIGHT + last * 3_600_000L, Step.HOUR); + } + private static long minuteBucket(final long minute) { return TimeBucket.getMinuteTimeBucket(MIDNIGHT + minute * 60_000L); } - private static TopNDatabaseStatement statement(final String database, final String segmentId, + private static long hourBucket(final long hour) { + return TimeBucket.getTimeBucket(MIDNIGHT + hour * 3_600_000L, DownSampling.Hour); + } + + private static String serviceId(final String name) { + return IDManager.ServiceID.buildId(name, true); + } + + private static String endpointId(final String service, final String endpoint) { + return IDManager.EndpointID.buildId(serviceId(service), endpoint); + } + + /** The id of a pool's row: the metric, the minute and the entity, as OAP stores a MAL metric. */ + private static String poolRowId(final String metric, final long pool, final long minute) { + return metric + "_" + minuteBucket(minute) + "_" + serviceId("pool-" + pool); + } + + private static Metrics oal(final String metric, final String entityId, final long timeBucket, + final Object... properties) throws Exception { + final Object[] all = Arrays.copyOf(properties, properties.length + 4); + all[properties.length] = "entityId"; + all[properties.length + 1] = entityId; + all[properties.length + 2] = "timeBucket"; + all[properties.length + 3] = timeBucket; + return OALTestMetrics.newMetrics(OALTestMetrics.metricsClass(metric), all); + } + + private static Metrics responseTime(final String metric, final String entityId, final long timeBucket, + final long value, final Object... properties) throws Exception { + // Two calls around the value, so the stored value is the average the function computes. + final LongAvgMetrics metrics = (LongAvgMetrics) oal(metric, entityId, timeBucket, properties); + metrics.combine(value - 1, 1); + metrics.combine(value + 1, 1); + metrics.calculate(); + return metrics; + } + + private static Metrics instanceResponseTime(final String instance, final long timeBucket, + final long value) throws Exception { + return responseTime(INSTANCE_RESP_TIME, IDManager.ServiceInstanceID.buildId(serviceId(SERVICE), instance), + timeBucket, value, "serviceId", serviceId(SERVICE)); + } + + private static Metrics callRate(final String metric, final String entityId, final long minute, + final Object... properties) throws Exception { + final CPMMetrics metrics = (CPMMetrics) oal(metric, entityId, minuteBucket(minute), properties); + metrics.combine(1); + metrics.calculate(); + return metrics; + } + + /** A service's percentiles of 100 calls, of {@code scale} x 10 to {@code scale} x 1000 ms. */ + private static Metrics percentile(final String service, final long minute, final int scale) throws Exception { + final PercentileMetrics2 metrics = + (PercentileMetrics2) oal(PERCENTILE, serviceId(service), minuteBucket(minute)); + for (int latency = 10; latency <= 1000; latency += 10) { + metrics.combine(latency * scale, 10); + } + metrics.calculate(); + return metrics; + } + + private static Metrics labeledMeter(final String service, final long minute, final DataTable value) { + final LabeledMeter meter = new LabeledMeter(); + meter.setTimeBucket(minuteBucket(minute)); + meter.accept(MeterEntity.newService(service, Layer.GENERAL), value); + meter.calculate(); + return meter; + } + + private static TopNDatabaseStatement statement(final String database, final int i, final long second, final long latency) { - final long timestamp = MIDNIGHT + second * 1000L; + return statementAt(database, i, MIDNIGHT + second * 1000L, latency); + } + + private static TopNDatabaseStatement statementAt(final String database, final int i, + final long timestamp, final long latency) { final TopNDatabaseStatement statement = new TopNDatabaseStatement(); - statement.setId(segmentId + "-1"); + statement.setId("segment-" + i + "-1"); statement.setEntityId(IDManager.ServiceID.buildId(database, false)); - statement.setStatement("select * from orders where customer_id = ?"); + statement.setStatement("select * from orders where id = " + i); statement.setLatency(latency); - statement.setTraceId("trace-" + segmentId); + statement.setTraceId("trace-" + i); statement.setTimestamp(timestamp); statement.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); return statement; } + private static TopNCacheReadCommand cacheCommand(final String database, final int i, + final long second, final long latency) { + final long timestamp = MIDNIGHT + second * 1000L; + final TopNCacheReadCommand command = new TopNCacheReadCommand(); + command.setId("segment-" + i + "-1"); + command.setEntityId(IDManager.ServiceID.buildId(database, false)); + command.setCommand("GET order:" + i); + command.setLatency(latency); + command.setTraceId("trace-" + i); + command.setTimestamp(timestamp); + command.setTimeBucket(TimeBucket.getRecordTimeBucket(timestamp)); + return command; + } + /** A MAL histogram meter: the class MAL generates adds nothing to its function but {@code createNew}. */ static class HistogramMeter extends AvgHistogramFunction { @Override @@ -296,4 +938,12 @@ public AcceptableValue createNew() { return new AvgMeter(); } } + + /** A MAL labeled meter of the {@code avgLabeled} function. */ + static class LabeledMeter extends AvgLabeledFunction { + @Override + public AcceptableValue createNew() { + return new LabeledMeter(); + } + } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryDAOTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryDAOTest.java index 3f8d7a179823..46bce684dd83 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryDAOTest.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryDAOTest.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Iterator; import java.util.List; import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; import org.apache.skywalking.oap.server.core.otlp.OTLPSpanRecord; @@ -101,8 +100,10 @@ void queryTraces_shouldJoinTheTagTableOncePerTagAndGroupByTrace() throws Excepti assertThat(sql).contains(OTLPSpanRecord.STATUS_CODE + " = ?"); assertThat(sql).contains(OTLPSpanRecord.DURATION + " >= ?"); assertThat(sql).doesNotContain(OTLPSpanRecord.DURATION + " <= ?"); + assertThat(sql).startsWith("select " + SPAN_TABLE + "." + OTLPSpanRecord.TRACE_ID + ", min(" + + OTLPSpanRecord.START_TIME + ") as rank_value from " + SPAN_TABLE + " "); assertThat(sql).endsWith("group by " + SPAN_TABLE + "." + OTLPSpanRecord.TRACE_ID - + " order by min(" + OTLPSpanRecord.START_TIME + ") desc limit 20"); + + " order by rank_value desc, " + OTLPSpanRecord.TRACE_ID + " limit 20"); assertThat(capturedParams.get(0)).containsExactly( OTLPSpanRecord.INDEX_NAME, "checkout", 2, 2, 1_000_000L, "span.http.request.method=GET", "resource.url.path=/cart", "span.url.path=/cart"); @@ -117,9 +118,11 @@ void queryTraces_shouldOrderByTheLongestSpanWhenAskedForDuration() throws Except dao.queryTraces(condition); final String sql = capturedSql.get(0); - assertThat(sql).startsWith("select " + SPAN_TABLE + "." + OTLPSpanRecord.TRACE_ID + ", max(" + OTLPSpanRecord.DURATION + ")"); + assertThat(sql).startsWith("select " + SPAN_TABLE + "." + OTLPSpanRecord.TRACE_ID + ", max(" + + OTLPSpanRecord.DURATION + ") as rank_value from " + SPAN_TABLE + " "); assertThat(sql).doesNotContain("inner join"); - assertThat(sql).endsWith("order by max(" + OTLPSpanRecord.DURATION + ") desc limit 5"); + assertThat(sql).endsWith(" group by " + SPAN_TABLE + "." + OTLPSpanRecord.TRACE_ID + + " order by rank_value desc, " + OTLPSpanRecord.TRACE_ID + " limit 5"); assertThat(capturedParams.get(0)).containsExactly(OTLPSpanRecord.INDEX_NAME); } @@ -140,7 +143,8 @@ void queryTraceById_shouldReadOnlyTheTablesAndSpansOfTheTimeRange() throws Excep duration.setStart("2026-09-17 1000"); duration.setEnd("2026-09-17 1010"); duration.setStep(Step.MINUTE); - when(tableHelper.getTablesForRead(OTLPSpanRecord.INDEX_NAME, duration.getStartTimeBucket(), duration.getEndTimeBucket())) + when(tableHelper.getTablesBetweenTimestamps( + OTLPSpanRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp())) .thenReturn(Collections.singletonList(SPAN_TABLE)); dao.queryTraceById("0af7651916cd43dd8448eb211c80319c", duration); @@ -155,32 +159,50 @@ void queryTraceById_shouldReadOnlyTheTablesAndSpansOfTheTimeRange() throws Excep } @Test - void queryTraces_shouldRankTheCandidatesOfEveryTableTogetherBeforeTheLimit() throws Exception { + void queryTraces_shouldRankTheTracesOfEveryTableInOneQuery() throws Exception { final String secondTable = OTLPSpanRecord.INDEX_NAME + "_20260918"; + final String secondTagTable = OTLPSpanRecord.ADDITIONAL_TAG_TABLE + "_20260918"; when(tableHelper.getTablesWithinTTL(OTLPSpanRecord.INDEX_NAME)).thenReturn(Arrays.asList(SPAN_TABLE, secondTable)); - // day one ranks a=10, b=20; day two ranks c=30, and b again with 25 because the trace crossed midnight - final ResultSet dayOne = rows(new String[] {"a", "b"}, new long[] {10L, 20L}); - final ResultSet dayTwo = rows(new String[] {"c", "b"}, new long[] {30L, 25L}); - final ResultSet none = mock(ResultSet.class); - when(none.next()).thenReturn(false); - final Iterator answers = Arrays.asList(dayOne, dayTwo, none, none).iterator(); + final ResultSet ranked = rows(new String[] {"c", "b"}, new long[] {30L, 20L}); doAnswer(invocation -> { - capturedSql.add(invocation.getArgument(0)); + final String sql = invocation.getArgument(0); + capturedSql.add(sql); final Object[] allArgs = invocation.getArguments(); capturedParams.add(Arrays.copyOfRange(allArgs, 2, allArgs.length)); - return invocation.>getArgument(1).handle(answers.next()); + return invocation.>getArgument(1) + .handle(sql.startsWith("select * from") ? none() : ranked); }).when(jdbcClient).executeQuery(anyString(), any(), any(Object[].class)); final OTLPTraceQueryCondition condition = new OTLPTraceQueryCondition(); + condition.setServiceName("checkout"); + condition.setTags(Collections.singletonList(new Tag("span.http.request.method", "GET"))); condition.setLimit(2); dao.queryTraces(condition); - // two ranking queries, then one fetch per table, each asking for the two best traces only: c (30), b (20) - assertThat(capturedSql).hasSize(4); - assertThat(capturedSql.get(0)).contains(" as rank_value from " + SPAN_TABLE).endsWith("limit 2"); - assertThat(capturedSql.get(2)).contains(OTLPSpanRecord.TRACE_ID + " in (?,?)"); + // one ranking query over both tables, each joining its own tag table, then one fetch per table of c and b + assertThat(capturedSql).hasSize(3); + assertThat(capturedSql.get(0)) + .contains(" from (select " + SPAN_TABLE + "." + OTLPSpanRecord.TRACE_ID + ", min(" + + OTLPSpanRecord.START_TIME + ") as rank_value from " + SPAN_TABLE + " inner join " + + TAG_TABLE + " " + TAG_TABLE + "0 on ") + .contains(" group by " + SPAN_TABLE + "." + OTLPSpanRecord.TRACE_ID + " union all select " + + secondTable + "." + OTLPSpanRecord.TRACE_ID + ", min(" + OTLPSpanRecord.START_TIME + + ") as rank_value from " + secondTable + " inner join " + secondTagTable + " " + + secondTagTable + "0 on ") + .endsWith(" group by " + secondTable + "." + OTLPSpanRecord.TRACE_ID + ") ranked group by " + + OTLPSpanRecord.TRACE_ID + " order by min(rank_value) desc, " + OTLPSpanRecord.TRACE_ID + + " limit 2"); + assertThat(capturedParams.get(0)).containsExactly( + OTLPSpanRecord.INDEX_NAME, "checkout", "span.http.request.method=GET", + OTLPSpanRecord.INDEX_NAME, "checkout", "span.http.request.method=GET"); + assertThat(capturedParams.get(1)).containsExactly(OTLPSpanRecord.INDEX_NAME, "c", "b"); assertThat(capturedParams.get(2)).containsExactly(OTLPSpanRecord.INDEX_NAME, "c", "b"); - assertThat(capturedParams.get(3)).containsExactly(OTLPSpanRecord.INDEX_NAME, "c", "b"); + } + + private static ResultSet none() throws Exception { + final ResultSet resultSet = mock(ResultSet.class); + when(resultSet.next()).thenReturn(false); + return resultSet; } private static ResultSet rows(final String[] traceIds, final long[] ranks) throws Exception { diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryITBase.java new file mode 100644 index 000000000000..0b58c286a93b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryITBase.java @@ -0,0 +1,616 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.Layer; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; +import org.apache.skywalking.oap.server.core.otlp.OTLPServiceRelationTraffic; +import org.apache.skywalking.oap.server.core.otlp.OTLPServiceSpanTraffic; +import org.apache.skywalking.oap.server.core.otlp.OTLPServiceTraffic; +import org.apache.skywalking.oap.server.core.otlp.OTLPSpanRecord; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.input.OTLPTraceQueryCondition; +import org.apache.skywalking.oap.server.core.query.type.QueryOrder; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.query.proto.Source; +import org.apache.skywalking.oap.server.core.storage.query.proto.SpanWrapper; +import org.joda.time.DateTime; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.testcontainers.containers.JdbcDatabaseContainer; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Reads natively stored OTLP spans the way the TraceQL API does on the JDBC storages, through + * {@link JDBCOTLPTraceQueryDAO}: the trace search, the read of one trace by id, and the catalogs of services, span + * names and peer services. A subclass names the server. + * + *

The search range is a minute either side of midnight, so it reads two day tables, and an attribute condition + * joins the spans' tag table of each. Each search condition is tried alone, in both orders, and with all the others. + * For every condition a near-miss trace matches all the others but that one, the two matching traces lie on the bounds + * of the range (a span is kept by its start millisecond, the range's end counted) and of the span duration, and a + * search asks for exactly as many traces as should match, so a near miss that leaks in pushes a match out. A search + * ranks a trace by its earliest matching span, or by its longest, and returns it whole: the first matching trace has a + * span of another service after midnight, the last one a root span of another service before the range. + * + *

The DAO serves {@code data_binary} untouched, so each span stores a label there instead of a + * {@code ResourceSpans}. The catalogs keep each name once, in the minute it was first seen, on either side of + * midnight, so in two day tables; a service name that arrived with two layers is two rows. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCOTLPTraceQueryITBase { + /** + * A minute either side of midnight, read by the second: its end timestamp, which the search counts, is the first + * millisecond after its last second. + */ + private static final Duration AROUND_MIDNIGHT = duration(MIDNIGHT - 60_000L, MIDNIGHT + 59_000L); + private static final long START = AROUND_MIDNIGHT.getStartTimestamp(); + private static final long END = AROUND_MIDNIGHT.getEndTimestamp(); + + /** The start of yesterday, the end instant of a range ending in the last minute of the day before. */ + private static final long YESTERDAY_START = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + private static final String NIGHTLY = "nightly"; + /** A span of the nightly service on yesterday's first instant, in yesterday's table. */ + private static final StoredSpan ON_THE_END_INSTANT = new StoredSpan( + "4bf92f3577b34da6a3ce929d0e0e0d01", "on the end instant", YESTERDAY_START, 1_000_000L) + .at(NIGHTLY, "nightly-1", "io.opentelemetry.quartz-2.0"); + /** A millisecond after it. */ + private static final StoredSpan AFTER_THE_END_INSTANT = new StoredSpan( + "4bf92f3577b34da6a3ce929d0e0e0d02", "after the end instant", YESTERDAY_START + 1, 1_000_000L) + .at(NIGHTLY, "nightly-1", "io.opentelemetry.quartz-2.0"); + + private static final String CHECKOUT = "checkout"; + private static final String CANARY = "checkout-canary"; + private static final String INVENTORY = "inventory"; + private static final String FRONTEND = "frontend"; + private static final String CHECKOUT_1 = "checkout-1"; + private static final String OKHTTP = "io.opentelemetry.okhttp-3.0"; + private static final String POST_STOCK = "POST /stock"; + private static final String PEER = INVENTORY; + private static final int INTERNAL = 1; + private static final int SERVER = 2; + private static final int CLIENT = 3; + private static final int STATUS_UNSET = 0; + private static final int STATUS_OK = 1; + private static final int STATUS_ERROR = 2; + private static final long MIN_DURATION_NS = 1_000_000L; + private static final long MAX_DURATION_NS = 9_000_000L; + private static final String ENVIRONMENT = "deployment.environment"; + private static final String TENANT = "tenant"; + + /** Found by its root span on the range's start; its child, after midnight, is the inventory's. */ + private static final String FIRST_MATCH = "4bf92f3577b34da6a3ce929d0e0e0a01"; + /** Found by a span on the range's end, whose root, before the range, is the frontend's; one digit off the first. */ + private static final String LAST_MATCH = "4bf92f3577b34da6a3ce929d0e0e0a02"; + private static final String OTHER_SERVICE = "4bf92f3577b34da6a3ce929d0e0e0b01"; + private static final String OTHER_INSTANCE = "4bf92f3577b34da6a3ce929d0e0e0b02"; + private static final String OTHER_SCOPE = "4bf92f3577b34da6a3ce929d0e0e0b03"; + private static final String OTHER_SPAN_NAME = "4bf92f3577b34da6a3ce929d0e0e0b04"; + private static final String OTHER_PEER = "4bf92f3577b34da6a3ce929d0e0e0b05"; + private static final String OTHER_KIND = "4bf92f3577b34da6a3ce929d0e0e0b06"; + private static final String OTHER_STATUS = "4bf92f3577b34da6a3ce929d0e0e0b07"; + private static final String BELOW_MIN_DURATION = "4bf92f3577b34da6a3ce929d0e0e0b08"; + private static final String ABOVE_MAX_DURATION = "4bf92f3577b34da6a3ce929d0e0e0b09"; + private static final String OTHER_RESOURCE_ATTRIBUTE = "4bf92f3577b34da6a3ce929d0e0e0b0a"; + private static final String OTHER_SPAN_ATTRIBUTE = "4bf92f3577b34da6a3ce929d0e0e0b0b"; + /** Has the tenant as a resource attribute, not as a span attribute. */ + private static final String TENANT_OF_RESOURCE = "4bf92f3577b34da6a3ce929d0e0e0b0c"; + private static final String BEFORE_START = "4bf92f3577b34da6a3ce929d0e0e0b0d"; + private static final String AFTER_END = "4bf92f3577b34da6a3ce929d0e0e0b0e"; + private static final String UNKNOWN = "4bf92f3577b34da6a3ce929d0e0e0fff"; + + /** The span each trace is found by. */ + private static final Map MATCHED_BY = new LinkedHashMap<>(); + /** Every trace's spans. */ + private static final Map> TRACES = fixture(); + /** The traces in the range, latest first. */ + private static final List IN_RANGE = Arrays.asList( + LAST_MATCH, TENANT_OF_RESOURCE, OTHER_SPAN_ATTRIBUTE, OTHER_RESOURCE_ATTRIBUTE, ABOVE_MAX_DURATION, + BELOW_MIN_DURATION, OTHER_STATUS, OTHER_KIND, OTHER_PEER, OTHER_SPAN_NAME, OTHER_SCOPE, OTHER_INSTANCE, + OTHER_SERVICE, FIRST_MATCH); + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCOTLPTraceQueryDAO dao; + private long yesterday; + private long today; + + JDBCOTLPTraceQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void writeSpans() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + final Model spans = support.install( + OTLPSpanRecord.class, DefaultScopeDefine.OTLP_SPAN, OTLPSpanRecord.INDEX_NAME, DownSampling.Second); + final Model services = support.install( + OTLPServiceTraffic.class, DefaultScopeDefine.OTLP_SERVICE, OTLPServiceTraffic.INDEX_NAME, + DownSampling.Minute); + final Model spanNames = support.install( + OTLPServiceSpanTraffic.class, DefaultScopeDefine.OTLP_SERVICE_SPAN, OTLPServiceSpanTraffic.INDEX_NAME, + DownSampling.Minute); + final Model peers = support.install( + OTLPServiceRelationTraffic.class, DefaultScopeDefine.OTLP_SERVICE_RELATION, + OTLPServiceRelationTraffic.INDEX_NAME, DownSampling.Minute); + + // Neither in time order nor against it, and each trace's spans child first. + final List records = new ArrayList<>(); + for (final String traceId : Arrays.asList( + OTHER_STATUS, LAST_MATCH, BELOW_MIN_DURATION, OTHER_SPAN_ATTRIBUTE, AFTER_END, OTHER_SERVICE, FIRST_MATCH, + OTHER_PEER, TENANT_OF_RESOURCE, OTHER_SCOPE, BEFORE_START, ABOVE_MAX_DURATION, OTHER_INSTANCE, + OTHER_RESOURCE_ATTRIBUTE, OTHER_KIND, OTHER_SPAN_NAME)) { + final List trace = new ArrayList<>(TRACES.get(traceId)); + Collections.reverse(trace); + for (final StoredSpan span : trace) { + records.add(span.record()); + } + } + support.writeRecords(spans, new OTLPSpanRecord.Builder(), records); + support.writeRecords(spans, new OTLPSpanRecord.Builder(), Arrays.asList( + ON_THE_END_INSTANT.record(), AFTER_THE_END_INSTANT.record())); + + yesterday = TimeBucket.getMinuteTimeBucket(MIDNIGHT - 120_000L); + today = TimeBucket.getMinuteTimeBucket(MIDNIGHT + 60_000L); + support.writeMetrics(services, new OTLPServiceTraffic.Builder(), Arrays.asList( + service(CHECKOUT, Layer.GENERAL.name(), yesterday), + service(CANARY, Layer.GENERAL.name(), yesterday), + service(FRONTEND, Layer.GENERAL.name(), yesterday), + service(CHECKOUT, Layer.MESH.name(), today), + service(INVENTORY, Layer.UNDEFINED.name(), today))); + support.writeMetrics(spanNames, new OTLPServiceSpanTraffic.Builder(), Arrays.asList( + spanName(CHECKOUT, POST_STOCK, yesterday), + spanName(CHECKOUT, "GET /stock", yesterday), + spanName(CANARY, POST_STOCK, yesterday), + spanName(CHECKOUT, "POST /orders", today), + spanName(INVENTORY, "POST /stock/reserve", today))); + support.writeMetrics(peers, new OTLPServiceRelationTraffic.Builder(), Arrays.asList( + peer(CHECKOUT, PEER, yesterday), + peer(CANARY, PEER, yesterday), + peer(CHECKOUT, "payments", today))); + + dao = new JDBCOTLPTraceQueryDAO(support.getClient(), support.getTableHelper()); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void listsEveryServiceRowOfEveryLayer() { + assertEquals(sorted(Arrays.asList( + CANARY + " in GENERAL since " + yesterday, + CHECKOUT + " in GENERAL since " + yesterday, + CHECKOUT + " in MESH since " + today, + FRONTEND + " in GENERAL since " + yesterday, + INVENTORY + " in UNDEFINED since " + today)), services(null)); + } + + @Test + public void listsTheServicesOfOneLayer() { + assertEquals(sorted(Arrays.asList( + CANARY + " in GENERAL since " + yesterday, + CHECKOUT + " in GENERAL since " + yesterday, + FRONTEND + " in GENERAL since " + yesterday)), services(Layer.GENERAL.name())); + assertEquals(Collections.singletonList(CHECKOUT + " in MESH since " + today), services(Layer.MESH.name())); + assertEquals(Collections.emptyList(), services(Layer.K8S.name())); + } + + @Test + public void listsTheSpanNamesOfEachService() { + assertEquals(Arrays.asList("GET /stock", "POST /orders", POST_STOCK), sorted(dao.getSpanNames(CHECKOUT))); + assertEquals(Collections.singletonList(POST_STOCK), sorted(dao.getSpanNames(CANARY))); + assertEquals(Collections.emptyList(), dao.getSpanNames("unknown")); + } + + @Test + public void listsThePeerServicesEachServiceCalled() { + assertEquals(Arrays.asList(PEER, "payments"), sorted(dao.getPeerServiceNames(CHECKOUT))); + assertEquals(Collections.singletonList(PEER), sorted(dao.getPeerServiceNames(CANARY))); + assertEquals(Collections.emptyList(), dao.getPeerServiceNames(INVENTORY)); + } + + /** + * A range bounds the read to the spans that started in it, the range's end counted; no range reads them all. + */ + @Test + public void readsATraceById() { + assertEquals(trace(FIRST_MATCH), render(dao.queryTraceById(FIRST_MATCH, AROUND_MIDNIGHT))); + assertEquals(trace(FIRST_MATCH), render(dao.queryTraceById(FIRST_MATCH, null))); + assertEquals(trace(LAST_MATCH), render(dao.queryTraceById(LAST_MATCH, null))); + assertEquals(Collections.singletonList(MATCHED_BY.get(LAST_MATCH).render()), + render(dao.queryTraceById(LAST_MATCH, AROUND_MIDNIGHT))); + assertEquals(Collections.emptyList(), dao.queryTraceById(BEFORE_START, AROUND_MIDNIGHT)); + assertEquals(Collections.emptyList(), dao.queryTraceById(AFTER_END, AROUND_MIDNIGHT)); + assertEquals(Collections.emptyList(), dao.queryTraceById(UNKNOWN, null)); + } + + @ParameterizedTest(name = "{0} {1}") + @MethodSource("conditionsInBothOrders") + public void findsTheTracesMatchingOneCondition(final Condition condition, final QueryOrder order) { + final List> expected = expected(condition.matching(), order); + assertEquals(expected, search(condition.criterion, order, expected.size())); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(QueryOrder.class) + public void findsOnlyTheTracesMatchingEveryCondition(final QueryOrder order) { + final Consumer everyCondition = condition -> { + for (final Condition each : Condition.values()) { + if (each != Condition.TRACE_ID) { + each.criterion.accept(condition); + } + } + }; + assertEquals(expected(Arrays.asList(FIRST_MATCH, LAST_MATCH), order), search(everyCondition, order, 2)); + assertEquals(expected(Collections.singletonList(FIRST_MATCH), order), + search(everyCondition.andThen(Condition.TRACE_ID.criterion), order, 2)); + } + + @Test + public void findsNoTraceWhenNoSpanMatches() { + assertEquals(Collections.emptyList(), + search(condition -> condition.setServiceName("unknown"), QueryOrder.BY_START_TIME, 10)); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(QueryOrder.class) + public void keepsAsManyTracesAsTheLimit(final QueryOrder order) { + final List> all = expected(Condition.SERVICE.matching(), order); + for (final int limit : new int[] {1, 3, 5}) { + assertEquals(all.subList(0, limit), search(Condition.SERVICE.criterion, order, limit), "limit " + limit); + } + } + + /** + * The first matching trace's span after midnight ranks it among the latest traces of that day, but the trace + * ranks by its earliest span, before midnight, so it is the last of them all. + */ + @ParameterizedTest(name = "limit {0}") + @ValueSource(ints = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}) + public void ranksATraceOfTwoDaysByItsEarliestSpan(final int limit) { + assertEquals(expected(IN_RANGE, QueryOrder.BY_START_TIME).subList(0, limit), + search(condition -> { }, QueryOrder.BY_START_TIME, limit)); + } + + /** + * A range is read through its end instant, the minute after its last one, which for a range ending in a day's last + * minute is the next day's first instant, in the next day table. + */ + @Test + public void readsTheSpansOnTheEndInstantOfARangeEndingInADaysLastMinute() { + final Duration lastHour = JDBCQueryITSupport.duration( + YESTERDAY_START - 3_600_000L, YESTERDAY_START - 60_000L, Step.MINUTE); + final OTLPTraceQueryCondition condition = new OTLPTraceQueryCondition(); + condition.setServiceName(NIGHTLY); + condition.setQueryDuration(lastHour); + condition.setLimit(10); + final List> found = new ArrayList<>(); + for (final List trace : dao.queryTraces(condition)) { + found.add(render(trace)); + } + assertEquals(Collections.singletonList(Collections.singletonList(ON_THE_END_INSTANT.render())), found); + assertEquals(Collections.singletonList(ON_THE_END_INSTANT.render()), + render(dao.queryTraceById(ON_THE_END_INSTANT.traceId, lastHour))); + } + + /** What the TraceQL search filters by, and the traces in the range each keeps. */ + enum Condition { + TIME_RANGE_ONLY(condition -> { }), + SERVICE(condition -> condition.setServiceName(CHECKOUT), OTHER_SERVICE), + INSTANCE(condition -> condition.setServiceInstance(CHECKOUT_1), OTHER_INSTANCE), + SCOPE(condition -> condition.setScopeName(OKHTTP), OTHER_SCOPE), + SPAN_NAME(condition -> condition.setSpanName(POST_STOCK), OTHER_SPAN_NAME), + PEER_SERVICE(condition -> condition.setPeerService(PEER), OTHER_PEER), + KIND(condition -> condition.setKind(CLIENT), OTHER_KIND), + STATUS(condition -> condition.setStatusCode(STATUS_ERROR), OTHER_STATUS), + MIN_DURATION(condition -> condition.setMinDurationNanos(MIN_DURATION_NS), BELOW_MIN_DURATION), + MAX_DURATION(condition -> condition.setMaxDurationNanos(MAX_DURATION_NS), ABOVE_MAX_DURATION), + RESOURCE_ATTRIBUTE(condition -> condition.getTags().add( + new Tag(OTLPSpanRecord.RESOURCE_TAG_PREFIX + ENVIRONMENT, "production")), OTHER_RESOURCE_ATTRIBUTE), + SPAN_ATTRIBUTE(condition -> condition.getTags().add(new Tag(OTLPSpanRecord.SPAN_TAG_PREFIX + TENANT, "acme")), + OTHER_SPAN_ATTRIBUTE, TENANT_OF_RESOURCE), + ATTRIBUTE_OF_EITHER_SCOPE(condition -> condition.getTags().add(new Tag(TENANT, "acme")), OTHER_SPAN_ATTRIBUTE), + // The last matching trace's id is one digit off the first's. + TRACE_ID(condition -> condition.setTraceId(FIRST_MATCH), only(FIRST_MATCH)); + + private final Consumer criterion; + private final List excluded; + private final List matching; + + Condition(final Consumer criterion, final String... excluded) { + this.criterion = criterion; + this.excluded = Arrays.asList(excluded); + this.matching = null; + } + + Condition(final Consumer criterion, final List matching) { + this.criterion = criterion; + this.excluded = Collections.emptyList(); + this.matching = matching; + } + + private List matching() { + if (matching != null) { + return matching; + } + final List kept = new ArrayList<>(IN_RANGE); + kept.removeAll(excluded); + return kept; + } + + private static List only(final String... traceIds) { + return Arrays.asList(traceIds); + } + } + + static Stream conditionsInBothOrders() { + return Arrays.stream(Condition.values()).flatMap(condition -> Stream.of( + Arguments.of(condition, QueryOrder.BY_START_TIME), + Arguments.of(condition, QueryOrder.BY_DURATION))); + } + + private List> search(final Consumer criteria, final QueryOrder order, + final int limit) { + final OTLPTraceQueryCondition condition = new OTLPTraceQueryCondition(); + criteria.accept(condition); + condition.setQueryDuration(AROUND_MIDNIGHT); + condition.setQueryOrder(order); + condition.setLimit(limit); + final List> traces = new ArrayList<>(); + for (final List trace : dao.queryTraces(condition)) { + traces.add(render(trace)); + } + return traces; + } + + /** The traces whole, the latest or the longest first by the span each is found by. */ + private static List> expected(final List traceIds, final QueryOrder order) { + final Comparator latestFirst = Comparator.comparingLong(traceId -> -MATCHED_BY.get(traceId).start); + final Comparator longestFirst = + Comparator.comparingLong(traceId -> -MATCHED_BY.get(traceId).duration); + return traceIds.stream() + .sorted(order == QueryOrder.BY_DURATION ? longestFirst : latestFirst) + .map(JDBCOTLPTraceQueryITBase::trace) + .collect(Collectors.toList()); + } + + private static List trace(final String traceId) { + return TRACES.get(traceId).stream().map(StoredSpan::render).sorted().collect(Collectors.toList()); + } + + /** The trace's spans in label order: a trace's spans come in no promised order. */ + private static List render(final List trace) { + return trace.stream() + .map(span -> span.getSpan().toStringUtf8() + " of " + span.getService() + ", " + span.getSource()) + .sorted() + .collect(Collectors.toList()); + } + + private List services(final String layer) { + return dao.getServices(layer).stream() + .map(service -> service.getServiceName() + " in " + service.getLayer() + " since " + + service.getTimeBucket()) + .sorted() + .collect(Collectors.toList()); + } + + private static List sorted(final List names) { + final List sorted = new ArrayList<>(names); + Collections.sort(sorted); + return sorted; + } + + private static Map> fixture() { + final Map> traces = new LinkedHashMap<>(); + found(traces, FIRST_MATCH, new StoredSpan(FIRST_MATCH, "first match", START, MIN_DURATION_NS), + new StoredSpan(FIRST_MATCH, "first match, the inventory's", MIDNIGHT + 40_000L, 10_000L) + .childOf("first match").at(INVENTORY, "inventory-1", "io.opentelemetry.servlet-5.0") + .call("POST /stock/reserve", SERVER, STATUS_UNSET, "").attributes("production", "acme")); + found(traces, OTHER_SERVICE, new StoredSpan(OTHER_SERVICE, "other service", MIDNIGHT - 55_000L, 1_500_000L) + .at(CANARY, CHECKOUT_1, OKHTTP)); + found(traces, OTHER_INSTANCE, new StoredSpan(OTHER_INSTANCE, "other instance", MIDNIGHT - 50_000L, 2_000_000L) + .at(CHECKOUT, "checkout-2", OKHTTP)); + found(traces, OTHER_SCOPE, new StoredSpan(OTHER_SCOPE, "other scope", MIDNIGHT - 45_000L, 2_500_000L) + .at(CHECKOUT, CHECKOUT_1, "io.opentelemetry.java-http-client")); + found(traces, OTHER_SPAN_NAME, new StoredSpan(OTHER_SPAN_NAME, "other span name", MIDNIGHT - 40_000L, + 3_000_000L).call("GET /stock", CLIENT, STATUS_ERROR, PEER)); + found(traces, OTHER_PEER, new StoredSpan(OTHER_PEER, "other peer", MIDNIGHT - 35_000L, 3_500_000L) + .call(POST_STOCK, CLIENT, STATUS_ERROR, "payments")); + found(traces, OTHER_KIND, new StoredSpan(OTHER_KIND, "other kind", MIDNIGHT - 30_000L, 4_000_000L) + .call(POST_STOCK, INTERNAL, STATUS_ERROR, PEER)); + found(traces, OTHER_STATUS, new StoredSpan(OTHER_STATUS, "other status", MIDNIGHT - 25_000L, 4_500_000L) + .call(POST_STOCK, CLIENT, STATUS_OK, PEER)); + found(traces, BELOW_MIN_DURATION, new StoredSpan(BELOW_MIN_DURATION, "below the minimum duration", + MIDNIGHT - 20_000L, MIN_DURATION_NS - 1)); + found(traces, ABOVE_MAX_DURATION, new StoredSpan(ABOVE_MAX_DURATION, "above the maximum duration", + MIDNIGHT - 15_000L, MAX_DURATION_NS + 1)); + found(traces, OTHER_RESOURCE_ATTRIBUTE, new StoredSpan(OTHER_RESOURCE_ATTRIBUTE, "other environment", + MIDNIGHT + 10_000L, 5_000_000L) + .attributes("staging", "acme")); + found(traces, OTHER_SPAN_ATTRIBUTE, new StoredSpan(OTHER_SPAN_ATTRIBUTE, "other tenant", MIDNIGHT + 20_000L, + 5_500_000L).attributes("production", "globex")); + found(traces, TENANT_OF_RESOURCE, new StoredSpan(TENANT_OF_RESOURCE, "tenant of the resource", + MIDNIGHT + 30_000L, 6_000_000L).tenantOfResource()); + found(traces, LAST_MATCH, new StoredSpan(LAST_MATCH, "last match", END, MAX_DURATION_NS).childOf("last root"), + new StoredSpan(LAST_MATCH, "last root", START - 30_000L, 125_000_000_000L) + .at(FRONTEND, "frontend-1", "io.opentelemetry.servlet-5.0") + .call("GET /checkout", SERVER, STATUS_ERROR, "")); + found(traces, BEFORE_START, new StoredSpan(BEFORE_START, "before the start", START - 1, 7_000_000L)); + found(traces, AFTER_END, new StoredSpan(AFTER_END, "after the end", END + 1, 7_500_000L)); + return traces; + } + + /** + * @param spans the span the trace is found by first + */ + private static void found(final Map> traces, final String traceId, + final StoredSpan... spans) { + MATCHED_BY.put(traceId, spans[0]); + traces.put(traceId, Arrays.asList(spans)); + } + + /** A span matching every condition unless told otherwise. */ + private static final class StoredSpan { + private final String traceId; + private final String label; + private final long start; + private final long duration; + private String parentLabel; + private String service = CHECKOUT; + private String instance = CHECKOUT_1; + private String scope = OKHTTP; + private String name = POST_STOCK; + private int kind = CLIENT; + private int status = STATUS_ERROR; + private String peer = PEER; + private String environment = "production"; + private String tenant = "acme"; + private String tenantScope = OTLPSpanRecord.SPAN_TAG_PREFIX; + + private StoredSpan(final String traceId, final String label, final long start, final long duration) { + this.traceId = traceId; + this.label = label; + this.start = start; + this.duration = duration; + } + + private StoredSpan childOf(final String parentLabel) { + this.parentLabel = parentLabel; + return this; + } + + private StoredSpan at(final String service, final String instance, final String scope) { + this.service = service; + this.instance = instance; + this.scope = scope; + return this; + } + + private StoredSpan call(final String name, final int kind, final int status, final String peer) { + this.name = name; + this.kind = kind; + this.status = status; + this.peer = peer; + return this; + } + + private StoredSpan attributes(final String environment, final String tenant) { + this.environment = environment; + this.tenant = tenant; + return this; + } + + private StoredSpan tenantOfResource() { + this.tenantScope = OTLPSpanRecord.RESOURCE_TAG_PREFIX; + return this; + } + + /** The record the OTLP receiver stores for the span. */ + private OTLPSpanRecord record() { + final OTLPSpanRecord record = new OTLPSpanRecord(); + record.setTraceId(traceId); + record.setSpanId(spanId(label)); + record.setParentSpanId(parentLabel == null ? "" : spanId(parentLabel)); + record.setServiceName(service); + record.setServiceInstance(instance); + record.setScopeName(scope); + record.setName(name); + record.setKind(kind); + record.setStatusCode(status); + record.setPeerService(peer); + record.setStartTime(start); + record.setDuration(duration); + record.setTags(Arrays.asList( + OTLPSpanRecord.RESOURCE_TAG_PREFIX + "service.name=" + service, + OTLPSpanRecord.RESOURCE_TAG_PREFIX + ENVIRONMENT + "=" + environment, + tenantScope + TENANT + "=" + tenant)); + record.setDataBinary(label.getBytes(StandardCharsets.UTF_8)); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(start)); + return record; + } + + /** The span as the DAO serves it. */ + private String render() { + return label + " of " + service + ", " + Source.OTLP; + } + } + + /** A span id derived from the span's label, unique within the fixture. */ + private static String spanId(final String label) { + return String.format("%016x", label.hashCode() & 0xffffffffL); + } + + private static OTLPServiceTraffic service(final String name, final String layer, final long minute) { + final OTLPServiceTraffic traffic = new OTLPServiceTraffic(); + traffic.setServiceName(name); + traffic.setLayer(layer); + traffic.setTimeBucket(minute); + return traffic; + } + + private static OTLPServiceSpanTraffic spanName(final String service, final String name, final long minute) { + final OTLPServiceSpanTraffic traffic = new OTLPServiceSpanTraffic(); + traffic.setServiceName(service); + traffic.setSpanName(name); + traffic.setTimeBucket(minute); + return traffic; + } + + private static OTLPServiceRelationTraffic peer(final String service, final String peer, final long minute) { + final OTLPServiceRelationTraffic traffic = new OTLPServiceRelationTraffic(); + traffic.setServiceName(service); + traffic.setPeerService(peer); + traffic.setTimeBucket(minute); + return traffic; + } + + private static Duration duration(final long start, final long end) { + return JDBCQueryITSupport.duration(start, end, Step.SECOND); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryMariaDBIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryMariaDBIT.java new file mode 100644 index 000000000000..a997d936c6a0 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryMariaDBIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCOTLPTraceQueryMariaDBIT extends JDBCOTLPTraceQueryITBase { + @Container + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); + + public JDBCOTLPTraceQueryMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryPostgreSQLIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryPostgreSQLIT.java new file mode 100644 index 000000000000..1f9cf5dfd4ef --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCOTLPTraceQueryPostgreSQLIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCOTLPTraceQueryPostgreSQLIT extends JDBCOTLPTraceQueryITBase { + @Container + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); + + public JDBCOTLPTraceQueryPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCProfilingDataQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCProfilingDataQueryITBase.java index 36a330221799..1e2252590256 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCProfilingDataQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCProfilingDataQueryITBase.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -28,12 +29,15 @@ import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.JFRProfilingDataRecord; import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingDataRecord; import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTargetType; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofProfilingDataRecord; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileLanguageType; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileThreadSnapshotRecord; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.jfr.type.JFREventType; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -44,9 +48,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; /** - * Reads trace-profiling thread snapshots the way the profile analysis does, and eBPF profiling data the way the eBPF - * analysis does, on data dumped and uploaded on both sides of midnight, so in two day tables. A subclass names the - * server. + * Reads trace-profiling thread snapshots the way the profile analysis does, eBPF profiling data the way the eBPF + * analysis does, and async-profiler (JFR) and pprof data the way their flame-graph analyses do, on data dumped and + * uploaded on both sides of midnight, so in two day tables. A subclass names the server. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) abstract class JDBCProfilingDataQueryITBase { @@ -62,12 +66,21 @@ abstract class JDBCProfilingDataQueryITBase { private static final String GO_SEGMENT_YESTERDAY = "segment-go-1"; private static final String GO_SEGMENT_TODAY = "segment-go-2"; private static final String SCHEDULE = "schedule-checkout"; + private static final String CHECKOUT_ID = IDManager.ServiceID.buildId("checkout", true); + private static final String INSTANCE_ID = IDManager.ServiceInstanceID.buildId(CHECKOUT_ID, "checkout-1"); + private static final String OTHER_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(CHECKOUT_ID, "checkout-2"); + private static final String ASYNC_PROFILER_TASK = (MIDNIGHT - 120_000L) + Const.ID_CONNECTOR + CHECKOUT_ID; + private static final String OTHER_ASYNC_PROFILER_TASK = (MIDNIGHT - 1_200_000L) + Const.ID_CONNECTOR + CHECKOUT_ID; + private static final String PPROF_TASK = (MIDNIGHT - 180_000L) + Const.ID_CONNECTOR + CHECKOUT_ID; + private static final String OTHER_PPROF_TASK = (MIDNIGHT - 1_500_000L) + Const.ID_CONNECTOR + CHECKOUT_ID; private final JdbcDatabaseContainer server; private final JDBCQueryITSupport.Target target; private JDBCQueryITSupport support; private JDBCProfileThreadSnapshotQueryDAO snapshotDAO; private ProfileThreadSnapshotRecord goSnapshotToday; + private final List jfrData = new ArrayList<>(); + private final List pprofData = new ArrayList<>(); JDBCProfilingDataQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { this.server = server; @@ -89,6 +102,7 @@ public void writeData() throws Exception { written.add(goSnapshotToday); support.writeRecords(snapshots, new ProfileThreadSnapshotRecord.Builder(), written); snapshotDAO = new JDBCProfileThreadSnapshotQueryDAO(support.getClient(), support.getTableHelper()); + writeAsyncProfilerAndPprofData(); final Model ebpfData = support.install( EBPFProfilingDataRecord.class, DefaultScopeDefine.EBPF_PROFILING_DATA, EBPFProfilingDataRecord.INDEX_NAME, @@ -97,6 +111,34 @@ public void writeData() throws Exception { ebpfData("stack-1", MIDNIGHT - 3_000L), ebpfData("stack-2", MIDNIGHT + 3_000L))); } + /** + * Uploads a second or two before and after midnight, so a task's data is in two day tables. The async-profiler + * task also has uploads by its other instance and of another event, and another task has an upload of the same + * instance and event; the same holds for the pprof task, without events. + */ + private void writeAsyncProfilerAndPprofData() throws Exception { + final String cpu = JFREventType.EXECUTION_SAMPLE.name(); + for (final int second : new int[] {-2, -1, 1, 2}) { + jfrData.add(jfr(ASYNC_PROFILER_TASK, INSTANCE_ID, cpu, second)); + pprofData.add(pprof(PPROF_TASK, INSTANCE_ID, second)); + } + for (final int second : new int[] {-1, 1}) { + jfrData.add(jfr(ASYNC_PROFILER_TASK, OTHER_INSTANCE_ID, cpu, second)); + pprofData.add(pprof(PPROF_TASK, OTHER_INSTANCE_ID, second)); + } + jfrData.add(jfr(ASYNC_PROFILER_TASK, INSTANCE_ID, JFREventType.LOCK.name(), 1)); + jfrData.add(jfr(OTHER_ASYNC_PROFILER_TASK, INSTANCE_ID, cpu, 1)); + pprofData.add(pprof(OTHER_PPROF_TASK, INSTANCE_ID, 1)); + support.writeRecords( + support.install(JFRProfilingDataRecord.class, DefaultScopeDefine.JFR_PROFILING_DATA, + JFRProfilingDataRecord.INDEX_NAME, DownSampling.Second), + new JFRProfilingDataRecord.Builder(), jfrData); + support.writeRecords( + support.install(PprofProfilingDataRecord.class, DefaultScopeDefine.PPROF_PROFILING_DATA, + PprofProfilingDataRecord.INDEX_NAME, DownSampling.Second), + new PprofProfilingDataRecord.Builder(), pprofData); + } + @AfterAll public void close() { if (support != null) { @@ -161,6 +203,73 @@ public void readsNoEBPFProfilingDataForNoSchedule() throws Exception { assertEquals(Collections.emptyList(), dao.queryData(Collections.emptyList(), begin, end)); } + /** + * A window's bounds are dump times of the segment: one a millisecond inside them leaves those snapshots out. + */ + @Test + public void findsNoSequenceJustOutsideAWindow() throws Exception { + assertEquals(Arrays.asList(2, 3), sequenceRange(JAVA_SEGMENT, dumpTime(1) + 1, dumpTime(4) - 1)); + } + + /** + * The analysis reads a segment's snapshots in batches of {@code [minSequence, maxSequence)}, here one per day + * table. + */ + @Test + public void readsAdjacentSequenceBatchesWithoutOverlap() throws Exception { + assertEquals(Arrays.asList(describe(javaSnapshot(0)), describe(javaSnapshot(1)), describe(javaSnapshot(2))), + describe(snapshotDAO.queryRecords(JAVA_SEGMENT, 0, 3))); + assertEquals(Arrays.asList(describe(javaSnapshot(3)), describe(javaSnapshot(4)), describe(javaSnapshot(5))), + describe(snapshotDAO.queryRecords(JAVA_SEGMENT, 3, JAVA_SNAPSHOTS))); + assertEquals(Collections.emptyList(), + snapshotDAO.queryRecords(JAVA_SEGMENT, JAVA_SNAPSHOTS, JAVA_SNAPSHOTS + 3)); + } + + @Test + public void readsTheJFRDataOfAnInstanceAndEventFromEveryDayTable() throws Exception { + final JDBCJFRDataQueryDAO dao = new JDBCJFRDataQueryDAO(support.getClient(), support.getTableHelper()); + final String cpu = JFREventType.EXECUTION_SAMPLE.name(); + final List instance = Collections.singletonList(INSTANCE_ID); + assertEquals(describeJFR(ASYNC_PROFILER_TASK, instance, cpu), + describeJFR(dao.getByTaskIdAndInstancesAndEvent(ASYNC_PROFILER_TASK, instance, cpu))); + } + + /** + * No instance listed reads the uploads of every instance of the task. + */ + @Test + public void readsTheJFRDataOfTheListedInstances() throws Exception { + final JDBCJFRDataQueryDAO dao = new JDBCJFRDataQueryDAO(support.getClient(), support.getTableHelper()); + final String cpu = JFREventType.EXECUTION_SAMPLE.name(); + final List instances = Arrays.asList(INSTANCE_ID, OTHER_INSTANCE_ID); + assertEquals(describeJFR(ASYNC_PROFILER_TASK, instances, cpu), + describeJFR(dao.getByTaskIdAndInstancesAndEvent(ASYNC_PROFILER_TASK, instances, cpu))); + assertEquals(describeJFR(ASYNC_PROFILER_TASK, instances, cpu), describeJFR( + dao.getByTaskIdAndInstancesAndEvent(ASYNC_PROFILER_TASK, Collections.emptyList(), cpu))); + final String lock = JFREventType.LOCK.name(); + assertEquals(describeJFR(ASYNC_PROFILER_TASK, instances, lock), + describeJFR(dao.getByTaskIdAndInstancesAndEvent(ASYNC_PROFILER_TASK, instances, lock))); + assertEquals(Collections.emptyList(), dao.getByTaskIdAndInstancesAndEvent(ASYNC_PROFILER_TASK, instances, "")); + assertEquals(Collections.emptyList(), dao.getByTaskIdAndInstancesAndEvent("", instances, cpu)); + } + + /** + * No instance listed reads the uploads of every instance of the task. + */ + @Test + public void readsThePprofDataOfTheListedInstancesFromEveryDayTable() throws Exception { + final JDBCPprofDataQueryDAO dao = new JDBCPprofDataQueryDAO(support.getClient(), support.getTableHelper()); + final List instance = Collections.singletonList(INSTANCE_ID); + assertEquals(describePprof(PPROF_TASK, instance), + describePprof(dao.getByTaskIdAndInstances(PPROF_TASK, instance))); + final List instances = Arrays.asList(INSTANCE_ID, OTHER_INSTANCE_ID); + assertEquals(describePprof(PPROF_TASK, instances), + describePprof(dao.getByTaskIdAndInstances(PPROF_TASK, instances))); + assertEquals(describePprof(PPROF_TASK, instances), + describePprof(dao.getByTaskIdAndInstances(PPROF_TASK, Collections.emptyList()))); + assertEquals(Collections.emptyList(), dao.getByTaskIdAndInstances("", instances)); + } + private List sequenceRange(final String segmentId, final long start, final long end) throws Exception { return Arrays.asList(snapshotDAO.queryMinSequence(segmentId, start, end), snapshotDAO.queryMaxSequence(segmentId, start, end)); @@ -229,4 +338,70 @@ private static EBPFProfilingDataRecord ebpfData(final String stackIdList, final data.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime)); return data; } + + /** + * A task's data in no promised order, sorted by instance and upload time. + */ + private static List describeJFR(final List data) { + return data.stream() + .map(row -> String.join("|", row.getInstanceId(), String.valueOf(row.getUploadTime()), + row.getTaskId(), row.getEventType(), String.valueOf(row.getTimeBucket()), + new String(row.getDataBinary(), StandardCharsets.UTF_8))) + .sorted() + .collect(Collectors.toList()); + } + + private List describeJFR(final String taskId, final List instanceIds, final String eventType) { + return describeJFR(jfrData.stream() + .filter(row -> row.getTaskId().equals(taskId) + && instanceIds.contains(row.getInstanceId()) + && row.getEventType().equals(eventType)) + .collect(Collectors.toList())); + } + + private static List describePprof(final List data) { + return data.stream() + .map(row -> String.join("|", row.getInstanceId(), String.valueOf(row.getUploadTime()), + row.getTaskId(), String.valueOf(row.getTimeBucket()), + new String(row.getDataBinary(), StandardCharsets.UTF_8))) + .sorted() + .collect(Collectors.toList()); + } + + private List describePprof(final String taskId, final List instanceIds) { + return describePprof(pprofData.stream() + .filter(row -> row.getTaskId().equals(taskId) + && instanceIds.contains(row.getInstanceId())) + .collect(Collectors.toList())); + } + + /** + * An upload as {@code JFRProfilingDataDispatcher} writes it, the data being the frame tree of one event. + */ + private static JFRProfilingDataRecord jfr(final String taskId, final String instanceId, final String eventType, + final int second) { + final long uploadTime = MIDNIGHT + second * 1_000L; + final JFRProfilingDataRecord data = new JFRProfilingDataRecord(); + data.setTaskId(taskId); + data.setInstanceId(instanceId); + data.setEventType(eventType); + data.setUploadTime(uploadTime); + data.setDataBinary(("{\"frame\":\"" + eventType + " " + second + "\"}").getBytes(StandardCharsets.UTF_8)); + data.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime)); + return data; + } + + /** + * An upload as {@code PprofProfilingDataDispatcher} writes it. + */ + private static PprofProfilingDataRecord pprof(final String taskId, final String instanceId, final int second) { + final long uploadTime = MIDNIGHT + second * 1_000L; + final PprofProfilingDataRecord data = new PprofProfilingDataRecord(); + data.setTaskId(taskId); + data.setInstanceId(instanceId); + data.setUploadTime(uploadTime); + data.setDataBinary(("{\"frame\":\"pprof " + second + "\"}").getBytes(StandardCharsets.UTF_8)); + data.setTimeBucket(TimeBucket.getRecordTimeBucket(uploadTime)); + return data; + } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCProfilingTaskQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCProfilingTaskQueryITBase.java index 06811963fef6..852a42d35f77 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCProfilingTaskQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCProfilingTaskQueryITBase.java @@ -19,8 +19,10 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; import com.google.gson.Gson; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.ToLongFunction; @@ -29,22 +31,41 @@ import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.AsyncProfilerMutationService; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.AsyncProfilerQueryService; +import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskLogRecord; import org.apache.skywalking.oap.server.core.profiling.asyncprofiler.storage.AsyncProfilerTaskRecord; import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTargetType; import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTaskRecord; import org.apache.skywalking.oap.server.core.profiling.ebpf.storage.EBPFProfilingTriggerType; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofMutationService; +import org.apache.skywalking.oap.server.core.profiling.pprof.PprofQueryService; +import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskLogRecord; import org.apache.skywalking.oap.server.core.profiling.pprof.storage.PprofTaskRecord; +import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskLogRecord; import org.apache.skywalking.oap.server.core.profiling.trace.ProfileTaskRecord; import org.apache.skywalking.oap.server.core.query.enumeration.Step; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerEventType; import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTask; +import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTaskCreationType; +import org.apache.skywalking.oap.server.core.query.type.AsyncProfilerTaskLogOperationType; import org.apache.skywalking.oap.server.core.query.type.EBPFProfilingTaskContinuousProfiling; import org.apache.skywalking.oap.server.core.query.type.PprofEventType; import org.apache.skywalking.oap.server.core.query.type.PprofTask; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskCreationType; +import org.apache.skywalking.oap.server.core.query.type.PprofTaskLogOperationType; import org.apache.skywalking.oap.server.core.query.type.ProfileTask; +import org.apache.skywalking.oap.server.core.query.type.ProfileTaskLogOperationType; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.StorageModule; import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.profiling.asyncprofiler.IAsyncProfilerTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profiling.pprof.IPprofTaskQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.TableHelper; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -53,16 +74,33 @@ import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** - * Reads profiling tasks the way the profiling services and the agent task caches do. OAP writes a task into the - * table of the day it creates it, so the tasks created before midnight are in yesterday's table and the others in - * today's. A subclass names the server. + * Reads profiling tasks and their logs the way the profiling services, the mutation services and the agent task + * caches do: task lists, a task by id, and task logs. OAP writes a task into the table of the day it creates it, so + * the tasks created before midnight are in yesterday's table and the others in today's; a log goes into the table of + * the day it is reported, and the logs read lie on both sides of midnight. Every filter has a near miss: a task of + * another service or endpoint, a task created or starting just outside a time range, a log of another task. A + * subclass names the server. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) abstract class JDBCProfilingTaskQueryITBase { private static final String SERVICE_ID = IDManager.ServiceID.buildId("checkout", true); private static final String INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-1"); + private static final String OTHER_SERVICE_ID = IDManager.ServiceID.buildId("payment", true); + private static final String OTHER_INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-2"); + /** When the other service's async-profiler and pprof task was created, in minutes from midnight. */ + private static final int OTHER_SERVICE_CREATED_MINUTE = -15; + /** A service with two async-profiler and two pprof tasks created in the same minute, half a minute apart. */ + private static final String SAME_MINUTE_SERVICE_ID = IDManager.ServiceID.buildId("inventory", true); + private static final int SAME_MINUTE = 5; + /** A service whose tasks the creation tests write as running now. */ + private static final String RUNNING_SERVICE_ID = IDManager.ServiceID.buildId("shipping", true); + /** The async-profiler and pprof task whose logs are read, with a log on each side of midnight. */ + private static final int READ_TASK_MINUTE = -10; private static final int TRACE_PROFILING_MINUTES = 10; /** When each async-profiler and each pprof task was created, in minutes from midnight. */ private static final int[] CREATED_MINUTES = {-30, -20, -10, 10, 20}; @@ -79,6 +117,14 @@ abstract class JDBCProfilingTaskQueryITBase { private ProfileTaskRecord scheduledForTomorrow; private JDBCProfileTaskQueryDAO traceProfilingDAO; private Duration aroundMidnight; + /** The other service's task, started five minutes before midnight and finished after it. */ + private ProfileTaskRecord otherServiceTask; + private Model traceProfilingLogModel; + private Model asyncProfilerModel; + private Model pprofModel; + private final List traceProfilingLogs = new ArrayList<>(); + private final List asyncProfilerLogs = new ArrayList<>(); + private final List pprofLogs = new ArrayList<>(); JDBCProfilingTaskQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { this.server = server; @@ -112,6 +158,7 @@ public void writeTasks() throws Exception { asyncProfiler, new AsyncProfilerTaskRecord.Builder(), asyncProfilerTask(createTime), createTime); support.writeNoneStream(pprof, new PprofTaskRecord.Builder(), pprofTask(createTime), createTime); } + writeOtherServiceTasksAndLogs(traceProfiling, asyncProfiler, pprof); final Model ebpf = support.install( EBPFProfilingTaskRecord.class, DefaultScopeDefine.EBPF_PROFILING_TASK, EBPFProfilingTaskRecord.INDEX_NAME, @@ -126,6 +173,62 @@ public void writeTasks() throws Exception { MIDNIGHT - TimeUnit.HOURS.toMillis(1), MIDNIGHT + TimeUnit.HOURS.toMillis(1), Step.MINUTE); } + /** + * A task of another service, started five minutes before midnight, and the logs of the tasks: the other + * service's task is notified before midnight and finishes after it, the task started today is notified to two + * instances. The async-profiler and pprof task read has a log on each side of midnight, and the tasks before + * and after it have logs of the same instance. + */ + private void writeOtherServiceTasksAndLogs(final Model traceProfiling, final Model asyncProfiler, + final Model pprof) throws Exception { + asyncProfilerModel = asyncProfiler; + pprofModel = pprof; + otherServiceTask = traceProfilingTask( + OTHER_SERVICE_ID, "/checkout", MIDNIGHT - TimeUnit.MINUTES.toMillis(5), + MIDNIGHT - TimeUnit.MINUTES.toMillis(5)); + support.writeNoneStream( + traceProfiling, new ProfileTaskRecord.Builder(), otherServiceTask, otherServiceTask.getCreateTime()); + final long otherCreateTime = createTime(OTHER_SERVICE_CREATED_MINUTE); + support.writeNoneStream(asyncProfiler, new AsyncProfilerTaskRecord.Builder(), + asyncProfilerTask(OTHER_SERVICE_ID, otherCreateTime), otherCreateTime); + support.writeNoneStream( + pprof, new PprofTaskRecord.Builder(), pprofTask(OTHER_SERVICE_ID, otherCreateTime), otherCreateTime); + for (final long createTime : sameMinuteCreateTimes()) { + support.writeNoneStream(asyncProfiler, new AsyncProfilerTaskRecord.Builder(), + asyncProfilerTask(SAME_MINUTE_SERVICE_ID, createTime), createTime); + support.writeNoneStream( + pprof, new PprofTaskRecord.Builder(), pprofTask(SAME_MINUTE_SERVICE_ID, createTime), createTime); + } + + traceProfilingLogs.add(traceProfilingLog(otherServiceTask, INSTANCE_ID, ProfileTaskLogOperationType.NOTIFIED, + createTime(-4))); + traceProfilingLogs.add(traceProfilingLog(otherServiceTask, INSTANCE_ID, + ProfileTaskLogOperationType.EXECUTION_FINISHED, createTime(6))); + traceProfilingLogs.add(traceProfilingLog(startedToday, INSTANCE_ID, ProfileTaskLogOperationType.NOTIFIED, + MIDNIGHT + 70_000L)); + traceProfilingLogs.add(traceProfilingLog(startedToday, OTHER_INSTANCE_ID, + ProfileTaskLogOperationType.NOTIFIED, MIDNIGHT + 80_000L)); + traceProfilingLogModel = support.install( + ProfileTaskLogRecord.class, DefaultScopeDefine.PROFILE_TASK_LOG, ProfileTaskLogRecord.INDEX_NAME, + DownSampling.Second); + support.writeRecords(traceProfilingLogModel, new ProfileTaskLogRecord.Builder(), traceProfilingLogs); + + // Each log: the minute its task was created, the minute it is reported, its operation type. + for (final int[] log : new int[][] { + {READ_TASK_MINUTE, -9, 1}, {READ_TASK_MINUTE, 1, 2}, {-20, -19, 1}, {-20, -14, 4}, {10, 11, 1}}) { + asyncProfilerLogs.add(asyncProfilerLog(taskId(log[0]), createTime(log[1]), log[2])); + pprofLogs.add(pprofLog(taskId(log[0]), createTime(log[1]), log[2])); + } + support.writeRecords( + support.install(AsyncProfilerTaskLogRecord.class, DefaultScopeDefine.ASYNC_PROFILER_TASK_LOG, + AsyncProfilerTaskLogRecord.INDEX_NAME, DownSampling.Second), + new AsyncProfilerTaskLogRecord.Builder(), asyncProfilerLogs); + support.writeRecords( + support.install(PprofTaskLogRecord.class, DefaultScopeDefine.PPROF_TASK_LOG, PprofTaskLogRecord.INDEX_NAME, + DownSampling.Second), + new PprofTaskLogRecord.Builder(), pprofLogs); + } + @AfterAll public void close() { if (support != null) { @@ -220,6 +323,209 @@ public void readsNoContinuousProfilingTaskForAPolicyWithoutTargets() throws Exce EBPFProfilingTriggerType.CONTINUOUS_PROFILING, triggeredSince, 0)); } + @Test + public void listsTheTraceProfilingTasksOfAnEndpoint() { + assertEquals(ids(scheduledForTomorrow, scheduledForToday), + ids(traceProfilingDAO.getTaskList(SERVICE_ID, "/checkout", null, null, null))); + } + + /** + * The agent task cache reads the tasks of every service starting within a window of minute buckets, both ends + * included. + */ + @Test + public void readsTheTraceProfilingTasksOfEveryServiceStartingWithinAWindow() { + final long first = otherServiceTask.getStartTime(); + final long last = startedToday.getStartTime(); + assertEquals(describe(startedToday, otherServiceTask), describe(traceProfilingDAO.getTaskList( + null, null, TimeBucket.getMinuteTimeBucket(first), TimeBucket.getMinuteTimeBucket(last), null))); + assertEquals(Collections.emptyList(), traceProfilingDAO.getTaskList( + null, null, TimeBucket.getMinuteTimeBucket(first + 60_000L), TimeBucket.getMinuteTimeBucket(last - 60_000L), + null)); + } + + /** + * The analysis reads a task by id: one created yesterday is in yesterday's table, though it starts today. + */ + @Test + public void readsATraceProfilingTaskById() { + assertEquals(describe(scheduledForToday), + describe(Collections.singletonList(traceProfilingDAO.getById(scheduledForToday.getTaskId())))); + assertEquals(describe(scheduledForTomorrow), describe( + Collections.singletonList(traceProfilingDAO.getById(scheduledForTomorrow.getTaskId())))); + assertNull(traceProfilingDAO.getById(MIDNIGHT + Const.ID_CONNECTOR + SERVICE_ID)); + assertNull(traceProfilingDAO.getById("")); + } + + /** + * The trace-profiling service reads the logs of every task and matches them to the tasks it lists. + */ + @Test + public void readsTheLogsOfEveryTraceProfilingTaskNewestFirst() throws Exception { + final JDBCProfileTaskLogQueryDAO dao = + new JDBCProfileTaskLogQueryDAO(support.getClient(), support.getTableHelper()); + final List expected = traceProfilingLogs.stream() + .sorted(Comparator.comparingLong(ProfileTaskLogRecord::getOperationTime).reversed()) + .map(log -> String.join( + "|", TableHelper.generateId(traceProfilingLogModel, log.id().build()), log.getTaskId(), + log.getInstanceId(), ProfileTaskLogOperationType.parse(log.getOperationType()).name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList()); + assertEquals(expected, dao.getTaskLogList().stream() + .map(log -> String.join("|", log.getId(), log.getTaskId(), log.getInstanceId(), + log.getOperationType().name(), String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList())); + } + + /** + * The task list of a duration bounds the tasks' second buckets, both ends included; the agent task cache reads + * the tasks of every service; creating a task checks the newest task of the service created up to now. + */ + @Test + public void listsTheAsyncProfilerTasksCreatedWithinADuration() throws Exception { + final JDBCAsyncProfilerTaskQueryDAO dao = + new JDBCAsyncProfilerTaskQueryDAO(support.getClient(), support.getTableHelper()); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, 10, -10, -20), describeAsyncProfilerTasks( + dao.getTaskList(SERVICE_ID, secondBucket(-20, 0), secondBucket(10, 0), null))); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, -10), describeAsyncProfilerTasks( + dao.getTaskList(SERVICE_ID, secondBucket(-20, 1), secondBucket(10, -1), null))); + final List cached = new ArrayList<>(describeAsyncProfilerTasks(SERVICE_ID, -10)); + cached.addAll(describeAsyncProfilerTasks(OTHER_SERVICE_ID, OTHER_SERVICE_CREATED_MINUTE)); + cached.addAll(describeAsyncProfilerTasks(SERVICE_ID, -20)); + assertEquals(cached, describeAsyncProfilerTasks( + dao.getTaskList(null, secondBucket(-20, 0), secondBucket(-10, 0), null))); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, -10), describeAsyncProfilerTasks( + dao.getTaskList(SERVICE_ID, null, secondBucket(10, -1), 1))); + } + + @Test + public void readsAnAsyncProfilerTaskById() throws Exception { + final JDBCAsyncProfilerTaskQueryDAO dao = + new JDBCAsyncProfilerTaskQueryDAO(support.getClient(), support.getTableHelper()); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, -30), + describeAsyncProfilerTasks(Collections.singletonList(dao.getById(taskId(-30))))); + assertEquals(describeAsyncProfilerTasks(SERVICE_ID, 20), + describeAsyncProfilerTasks(Collections.singletonList(dao.getById(taskId(20))))); + assertNull(dao.getById(taskId(0))); + } + + /** + * A task log's id is the id of its task. + */ + @Test + public void readsTheLogsOfOneAsyncProfilerTaskNewestFirst() throws Exception { + final JDBCAsyncProfilerTaskLogQueryDAO dao = + new JDBCAsyncProfilerTaskLogQueryDAO(support.getClient(), support.getTableHelper()); + final String task = taskId(READ_TASK_MINUTE); + final List expected = asyncProfilerLogs.stream() + .filter(log -> log.getTaskId().equals(task)) + .sorted(Comparator.comparingLong(AsyncProfilerTaskLogRecord::getOperationTime).reversed()) + .map(log -> String.join("|", log.getTaskId(), log.getInstanceId(), + AsyncProfilerTaskLogOperationType.parse(log.getOperationType()).name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList()); + assertEquals(2, expected.size()); + assertEquals(expected, dao.getTaskLogList(task).stream() + .map(log -> String.join("|", log.getId(), log.getInstanceId(), log.getOperationType().name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList())); + assertEquals(Collections.emptyList(), dao.getTaskLogList(taskId(0))); + } + + @Test + public void listsThePprofTasksCreatedWithinADuration() throws Exception { + final JDBCPprofTaskQueryDAO dao = new JDBCPprofTaskQueryDAO(support.getClient(), support.getTableHelper()); + assertEquals(describePprofTasks(SERVICE_ID, 10, -10, -20), describePprofTasks( + dao.getTaskList(SERVICE_ID, secondBucket(-20, 0), secondBucket(10, 0), null))); + assertEquals(describePprofTasks(SERVICE_ID, -10), describePprofTasks( + dao.getTaskList(SERVICE_ID, secondBucket(-20, 1), secondBucket(10, -1), null))); + final List cached = new ArrayList<>(describePprofTasks(SERVICE_ID, -10)); + cached.addAll(describePprofTasks(OTHER_SERVICE_ID, OTHER_SERVICE_CREATED_MINUTE)); + cached.addAll(describePprofTasks(SERVICE_ID, -20)); + assertEquals(cached, describePprofTasks( + dao.getTaskList(null, secondBucket(-20, 0), secondBucket(-10, 0), null))); + assertEquals(describePprofTasks(SERVICE_ID, -10, -20, -30), describePprofTasks( + dao.getTaskList(SERVICE_ID, null, secondBucket(10, -1), null))); + } + + /** + * The GraphQL task lists accept a duration of seconds, and the query services bound the tasks by its seconds: of + * two tasks created in the same minute, a duration ending between them, with a limit of one, lists the first. + */ + @Test + public void listsTheTasksOfADurationOfSecondsExactly() throws Exception { + final ModuleManager moduleManager = storageServices(); + final AsyncProfilerQueryService async = new AsyncProfilerQueryService(moduleManager); + final PprofQueryService pprof = new PprofQueryService(moduleManager); + final long[] created = sameMinuteCreateTimes(); + final long minuteStart = createTime(SAME_MINUTE); + final List first = Collections.singletonList(created[0] + Const.ID_CONNECTOR + SAME_MINUTE_SERVICE_ID); + final List both = Arrays.asList(created[1] + Const.ID_CONNECTOR + SAME_MINUTE_SERVICE_ID, + created[0] + Const.ID_CONNECTOR + SAME_MINUTE_SERVICE_ID); + final Duration endingBetween = JDBCQueryITSupport.duration(minuteStart, minuteStart + 30_000L, Step.SECOND); + assertEquals(first, asyncProfilerIds(async.queryTask(SAME_MINUTE_SERVICE_ID, endingBetween, 1))); + assertEquals(first, pprofIds(pprof.queryTask(SAME_MINUTE_SERVICE_ID, endingBetween, 1))); + final Duration onBoth = JDBCQueryITSupport.duration(created[0], created[1], Step.SECOND); + assertEquals(both, asyncProfilerIds(async.queryTask(SAME_MINUTE_SERVICE_ID, onBoth, null))); + assertEquals(both, pprofIds(pprof.queryTask(SAME_MINUTE_SERVICE_ID, onBoth, null))); + final Duration between = JDBCQueryITSupport.duration(created[0] + 1_000L, created[1] - 1_000L, Step.SECOND); + assertEquals(Collections.emptyList(), async.queryTask(SAME_MINUTE_SERVICE_ID, between, null)); + assertEquals(Collections.emptyList(), pprof.queryTask(SAME_MINUTE_SERVICE_ID, between, null)); + } + + /** + * A task created ten seconds ago with the second bucket the mutation services write, as earlier versions wrote it + * too, runs: creating another task of the service is refused, as the check finds the running task by that bucket. + */ + @Test + public void refusesASecondTaskOfAServiceWhileTheFirstRuns() throws Exception { + final long createTime = System.currentTimeMillis() - 10_000L; + support.writeNoneStream(asyncProfilerModel, new AsyncProfilerTaskRecord.Builder(), + asyncProfilerTask(RUNNING_SERVICE_ID, createTime), createTime); + support.writeNoneStream( + pprofModel, new PprofTaskRecord.Builder(), pprofTask(RUNNING_SERVICE_ID, createTime), createTime); + final ModuleManager moduleManager = storageServices(); + assertEquals(AsyncProfilerTaskCreationType.ALREADY_PROFILING_ERROR, + new AsyncProfilerMutationService(moduleManager).createTask( + RUNNING_SERVICE_ID, Collections.singletonList(INSTANCE_ID), 60, + Collections.singletonList(AsyncProfilerEventType.CPU), "").getCode()); + assertEquals(PprofTaskCreationType.ALREADY_PROFILING_ERROR, + new PprofMutationService(moduleManager).createTask( + RUNNING_SERVICE_ID, Collections.singletonList(INSTANCE_ID), 5, PprofEventType.CPU, 10 + ).getCode()); + } + + @Test + public void readsAPprofTaskById() throws Exception { + final JDBCPprofTaskQueryDAO dao = new JDBCPprofTaskQueryDAO(support.getClient(), support.getTableHelper()); + assertEquals(describePprofTasks(SERVICE_ID, -30), + describePprofTasks(Collections.singletonList(dao.getById(taskId(-30))))); + assertEquals(describePprofTasks(SERVICE_ID, 20), + describePprofTasks(Collections.singletonList(dao.getById(taskId(20))))); + assertNull(dao.getById(taskId(0))); + } + + @Test + public void readsTheLogsOfOnePprofTaskNewestFirst() throws Exception { + final JDBCPprofTaskLogQueryDAO dao = + new JDBCPprofTaskLogQueryDAO(support.getClient(), support.getTableHelper()); + final String task = taskId(READ_TASK_MINUTE); + final List expected = pprofLogs.stream() + .filter(log -> log.getTaskId().equals(task)) + .sorted(Comparator.comparingLong(PprofTaskLogRecord::getOperationTime).reversed()) + .map(log -> String.join("|", log.getTaskId(), log.getInstanceId(), + PprofTaskLogOperationType.parse(log.getOperationType()).name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList()); + assertEquals(2, expected.size()); + assertEquals(expected, dao.getTaskLogList(task).stream() + .map(log -> String.join("|", log.getId(), log.getInstanceId(), log.getOperationType().name(), + String.valueOf(log.getOperationTime()))) + .collect(Collectors.toList())); + assertEquals(Collections.emptyList(), dao.getTaskLogList(taskId(0))); + assertEquals(Collections.emptyList(), dao.getTaskLogList("")); + } + private List readAroundStart(final ProfileTaskRecord task) { final long start = task.getStartTime(); return traceProfilingDAO.getTaskList( @@ -244,9 +550,14 @@ private static List ids(final List tasks) { private static ProfileTaskRecord traceProfilingTask(final String endpointName, final long createTime, final long startTime) { + return traceProfilingTask(SERVICE_ID, endpointName, createTime, startTime); + } + + private static ProfileTaskRecord traceProfilingTask(final String serviceId, final String endpointName, + final long createTime, final long startTime) { final ProfileTaskRecord task = new ProfileTaskRecord(); - task.setTaskId(createTime + Const.ID_CONNECTOR + SERVICE_ID); - task.setServiceId(SERVICE_ID); + task.setTaskId(createTime + Const.ID_CONNECTOR + serviceId); + task.setServiceId(serviceId); task.setEndpointName(endpointName); task.setStartTime(startTime); task.setDuration(TRACE_PROFILING_MINUTES); @@ -259,21 +570,30 @@ private static ProfileTaskRecord traceProfilingTask(final String endpointName, f } private static AsyncProfilerTaskRecord asyncProfilerTask(final long createTime) { + return asyncProfilerTask(SERVICE_ID, createTime); + } + + private static AsyncProfilerTaskRecord asyncProfilerTask(final String serviceId, final long createTime) { final AsyncProfilerTaskRecord task = new AsyncProfilerTaskRecord(); - task.setTaskId(createTime + Const.ID_CONNECTOR + SERVICE_ID); - task.setServiceId(SERVICE_ID); + task.setTaskId(createTime + Const.ID_CONNECTOR + serviceId); + task.setServiceId(serviceId); task.setServiceInstanceIdsFromList(Collections.singletonList(INSTANCE_ID)); task.setDuration((int) TimeUnit.MINUTES.toSeconds(5)); task.setEventsFromList(Collections.singletonList(AsyncProfilerEventType.CPU.toString())); + task.setExecArgs("interval=10ms"); task.setCreateTime(createTime); task.setTimeBucket(TimeBucket.getRecordTimeBucket(createTime)); return task; } private static PprofTaskRecord pprofTask(final long createTime) { + return pprofTask(SERVICE_ID, createTime); + } + + private static PprofTaskRecord pprofTask(final String serviceId, final long createTime) { final PprofTaskRecord task = new PprofTaskRecord(); - task.setTaskId(createTime + Const.ID_CONNECTOR + SERVICE_ID); - task.setServiceId(SERVICE_ID); + task.setTaskId(createTime + Const.ID_CONNECTOR + serviceId); + task.setServiceId(serviceId); task.setServiceInstanceIdsFromList(Collections.singletonList(INSTANCE_ID)); task.setDuration(5); task.setEvents(PprofEventType.CPU.toString()); @@ -301,9 +621,157 @@ private static EBPFProfilingTaskRecord continuousProfilingTask(final EBPFProfili task.setTargetType(targetType.value()); task.setCreateTime(time); task.setLastUpdateTime(time); - task.setTimeBucket(TimeBucket.getRecordTimeBucket(time)); + task.setTimeBucket(TimeBucket.getMinuteTimeBucket(time)); task.setContinuousProfilingJson(GSON.toJson(continuousProfiling)); task.generateLogicalId(); return task; } + + private static long createTime(final int minute) { + return MIDNIGHT + TimeUnit.MINUTES.toMillis(minute); + } + + private static long[] sameMinuteCreateTimes() { + return new long[] {createTime(SAME_MINUTE) + 10_000L, createTime(SAME_MINUTE) + 40_000L}; + } + + private static List asyncProfilerIds(final List tasks) { + return tasks.stream().map(AsyncProfilerTask::getId).collect(Collectors.toList()); + } + + private static List pprofIds(final List tasks) { + return tasks.stream().map(PprofTask::getId).collect(Collectors.toList()); + } + + /** + * Hands the profiling services this storage's task DAOs, as the storage provider registers them. + */ + private ModuleManager storageServices() { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(moduleManager.find(StorageModule.NAME)).thenReturn(provider); + when(provider.provider()).thenReturn(services); + when(services.getService(IAsyncProfilerTaskQueryDAO.class)).thenReturn( + new JDBCAsyncProfilerTaskQueryDAO(support.getClient(), support.getTableHelper())); + when(services.getService(IPprofTaskQueryDAO.class)).thenReturn( + new JDBCPprofTaskQueryDAO(support.getClient(), support.getTableHelper())); + return moduleManager; + } + + private static long secondBucket(final int minute, final int seconds) { + return TimeBucket.getRecordTimeBucket(createTime(minute) + TimeUnit.SECONDS.toMillis(seconds)); + } + + /** + * The id the mutation services give a task of the service created at the minute. + */ + private static String taskId(final int minute) { + return createTime(minute) + Const.ID_CONNECTOR + SERVICE_ID; + } + + private static List describe(final ProfileTaskRecord... tasks) { + return Arrays.stream(tasks) + .map(task -> String.join( + "|", task.getTaskId(), task.getServiceId(), task.getEndpointName(), + String.valueOf(task.getStartTime()), String.valueOf(task.getCreateTime()), + String.valueOf(task.getDuration()), String.valueOf(task.getMinDurationThreshold()), + String.valueOf(task.getDumpPeriod()), String.valueOf(task.getMaxSamplingCount()))) + .collect(Collectors.toList()); + } + + private static List describe(final List tasks) { + return tasks.stream() + .map(task -> String.join( + "|", task.getId(), task.getServiceId(), task.getEndpointName(), + String.valueOf(task.getStartTime()), String.valueOf(task.getCreateTime()), + String.valueOf(task.getDuration()), String.valueOf(task.getMinDurationThreshold()), + String.valueOf(task.getDumpPeriod()), String.valueOf(task.getMaxSamplingCount()))) + .collect(Collectors.toList()); + } + + private static List describeAsyncProfilerTasks(final String serviceId, final int... minutes) { + return describeAsyncProfilerTasks(Arrays.stream(minutes).mapToObj(minute -> { + final AsyncProfilerTaskRecord record = asyncProfilerTask(serviceId, createTime(minute)); + return AsyncProfilerTask.builder() + .id(record.getTaskId()) + .serviceId(record.getServiceId()) + .serviceInstanceIds(Collections.singletonList(INSTANCE_ID)) + .createTime(record.getCreateTime()) + .events(Collections.singletonList(AsyncProfilerEventType.CPU)) + .duration(record.getDuration()) + .execArgs(record.getExecArgs()) + .build(); + }).collect(Collectors.toList())); + } + + private static List describeAsyncProfilerTasks(final List tasks) { + return tasks.stream() + .map(task -> String.join( + "|", task.getId(), task.getServiceId(), String.valueOf(task.getServiceInstanceIds()), + String.valueOf(task.getCreateTime()), String.valueOf(task.getEvents()), + String.valueOf(task.getDuration()), task.getExecArgs())) + .collect(Collectors.toList()); + } + + private static List describePprofTasks(final String serviceId, final int... minutes) { + return describePprofTasks(Arrays.stream(minutes).mapToObj(minute -> { + final PprofTaskRecord record = pprofTask(serviceId, createTime(minute)); + return PprofTask.builder() + .id(record.getTaskId()) + .serviceId(record.getServiceId()) + .serviceInstanceIds(Collections.singletonList(INSTANCE_ID)) + .createTime(record.getCreateTime()) + .events(PprofEventType.CPU) + .duration(record.getDuration()) + .dumpPeriod(record.getDumpPeriod()) + .build(); + }).collect(Collectors.toList())); + } + + private static List describePprofTasks(final List tasks) { + return tasks.stream() + .map(task -> String.join( + "|", task.getId(), task.getServiceId(), String.valueOf(task.getServiceInstanceIds()), + String.valueOf(task.getCreateTime()), String.valueOf(task.getEvents()), + String.valueOf(task.getDuration()), String.valueOf(task.getDumpPeriod()))) + .collect(Collectors.toList()); + } + + private static ProfileTaskLogRecord traceProfilingLog(final ProfileTaskRecord task, final String instanceId, + final ProfileTaskLogOperationType operationType, + final long operationTime) { + final ProfileTaskLogRecord log = new ProfileTaskLogRecord(); + log.setTaskId(task.getTaskId()); + log.setInstanceId(instanceId); + log.setOperationType(operationType.getCode()); + log.setOperationTime(operationTime); + log.setTimestamp(operationTime); + log.setTimeBucket(TimeBucket.getRecordTimeBucket(operationTime)); + return log; + } + + private static AsyncProfilerTaskLogRecord asyncProfilerLog(final String taskId, final long operationTime, + final int operationType) { + final AsyncProfilerTaskLogRecord log = new AsyncProfilerTaskLogRecord(); + log.setTaskId(taskId); + log.setInstanceId(INSTANCE_ID); + log.setOperationType(operationType); + log.setOperationTime(operationTime); + log.setTimestamp(operationTime); + log.setTimeBucket(TimeBucket.getRecordTimeBucket(operationTime)); + return log; + } + + private static PprofTaskLogRecord pprofLog(final String taskId, final long operationTime, + final int operationType) { + final PprofTaskLogRecord log = new PprofTaskLogRecord(); + log.setTaskId(taskId); + log.setInstanceId(INSTANCE_ID); + log.setOperationType(operationType); + log.setOperationTime(operationTime); + log.setTimestamp(operationTime); + log.setTimeBucket(TimeBucket.getRecordTimeBucket(operationTime)); + return log; + } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCQueryITSupport.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCQueryITSupport.java index 9410bdecd7f1..6dd982147022 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCQueryITSupport.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCQueryITSupport.java @@ -209,6 +209,21 @@ void writeMetrics(final Model model, final StorageBuilder bui flush(requests); } + /** + * Writes metrics again the way the metrics worker does when a stored row changes: an update of the row, in the + * day table of the metric's time bucket, through OAP's metrics DAO and its batch executor. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + void updateMetrics(final Model model, final StorageBuilder builder, + final List metrics) throws Exception { + final JDBCMetricsDAO dao = new JDBCMetricsDAO(client, (StorageBuilder) (StorageBuilder) builder); + final List requests = new ArrayList<>(); + for (final Metrics metric : metrics) { + requests.add(dao.prepareBatchUpdate(model, metric, (SessionCacheCallback) null)); + } + flush(requests); + } + /** * Writes a task row as {@link JDBCNoneStreamDAO} does, at once, into the table of the day it is created on. * {@code JDBCNoneStreamDAO} names that day "now"; the fixtures name it by the row's creation time, so a run that diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCRecordsQueryDAOTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCRecordsQueryDAOTest.java index 1a6488917573..5aa99c869903 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCRecordsQueryDAOTest.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCRecordsQueryDAOTest.java @@ -67,15 +67,16 @@ void buildSQL_shouldContainTableColumnCondition() { } @Test - void buildSQL_shouldContainEntityIdAndTimeBucket() { + void buildSQL_shouldContainEntityIdAndTheTimestampsThroughTheEndInstant() { final RecordCondition condition = buildCondition("top_n_db", 10, Order.DES); final Duration duration = buildDuration(); final SQLAndParameters result = JDBCRecordsQueryDAO.buildSQL(condition, "value", duration, TABLE); assertThat(result.sql()).contains(TopN.ENTITY_ID + " = ?"); - assertThat(result.sql()).contains(TopN.TIME_BUCKET + " >= ?"); - assertThat(result.sql()).contains(TopN.TIME_BUCKET + " <= ?"); + assertThat(result.sql()).contains(TopN.TIMESTAMP + " >= ?"); + assertThat(result.sql()).contains(TopN.TIMESTAMP + " <= ?"); + assertThat(result.parameters()).contains(duration.getStartTimestamp(), duration.getEndTimestamp()); } @Test diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryITBase.java new file mode 100644 index 000000000000..ed8290abf9b7 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryITBase.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.skywalking.apm.network.language.agent.v3.SpanAttachedEvent; +import org.apache.skywalking.oap.query.zipkin.ZipkinQueryService; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.spanattach.SWSpanAttachedEventRecord; +import org.apache.skywalking.oap.server.core.analysis.manual.spanattach.SpanAttachedEventRecord; +import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; +import org.apache.skywalking.oap.server.core.query.TraceQueryService; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.query.ISpanAttachedEventQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IZipkinQueryDAO; +import org.apache.skywalking.oap.server.core.zipkin.ZipkinSpanRecord; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.JdbcDatabaseContainer; +import zipkin2.storage.QueryRequest; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.DURATION; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.MIDNIGHT; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.stored; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Reads span attached events the way the trace and the Zipkin queries do on the JDBC storages: the V1 branch reads + * the spans from the trace DAOs and the events from {@link JDBCSpanAttachedEventQueryDAO}, and the query services + * attach each event to the span it belongs to. A subclass names the server. + * + *

The events are rover's network profiling, see {@link NativeTraceFixture} and {@link ZipkinTraceFixture}. The + * native trace's events lie on both sides of midnight, so in two day tables; events of traces not asked for name the + * same spans, events stored with the other trace type name the same traces, and some events lie on and just outside + * the queries' duration, which a by-id read on JDBC does not bound, as it does not bound the read of the trace's spans. + * The callers sort the events by start time themselves; the DAO's order, start second then nanos, is what they keep + * for events starting at the same instant. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCSpanAttachedEventQueryITBase { + private static final SWSpanAttachedEventRecord.Builder SKYWALKING_EVENT = new SWSpanAttachedEventRecord.Builder(); + private static final SpanAttachedEventRecord.Builder ZIPKIN_EVENT = new SpanAttachedEventRecord.Builder(); + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCSpanAttachedEventQueryDAO eventDAO; + private JDBCTraceQueryDAO traceDAO; + private JDBCZipkinQueryDAO zipkinDAO; + private TraceQueryService traceQueryService; + private ZipkinQueryService zipkinQueryService; + private final List skyWalkingEvents = new ArrayList<>(); + private final List zipkinEvents = new ArrayList<>(); + + JDBCSpanAttachedEventQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void writeTraces() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + final Model segments = install(SegmentRecord.class); + final Model zipkinSpans = install(ZipkinSpanRecord.class); + final Model skyWalkingEventModel = install(SWSpanAttachedEventRecord.class); + final Model zipkinEventModel = install(SpanAttachedEventRecord.class); + + support.writeRecords(segments, new SegmentRecord.Builder(), NativeTraceFixture.SEGMENTS.stream() + .map(NativeTraceFixture::record).collect(Collectors.toList())); + support.writeRecords(zipkinSpans, new ZipkinSpanRecord.Builder(), ZipkinTraceFixture.SPANS.stream() + .map(ZipkinSpanRecords::of).collect(Collectors.toList())); + final List events = new ArrayList<>(NativeTraceFixture.EVENTS); + events.addAll(ZipkinTraceFixture.EVENTS); + for (final SpanAttachedEvent event : events) { + if (event.getTraceContext().getType() == SpanAttachedEvent.SpanReferenceType.SKYWALKING) { + skyWalkingEvents.add(AttachedEventFixture.skyWalkingRecord(event)); + } else { + zipkinEvents.add(AttachedEventFixture.zipkinRecord(event)); + } + } + support.writeRecords(skyWalkingEventModel, SKYWALKING_EVENT, skyWalkingEvents); + support.writeRecords(zipkinEventModel, ZIPKIN_EVENT, zipkinEvents); + + final JDBCClient client = support.getClient(); + eventDAO = new JDBCSpanAttachedEventQueryDAO(client, support.getTableHelper()); + traceDAO = new JDBCTraceQueryDAO(support.getModuleManager(), client, support.getTableHelper()); + zipkinDAO = new JDBCZipkinQueryDAO(client, support.getTableHelper()); + final ModuleManager moduleManager = moduleManager(); + traceQueryService = new TraceQueryService(moduleManager); + zipkinQueryService = new ZipkinQueryService(moduleManager); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsTheSkyWalkingEventsOfTheListedTracesInStartOrder() throws IOException { + assertEquals( + stored(SKYWALKING_EVENT, skyWalkingRecords(NativeTraceFixture.EVENTS_OF_BOTH_TRACES)), + stored(SKYWALKING_EVENT, eventDAO.querySWSpanAttachedEvents( + Arrays.asList(NativeTraceFixture.TRACE, NativeTraceFixture.ONE_SEGMENT_TRACE), DURATION))); + assertEquals( + stored(SKYWALKING_EVENT, skyWalkingRecords(NativeTraceFixture.EVENTS_OF_THE_TRACE)), + stored(SKYWALKING_EVENT, eventDAO.querySWSpanAttachedEvents( + Collections.singletonList(NativeTraceFixture.TRACE), null))); + } + + @Test + public void readsTheZipkinEventsOfTheListedTracesInStartOrder() throws IOException { + assertEquals( + stored(ZIPKIN_EVENT, zipkinRecords(ZipkinTraceFixture.EVENTS_OF_BOTH_TRACES)), + stored(ZIPKIN_EVENT, eventDAO.queryZKSpanAttachedEvents( + Arrays.asList(ZipkinTraceFixture.TRACE, ZipkinTraceFixture.ONE_SERVICE_TRACE), DURATION))); + } + + /** + * Two events with the same name and start instant, as each pair below, are kept apart only by their trace and + * span. Rover reports a Zipkin trace context without a segment id. + */ + @Test + public void keepsTheZipkinEventsOfTwoTracesOnTheSameSpanIdAtTheSameInstant() throws IOException { + assertEquals( + stored(ZIPKIN_EVENT, zipkinRecords(Arrays.asList( + ZipkinTraceFixture.SAME_SPAN_ID_REQUEST, ZipkinTraceFixture.OTHER_SAME_SPAN_ID_REQUEST))), + byTraceAndSpan(stored(ZIPKIN_EVENT, eventDAO.queryZKSpanAttachedEvents(Arrays.asList( + ZipkinTraceFixture.SAME_SPAN_ID_TRACE, ZipkinTraceFixture.OTHER_SAME_SPAN_ID_TRACE), null)))); + } + + @Test + public void keepsTheZipkinEventsOfTwoSpansOfATraceAtTheSameInstant() throws IOException { + assertEquals( + stored(ZIPKIN_EVENT, zipkinRecords(Arrays.asList( + ZipkinTraceFixture.FIRST_SPAN_REQUEST, ZipkinTraceFixture.SECOND_SPAN_REQUEST))), + byTraceAndSpan(stored(ZIPKIN_EVENT, eventDAO.queryZKSpanAttachedEvents( + Collections.singletonList(ZipkinTraceFixture.TWO_SPANS_TRACE), null)))); + } + + /** + * Two native events of different traces cannot share a segment: rover takes the trace id and the segment id + * from the same {@code sw8} header, written by the segment's own agent. + */ + @Test + public void keepsTheSkyWalkingEventsOfTwoSpansOfASegmentAtTheSameInstant() throws IOException { + assertEquals( + stored(SKYWALKING_EVENT, skyWalkingRecords(Arrays.asList( + NativeTraceFixture.FIRST_SPAN_SEND, NativeTraceFixture.SECOND_SPAN_SEND))), + byTraceAndSpan(stored(SKYWALKING_EVENT, eventDAO.querySWSpanAttachedEvents( + Collections.singletonList(NativeTraceFixture.TWO_SPANS_TRACE), null)))); + } + + @Test + public void readsNoEventsOfATraceWithoutEvents() throws IOException { + assertEquals(Collections.emptyList(), eventDAO.querySWSpanAttachedEvents( + Collections.singletonList(NativeTraceFixture.UNKNOWN_TRACE), null)); + assertEquals(Collections.emptyList(), eventDAO.queryZKSpanAttachedEvents( + Collections.singletonList(ZipkinTraceFixture.UNKNOWN_TRACE), null)); + } + + @Test + public void attachesEachEventToItsSpanOfANativeTrace() throws IOException { + assertEquals(NativeTraceFixture.expectedSpans(NativeTraceFixture.TRACE), + NativeTraceFixture.render(traceQueryService.queryTrace(NativeTraceFixture.TRACE, DURATION))); + assertEquals(NativeTraceFixture.expectedSpans(NativeTraceFixture.ONE_SEGMENT_TRACE), + NativeTraceFixture.render( + traceQueryService.queryTrace(NativeTraceFixture.ONE_SEGMENT_TRACE, null))); + } + + @Test + public void attachesEachEventToItsSpanOfAZipkinTraceReadById() throws IOException { + assertEquals(ZipkinTraceFixture.expectedTraces(ZipkinTraceFixture.TRACE), + ZipkinTraceFixture.canonical(Collections.singletonList( + zipkinQueryService.getTraceById(ZipkinTraceFixture.TRACE, DURATION)))); + assertEquals(ZipkinTraceFixture.expectedTraces(ZipkinTraceFixture.ONE_SERVICE_TRACE), + ZipkinTraceFixture.canonical(Collections.singletonList( + zipkinQueryService.getTraceById(ZipkinTraceFixture.ONE_SERVICE_TRACE, null)))); + } + + @Test + public void attachesEachEventToItsSpanOfTheZipkinTracesSearched() throws IOException { + final QueryRequest request = QueryRequest.newBuilder() + .endTs(MIDNIGHT + 59_000L) + .lookback(119_000L) + .limit(2) + .build(); + assertEquals(ZipkinTraceFixture.expectedTraces(ZipkinTraceFixture.TRACE, ZipkinTraceFixture.ONE_SERVICE_TRACE), + ZipkinTraceFixture.canonical(zipkinQueryService.getTraces(request, DURATION))); + } + + @Test + public void attachesEachEventToItsSpanOfTheZipkinTracesReadByIds() throws IOException { + final Set ids = new LinkedHashSet<>(Arrays.asList( + ZipkinTraceFixture.TRACE, ZipkinTraceFixture.ONE_SERVICE_TRACE, ZipkinTraceFixture.UNKNOWN_TRACE)); + assertEquals(ZipkinTraceFixture.expectedTraces(ZipkinTraceFixture.TRACE, ZipkinTraceFixture.ONE_SERVICE_TRACE), + ZipkinTraceFixture.canonical(zipkinQueryService.getTracesByIds(ids, null))); + } + + /** + * Installs a record's tables the way OAP does at boot, from the record class's own annotations. + */ + private Model install(final Class recordClass) throws Exception { + final Stream stream = recordClass.getAnnotation(Stream.class); + return support.install(recordClass, stream.scopeId(), stream.name(), DownSampling.Second); + } + + /** + * Hands the query services this plugin's DAOs, as the storage module would. + */ + private ModuleManager moduleManager() { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleServiceHolder storage = services(moduleManager, StorageModule.NAME); + when(storage.getService(ITraceQueryDAO.class)).thenReturn(traceDAO); + when(storage.getService(IZipkinQueryDAO.class)).thenReturn(zipkinDAO); + when(storage.getService(ISpanAttachedEventQueryDAO.class)).thenReturn(eventDAO); + final IComponentLibraryCatalogService components = mock(IComponentLibraryCatalogService.class); + when(components.getComponentName(anyInt())).thenAnswer( + invocation -> NativeTraceFixture.COMPONENTS.get(invocation.getArgument(0))); + final ModuleServiceHolder core = services(moduleManager, CoreModule.NAME); + when(core.getService(IComponentLibraryCatalogService.class)).thenReturn(components); + return moduleManager; + } + + private static ModuleServiceHolder services(final ModuleManager moduleManager, final String moduleName) { + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(moduleManager.find(moduleName)).thenReturn(provider); + when(provider.provider()).thenReturn(services); + return services; + } + + private static List skyWalkingRecords(final List events) { + return events.stream().map(AttachedEventFixture::skyWalkingRecord).collect(Collectors.toList()); + } + + private static List zipkinRecords(final List events) { + return events.stream().map(AttachedEventFixture::zipkinRecord).collect(Collectors.toList()); + } + + /** Events starting at the same instant come in no order. */ + private static List> byTraceAndSpan(final List> events) { + events.sort(Comparator.comparing((Map columns) -> + (String) columns.get(SpanAttachedEventRecord.RELATED_TRACE_ID)) + .thenComparing(columns -> (String) columns.get(SpanAttachedEventRecord.TRACE_SPAN_ID))); + return events; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryMariaDBIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryMariaDBIT.java new file mode 100644 index 000000000000..ac8023fcc8d5 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryMariaDBIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCSpanAttachedEventQueryMariaDBIT extends JDBCSpanAttachedEventQueryITBase { + @Container + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); + + public JDBCSpanAttachedEventQueryMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); + } +} diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch3ShardedIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryPostgreSQLIT.java similarity index 61% rename from oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch3ShardedIT.java rename to oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryPostgreSQLIT.java index 0d364005e7b9..971e6c4bcc1f 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/query/SpanAttachedEventQueryOpenSearch3ShardedIT.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCSpanAttachedEventQueryPostgreSQLIT.java @@ -16,22 +16,18 @@ * */ -package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.query; +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; -import org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ElasticSearchQueryITSupport; -import org.junit.jupiter.api.Tag; -import org.testcontainers.elasticsearch.ElasticsearchContainer; +import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -@Tag("slow") -@Tag("elastic-it") @Testcontainers -public class SpanAttachedEventQueryOpenSearch3ShardedIT extends SpanAttachedEventQueryITBase { +public class JDBCSpanAttachedEventQueryPostgreSQLIT extends JDBCSpanAttachedEventQueryITBase { @Container - private static final ElasticsearchContainer SERVER = ElasticSearchQueryITSupport.openSearch("3.0.0"); + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); - public SpanAttachedEventQueryOpenSearch3ShardedIT() { - super(SERVER, ElasticSearchQueryITSupport.Mode.LOGIC_SHARDING_AND_CUSTOM_ROUTING); + public JDBCSpanAttachedEventQueryPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryITBase.java new file mode 100644 index 000000000000..1390113def1f --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryITBase.java @@ -0,0 +1,776 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; +import org.apache.skywalking.oap.server.core.analysis.DownSampling; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessDetectType; +import org.apache.skywalking.oap.server.core.analysis.manual.process.ProcessTraffic; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.endpoint.EndpointRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.process.ProcessRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.process.ProcessRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.IntList; +import org.apache.skywalking.oap.server.core.cache.NetworkAddressAliasCache; +import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; +import org.apache.skywalking.oap.server.core.query.MetadataQueryService; +import org.apache.skywalking.oap.server.core.query.TopologyQueryService; +import org.apache.skywalking.oap.server.core.query.enumeration.ProfilingSupportStatus; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; +import org.apache.skywalking.oap.server.core.query.type.Call; +import org.apache.skywalking.oap.server.core.query.type.ProcessNode; +import org.apache.skywalking.oap.server.core.query.type.ProcessTopology; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.StorageModels; +import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.containers.JdbcDatabaseContainer; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Reads the relations a topology is built from, the way the topology query service does, through + * {@link JDBCTopologyQueryDAO}: the service relations of the whole range or of some services, the instance relations + * of a service pair, the relations of an endpoint and the process relations of an instance, each from the table of + * the side that detected it, and the service and process topologies the query service builds from them. A subclass + * names the server. + * + *

The range is the minute before midnight and the minute of midnight, so every read crosses two day tables, and + * relations lie on both its bounds and in the minutes just outside them. The server and the client side each detect + * relations the other does not, so a read of the wrong side returns other relations. A service or process relation + * may name another component in each minute, a service relation several in one minute, and each component is a call + * of its own. Every filter has a near miss: a relation of a service, an instance, an endpoint or a process not + * asked for, an instance relation from one service of the pair to a third one, an endpoint of the same name on + * another service, and relations rolled up per hour in the same tables, which a minute read must not return and an + * hour read must. A scheduler service and process relation each name another component in each of twelve + * minutes. + * + *

The topology builders key the calls by id, so a read is compared as the set of calls it returns: the day tables + * are read one after another, and a relation stored in both returns its calls once from each. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +abstract class JDBCTopologyQueryITBase { + private static final int TOMCAT = 1; + private static final int MYSQL = 5; + private static final int GRPC = 23; + private static final int HTTP = 49; + /** Readable names of the ids the fixture builds, to print the calls read. */ + private static final Map NAMES = new HashMap<>(); + + private static final String USER = service("User", false); + private static final String GATEWAY = service("gateway", true); + private static final String CHECKOUT = service("checkout", true); + private static final String PAYMENT = service("payment", true); + private static final String LEDGER = service("ledger", true); + private static final String AUDIT = service("audit", true); + private static final String ORDERS_DB = service("orders-db:3306", false); + + private static final String GATEWAY_1 = instance(GATEWAY, "gateway-1"); + private static final String GATEWAY_2 = instance(GATEWAY, "gateway-2"); + private static final String CHECKOUT_1 = instance(CHECKOUT, "checkout-1"); + private static final String PAYMENT_1 = instance(PAYMENT, "payment-1"); + + private static final String CHECKOUT_1_ENVOY = process(CHECKOUT_1, "envoy"); + private static final String CHECKOUT_1_JAVA = process(CHECKOUT_1, "java"); + private static final String CHECKOUT_1_PYTHON = process(CHECKOUT_1, "python"); + private static final String PAYMENT_1_ENVOY = process(PAYMENT_1, "envoy"); + private static final String PAYMENT_1_JAVA = process(PAYMENT_1, "java"); + + private static final String SCHEDULER = service("scheduler", true); + private static final String SCHEDULER_1 = instance(SCHEDULER, "scheduler-1"); + private static final String SCHEDULER_1_CRON = process(SCHEDULER_1, "cron"); + private static final String SCHEDULER_1_WORKER = process(SCHEDULER_1, "worker"); + /** The first of the minutes in which a scheduler relation names another component each. */ + private static final int FIRST_COMPONENT_MINUTE = 10; + /** One component per minute, more than ten. */ + private static final int COMPONENTS = 12; + /** The names the component library gives the component ids of the fixture. */ + private static final Map COMPONENT_NAMES = componentNames(); + + private final JdbcDatabaseContainer server; + private final JDBCQueryITSupport.Target target; + private JDBCQueryITSupport support; + private JDBCTopologyQueryDAO dao; + private TopologyQueryService topologyQueryService; + + JDBCTopologyQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { + this.server = server; + this.target = target; + } + + @BeforeAll + public void writeRelations() throws Exception { + support = JDBCQueryITSupport.connect(server, target); + + final Model services = support.install( + ServiceRelationServerSideMetrics.class, DefaultScopeDefine.SERVICE_RELATION, + ServiceRelationServerSideMetrics.INDEX_NAME, DownSampling.Minute); + support.writeMetrics( + services, + new ServiceRelationServerSideMetrics.Builder(), + Arrays.asList( + serverSide(USER, GATEWAY, minute(-1), HTTP), + serverSide(USER, GATEWAY, minute(0), HTTP), + serverSide(GATEWAY, CHECKOUT, minute(-1), HTTP), + serverSide(CHECKOUT, PAYMENT, minute(-1), HTTP), + serverSide(CHECKOUT, PAYMENT, minute(0), GRPC), + serverSide(PAYMENT, CHECKOUT, minute(0), GRPC, HTTP), + serverSide(LEDGER, AUDIT, minute(-2), HTTP), + serverSide(LEDGER, AUDIT, minute(1), HTTP))); + support.writeMetrics( + support.install(ServiceRelationServerSideMetrics.class, DefaultScopeDefine.SERVICE_RELATION, + ServiceRelationServerSideMetrics.INDEX_NAME, DownSampling.Hour), + new ServiceRelationServerSideMetrics.Builder(), + Arrays.asList( + serverSide(AUDIT, GATEWAY, hour(-1), HTTP), + serverSide(GATEWAY, AUDIT, hour(0), TOMCAT), + serverSide(CHECKOUT, PAYMENT, hour(-1), GRPC), + serverSide(LEDGER, GATEWAY, hour(-2), HTTP), + serverSide(LEDGER, GATEWAY, hour(1), HTTP))); + final Model servicesAtClient = support.install( + ServiceRelationClientSideMetrics.class, DefaultScopeDefine.SERVICE_RELATION, + ServiceRelationClientSideMetrics.INDEX_NAME, DownSampling.Minute); + support.writeMetrics( + servicesAtClient, + new ServiceRelationClientSideMetrics.Builder(), + Arrays.asList( + clientSide(GATEWAY, CHECKOUT, minute(0), HTTP), + clientSide(CHECKOUT, PAYMENT, minute(-1), HTTP), + clientSide(CHECKOUT, PAYMENT, minute(0), HTTP, GRPC), + clientSide(PAYMENT, CHECKOUT, minute(0), GRPC), + clientSide(CHECKOUT, ORDERS_DB, minute(-1), MYSQL), + clientSide(CHECKOUT, ORDERS_DB, minute(0), MYSQL), + clientSide(LEDGER, AUDIT, minute(-2), HTTP), + clientSide(LEDGER, AUDIT, minute(1), HTTP))); + + support.writeMetrics( + support.install(ServiceInstanceRelationServerSideMetrics.class, + DefaultScopeDefine.SERVICE_INSTANCE_RELATION, + ServiceInstanceRelationServerSideMetrics.INDEX_NAME, DownSampling.Minute), + new ServiceInstanceRelationServerSideMetrics.Builder(), + Arrays.asList( + instanceServerSide(GATEWAY_1, CHECKOUT_1, -1), + instanceServerSide(GATEWAY_2, CHECKOUT_1, 0), + instanceServerSide(CHECKOUT_1, GATEWAY_1, 0), + instanceServerSide(GATEWAY_1, PAYMENT_1, -1), + instanceServerSide(PAYMENT_1, CHECKOUT_1, 0), + instanceServerSide(CHECKOUT_1, GATEWAY_2, -2), + instanceServerSide(CHECKOUT_1, GATEWAY_2, 1))); + support.writeMetrics( + support.install(ServiceInstanceRelationClientSideMetrics.class, + DefaultScopeDefine.SERVICE_INSTANCE_RELATION, + ServiceInstanceRelationClientSideMetrics.INDEX_NAME, DownSampling.Minute), + new ServiceInstanceRelationClientSideMetrics.Builder(), + Arrays.asList( + instanceClientSide(GATEWAY_1, CHECKOUT_1, -1), + instanceClientSide(GATEWAY_1, CHECKOUT_1, 0), + instanceClientSide(CHECKOUT_1, GATEWAY_2, 0), + instanceClientSide(PAYMENT_1, CHECKOUT_1, -1), + instanceClientSide(GATEWAY_2, CHECKOUT_1, -2), + instanceClientSide(GATEWAY_2, CHECKOUT_1, 1))); + + support.writeMetrics( + support.install(EndpointRelationServerSideMetrics.class, DefaultScopeDefine.ENDPOINT_RELATION, + EndpointRelationServerSideMetrics.INDEX_NAME, DownSampling.Minute), + new EndpointRelationServerSideMetrics.Builder(), + Arrays.asList( + endpointRelation(GATEWAY, "/checkout", CHECKOUT, "/pay", -1), + endpointRelation(CHECKOUT, "/pay", PAYMENT, "/charge", 0), + endpointRelation(GATEWAY, "/cart", CHECKOUT, "/stock", -1), + endpointRelation(GATEWAY, "/cart", CHECKOUT, "/stock", 0), + endpointRelation(GATEWAY, "/pay", PAYMENT, "/charge", 0), + endpointRelation(LEDGER, "/refund", CHECKOUT, "/pay", -2), + endpointRelation(LEDGER, "/refund", CHECKOUT, "/pay", 1))); + + final Model processes = support.install( + ProcessRelationServerSideMetrics.class, DefaultScopeDefine.PROCESS_RELATION, + ProcessRelationServerSideMetrics.INDEX_NAME, DownSampling.Minute); + support.writeMetrics( + processes, + new ProcessRelationServerSideMetrics.Builder(), + Arrays.asList( + processServerSide(CHECKOUT_1, CHECKOUT_1_ENVOY, CHECKOUT_1_JAVA, -1, HTTP), + processServerSide(CHECKOUT_1, CHECKOUT_1_ENVOY, CHECKOUT_1_JAVA, 0, HTTP), + processServerSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_ENVOY, -1, GRPC), + processServerSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_ENVOY, 0, HTTP), + processServerSide(PAYMENT_1, PAYMENT_1_ENVOY, PAYMENT_1_JAVA, 0, HTTP), + processServerSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_PYTHON, -2, HTTP), + processServerSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_PYTHON, 1, HTTP))); + final Model processesAtClient = support.install( + ProcessRelationClientSideMetrics.class, DefaultScopeDefine.PROCESS_RELATION, + ProcessRelationClientSideMetrics.INDEX_NAME, DownSampling.Minute); + support.writeMetrics( + processesAtClient, + new ProcessRelationClientSideMetrics.Builder(), + Arrays.asList( + processClientSide(CHECKOUT_1, CHECKOUT_1_ENVOY, CHECKOUT_1_JAVA, -1, HTTP), + processClientSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_PYTHON, 0, GRPC), + processClientSide(PAYMENT_1, PAYMENT_1_ENVOY, PAYMENT_1_JAVA, -1, HTTP), + processClientSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_ENVOY, -2, HTTP), + processClientSide(CHECKOUT_1, CHECKOUT_1_JAVA, CHECKOUT_1_ENVOY, 1, HTTP))); + + final Model processTrafficModel = support.install( + ProcessTraffic.class, DefaultScopeDefine.PROCESS, ProcessTraffic.INDEX_NAME, DownSampling.Minute); + final List schedulerCalls = new ArrayList<>(); + final List schedulerCallsAtClient = new ArrayList<>(); + final List cronCalls = new ArrayList<>(); + final List cronCallsAtClient = new ArrayList<>(); + for (int component = 1; component <= COMPONENTS; component++) { + final int minute = FIRST_COMPONENT_MINUTE + component - 1; + schedulerCalls.add(serverSide(SCHEDULER, LEDGER, minute(minute), component)); + schedulerCallsAtClient.add(clientSide(SCHEDULER, LEDGER, minute(minute), component)); + cronCalls.add(processServerSide(SCHEDULER_1, SCHEDULER_1_CRON, SCHEDULER_1_WORKER, minute, component)); + cronCallsAtClient.add( + processClientSide(SCHEDULER_1, SCHEDULER_1_CRON, SCHEDULER_1_WORKER, minute, component)); + } + support.writeMetrics(services, new ServiceRelationServerSideMetrics.Builder(), schedulerCalls); + support.writeMetrics(servicesAtClient, new ServiceRelationClientSideMetrics.Builder(), schedulerCallsAtClient); + support.writeMetrics(processes, new ProcessRelationServerSideMetrics.Builder(), cronCalls); + support.writeMetrics(processesAtClient, new ProcessRelationClientSideMetrics.Builder(), cronCallsAtClient); + + support.writeMetrics( + processTrafficModel, + new ProcessTraffic.Builder(), + Arrays.asList( + processTraffic(CHECKOUT_1, "envoy", -1), + processTraffic(CHECKOUT_1, "java", -1), + processTraffic(CHECKOUT_1, "python", 0), + processTraffic(PAYMENT_1, "envoy", 0), + processTraffic(PAYMENT_1, "java", 0), + processTraffic(SCHEDULER_1, "cron", -1), + processTraffic(SCHEDULER_1, "worker", 0))); + + dao = new JDBCTopologyQueryDAO(support.getClient(), support.getTableHelper()); + topologyQueryService = topologyQueryService(new JDBCStorageDAO(support.getClient()), processTrafficModel); + } + + @AfterAll + public void close() { + if (support != null) { + support.close(); + } + } + + @Test + public void readsEveryServiceRelationDetectedAtTheServerSide() throws Exception { + assertEquals( + set( + "checkout -> payment #23 SERVER", + "checkout -> payment #49 SERVER", + "gateway -> checkout #49 SERVER", + "payment -> checkout #23 SERVER", + "payment -> checkout #49 SERVER", + "User -> gateway #49 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(minutes()))); + } + + @Test + public void readsEveryServiceRelationDetectedAtTheClientSide() throws Exception { + assertEquals( + set( + "checkout -> orders-db:3306 #5 CLIENT", + "checkout -> payment #23 CLIENT", + "checkout -> payment #49 CLIENT", + "gateway -> checkout #49 CLIENT", + "payment -> checkout #23 CLIENT"), + calls(dao.loadServiceRelationDetectedAtClientSide(minutes()))); + } + + @Test + public void readsTheServiceRelationsOfOneServiceEitherWay() throws Exception { + final List services = Collections.singletonList(CHECKOUT); + assertEquals( + set( + "checkout -> payment #23 SERVER", + "checkout -> payment #49 SERVER", + "gateway -> checkout #49 SERVER", + "payment -> checkout #23 SERVER", + "payment -> checkout #49 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(minutes(), services))); + assertEquals( + set( + "checkout -> orders-db:3306 #5 CLIENT", + "checkout -> payment #23 CLIENT", + "checkout -> payment #49 CLIENT", + "gateway -> checkout #49 CLIENT", + "payment -> checkout #23 CLIENT"), + calls(dao.loadServiceRelationDetectedAtClientSide(minutes(), services))); + } + + @Test + public void readsTheServiceRelationsOfSomeServices() throws Exception { + assertEquals( + set( + "gateway -> checkout #49 SERVER", + "User -> gateway #49 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(minutes(), Arrays.asList(GATEWAY, ORDERS_DB)))); + assertEquals( + set( + "checkout -> orders-db:3306 #5 CLIENT", + "gateway -> checkout #49 CLIENT"), + calls(dao.loadServiceRelationDetectedAtClientSide(minutes(), Arrays.asList(GATEWAY, ORDERS_DB)))); + } + + @Test + public void readsNoServiceRelationOfServicesThatOnlyCalledOutsideTheRange() throws Exception { + final List services = Arrays.asList(LEDGER, AUDIT); + assertEquals(set(), calls(dao.loadServiceRelationsDetectedAtServerSide(minutes(), services))); + assertEquals(set(), calls(dao.loadServiceRelationDetectedAtClientSide(minutes(), services))); + } + + @Test + public void readsTheHourlyServiceRelationsAtAnHourStep() throws Exception { + assertEquals( + set( + "audit -> gateway #49 SERVER", + "checkout -> payment #23 SERVER", + "gateway -> audit #1 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(hours()))); + assertEquals( + set( + "audit -> gateway #49 SERVER", + "gateway -> audit #1 SERVER"), + calls(dao.loadServiceRelationsDetectedAtServerSide(hours(), Collections.singletonList(GATEWAY)))); + } + + @Test + public void readsTheInstanceRelationsOfAServicePairBothWays() throws Exception { + final Set serverSide = set( + "checkout-1 -> gateway-1 #null SERVER", + "gateway-1 -> checkout-1 #null SERVER", + "gateway-2 -> checkout-1 #null SERVER"); + assertEquals(serverSide, calls(dao.loadInstanceRelationDetectedAtServerSide(GATEWAY, CHECKOUT, minutes()))); + assertEquals(serverSide, calls(dao.loadInstanceRelationDetectedAtServerSide(CHECKOUT, GATEWAY, minutes()))); + + final Set clientSide = set( + "checkout-1 -> gateway-2 #null CLIENT", + "gateway-1 -> checkout-1 #null CLIENT"); + assertEquals(clientSide, calls(dao.loadInstanceRelationDetectedAtClientSide(GATEWAY, CHECKOUT, minutes()))); + assertEquals(clientSide, calls(dao.loadInstanceRelationDetectedAtClientSide(CHECKOUT, GATEWAY, minutes()))); + } + + @Test + public void readsTheRelationsOfAnEndpointEitherWay() throws Exception { + assertEquals( + set( + "checkout:/pay -> payment:/charge #0 SERVER", + "gateway:/checkout -> checkout:/pay #0 SERVER"), + calls(dao.loadEndpointRelation(minutes(), endpoint(CHECKOUT, "/pay")))); + assertEquals( + set("gateway:/cart -> checkout:/stock #0 SERVER"), + calls(dao.loadEndpointRelation(minutes(), endpoint(GATEWAY, "/cart")))); + assertEquals(set(), calls(dao.loadEndpointRelation(minutes(), endpoint(LEDGER, "/refund")))); + } + + @Test + public void readsTheProcessRelationsOfAnInstance() throws Exception { + assertEquals( + set( + "checkout-1/envoy -> checkout-1/java #49 SERVER", + "checkout-1/java -> checkout-1/envoy #23 SERVER", + "checkout-1/java -> checkout-1/envoy #49 SERVER"), + calls(dao.loadProcessRelationDetectedAtServerSide(CHECKOUT_1, minutes()))); + assertEquals( + set( + "checkout-1/envoy -> checkout-1/java #49 CLIENT", + "checkout-1/java -> checkout-1/python #23 CLIENT"), + calls(dao.loadProcessRelationDetectedAtClientSide(CHECKOUT_1, minutes()))); + assertEquals( + set("payment-1/envoy -> payment-1/java #49 SERVER"), + calls(dao.loadProcessRelationDetectedAtServerSide(PAYMENT_1, minutes()))); + } + + @Test + public void readsEveryComponentOfARelationThatChangesItEveryMinute() throws Exception { + final Duration range = componentMinutes(); + final Set services = new TreeSet<>(); + final Set servicesAtClient = new TreeSet<>(); + final Set processes = new TreeSet<>(); + final Set processesAtClient = new TreeSet<>(); + for (int component = 1; component <= COMPONENTS; component++) { + services.add("scheduler -> ledger #" + component + " SERVER"); + servicesAtClient.add("scheduler -> ledger #" + component + " CLIENT"); + processes.add("scheduler-1/cron -> scheduler-1/worker #" + component + " SERVER"); + processesAtClient.add("scheduler-1/cron -> scheduler-1/worker #" + component + " CLIENT"); + } + assertEquals(services, calls(dao.loadServiceRelationsDetectedAtServerSide(range))); + assertEquals(services, calls(dao.loadServiceRelationsDetectedAtServerSide( + range, Collections.singletonList(SCHEDULER)))); + assertEquals(servicesAtClient, calls(dao.loadServiceRelationDetectedAtClientSide(range))); + assertEquals(processes, calls(dao.loadProcessRelationDetectedAtServerSide(SCHEDULER_1, range))); + assertEquals(processesAtClient, calls(dao.loadProcessRelationDetectedAtClientSide(SCHEDULER_1, range))); + } + + /** + * A built call keeps every component its relation named in the range: the client side's as its source + * components, the server side's as its target components. No network address alias and no service is known, so + * no call is folded onto another service. + */ + @Test + public void buildsTheServiceTopologyWithEveryComponentOfBothSides() throws Exception { + assertEquals( + sorted( + "User -> gateway [SERVER] [] [http]", + "checkout -> orders-db:3306 [CLIENT] [Mysql] []", + "checkout -> payment [CLIENT, SERVER] [GRPC, http] [GRPC, http]", + "gateway -> checkout [CLIENT, SERVER] [http] [http]", + "payment -> checkout [CLIENT, SERVER] [GRPC] [GRPC, http]"), + builtCalls(topologyQueryService.getGlobalTopology(minutes(), null).getCalls())); + assertEquals( + sorted("scheduler -> ledger [CLIENT, SERVER] " + library() + " " + library()), + builtCalls(topologyQueryService.getGlobalTopology(componentMinutes(), null).getCalls())); + } + + @Test + public void buildsTheProcessTopologyWithEveryComponentOfBothSides() throws Exception { + final ProcessTopology checkout = topologyQueryService.getProcessTopology(CHECKOUT_1, minutes()); + assertEquals( + sorted( + "checkout-1/envoy -> checkout-1/java [CLIENT, SERVER] [http] [http]", + "checkout-1/java -> checkout-1/envoy [SERVER] [] [GRPC, http]", + "checkout-1/java -> checkout-1/python [CLIENT] [GRPC] []"), + builtCalls(checkout.getCalls())); + assertEquals( + sorted("checkout-1/envoy of checkout", "checkout-1/java of checkout", "checkout-1/python of checkout"), + processNodes(checkout.getNodes())); + + final ProcessTopology scheduler = topologyQueryService.getProcessTopology(SCHEDULER_1, componentMinutes()); + assertEquals( + sorted("scheduler-1/cron -> scheduler-1/worker [CLIENT, SERVER] " + library() + " " + library()), + builtCalls(scheduler.getCalls())); + assertEquals( + sorted("scheduler-1/cron of scheduler", "scheduler-1/worker of scheduler"), + processNodes(scheduler.getNodes())); + } + + /** + * The calls read, as {@code source -> target #component detect point}. A call whose id is not the relation of + * its source and target is printed with its id. + */ + private static Set calls(final List calls) { + return calls.stream().map(call -> { + final String relation = name(call.getSource()) + " -> " + name(call.getTarget()); + final String id = relation.equals(name(call.getId())) ? "" : " id " + name(call.getId()); + return relation + " #" + call.getComponentId() + " " + call.getDetectPoint() + id; + }).collect(Collectors.toCollection(TreeSet::new)); + } + + /** + * The calls of a built topology, sorted, as {@code source -> target [detect points] [source components] [target + * components]}. A call whose id is not the relation of its source and target is printed with its id. + */ + private static List builtCalls(final List calls) { + return calls.stream().map(call -> { + final String relation = name(call.getSource()) + " -> " + name(call.getTarget()); + final String id = relation.equals(name(call.getId())) ? "" : " id " + name(call.getId()); + return relation + " " + sortedValues(call.getDetectPoints()) + + " " + sortedValues(call.getSourceComponents()) + + " " + sortedValues(call.getTargetComponents()) + id; + }).sorted().collect(Collectors.toList()); + } + + /** The nodes of a process topology, sorted, as {@code instance/process of service}. */ + private static List processNodes(final List nodes) { + return nodes.stream() + .map(node -> node.getServiceInstanceName() + "/" + node.getName() + " of " + node.getServiceName() + + (name(node.getId()).equals(node.getServiceInstanceName() + "/" + node.getName()) + ? "" : " id " + name(node.getId())) + + (node.isReal() ? "" : " virtual")) + .sorted() + .collect(Collectors.toList()); + } + + private static List sortedValues(final List values) { + return values.stream().map(String::valueOf).sorted().collect(Collectors.toList()); + } + + /** The names of the components the scheduler relations name, one per minute. */ + private static List library() { + return sortedValues( + IntStream.rangeClosed(1, COMPONENTS).mapToObj(COMPONENT_NAMES::get).collect(Collectors.toList())); + } + + private static List sorted(final String... values) { + return Arrays.stream(values).sorted().collect(Collectors.toList()); + } + + private static Map componentNames() { + final Map names = new HashMap<>(); + final String[] library = { + "Tomcat", "HttpClient", "Dubbo", "H2", "Mysql", "ORACLE", "Redis", "Motan", "MongoDB", "Resin", "Feign", + "OKHttp" + }; + for (int i = 0; i < library.length; i++) { + names.put(i + 1, library[i]); + } + names.put(GRPC, "GRPC"); + names.put(HTTP, "http"); + return names; + } + + /** + * The topology query service over the DAO under test, with OAP's storage DAO to read the process traffic. + */ + private TopologyQueryService topologyQueryService(final StorageDAO storageDAO, final Model processTraffic) { + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleServiceHolder storage = services(moduleManager, StorageModule.NAME); + when(storage.getService(ITopologyQueryDAO.class)).thenReturn(dao); + when(storage.getService(StorageDAO.class)).thenReturn(storageDAO); + final IComponentLibraryCatalogService components = mock(IComponentLibraryCatalogService.class); + when(components.getComponentName(anyInt())).thenAnswer( + invocation -> COMPONENT_NAMES.get(invocation.getArgument(0))); + final ModuleServiceHolder core = services(moduleManager, CoreModule.NAME); + when(core.getService(IComponentLibraryCatalogService.class)).thenReturn(components); + when(core.getService(NetworkAddressAliasCache.class)).thenReturn( + new NetworkAddressAliasCache(new CoreModuleConfig())); + when(core.getService(MetadataQueryService.class)).thenReturn(mock(MetadataQueryService.class)); + final StorageModels models = mock(StorageModels.class); + when(models.allModels()).thenReturn(Collections.singletonList(processTraffic)); + return new TopologyQueryService(moduleManager, models); + } + + private static ModuleServiceHolder services(final ModuleManager moduleManager, final String moduleName) { + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(moduleManager.find(moduleName)).thenReturn(provider); + when(provider.provider()).thenReturn(services); + return services; + } + + private static Set set(final String... calls) { + return new TreeSet<>(Arrays.asList(calls)); + } + + private static String name(final String id) { + return NAMES.getOrDefault(id, id); + } + + private static Duration minutes() { + return JDBCQueryITSupport.duration(MIDNIGHT - 60_000L, MIDNIGHT, Step.MINUTE); + } + + private static Duration minutes(final long first, final long last) { + return JDBCQueryITSupport.duration(MIDNIGHT + first * 60_000L, MIDNIGHT + last * 60_000L, Step.MINUTE); + } + + private static Duration componentMinutes() { + return minutes(FIRST_COMPONENT_MINUTE, FIRST_COMPONENT_MINUTE + COMPONENTS - 1); + } + + private static Duration hours() { + return JDBCQueryITSupport.duration(MIDNIGHT - 3_600_000L, MIDNIGHT, Step.HOUR); + } + + private static long minute(final long minute) { + return TimeBucket.getMinuteTimeBucket(MIDNIGHT + minute * 60_000L); + } + + private static long hour(final long hour) { + return TimeBucket.getTimeBucket(MIDNIGHT + hour * 3_600_000L, DownSampling.Hour); + } + + private static String service(final String name, final boolean normal) { + final String id = IDManager.ServiceID.buildId(name, normal); + NAMES.put(id, name); + return id; + } + + private static String instance(final String serviceId, final String name) { + final String id = IDManager.ServiceInstanceID.buildId(serviceId, name); + NAMES.put(id, name); + return id; + } + + private static String endpoint(final String serviceId, final String name) { + final String id = IDManager.EndpointID.buildId(serviceId, name); + NAMES.put(id, NAMES.get(serviceId) + ":" + name); + return id; + } + + private static String process(final String instanceId, final String name) { + final String id = IDManager.ProcessID.buildId(instanceId, name); + NAMES.put(id, NAMES.get(instanceId) + "/" + name); + return id; + } + + private static String relation(final String relationId, final String source, final String dest) { + NAMES.put(relationId, name(source) + " -> " + name(dest)); + return relationId; + } + + private static String serviceRelation(final String source, final String dest) { + return relation(IDManager.ServiceID.buildRelationId( + new IDManager.ServiceID.ServiceRelationDefine(source, dest)), source, dest); + } + + private static String instanceRelation(final String source, final String dest) { + return relation(IDManager.ServiceInstanceID.buildRelationId( + new IDManager.ServiceInstanceID.ServiceInstanceRelationDefine(source, dest)), source, dest); + } + + private static String processRelation(final String source, final String dest) { + return relation(IDManager.ProcessID.buildRelationId( + new IDManager.ProcessID.ProcessRelationDefine(source, dest)), source, dest); + } + + private static ProcessTraffic processTraffic(final String instanceId, final String name, + final long firstSeenMinute) { + final ProcessTraffic process = new ProcessTraffic(); + process.setServiceId(IDManager.ServiceInstanceID.analysisId(instanceId).getServiceId()); + process.setInstanceId(instanceId); + process.setName(name); + process.setAgentId("rover-1"); + process.setDetectType(ProcessDetectType.VM.value()); + process.setProfilingSupportStatus(ProfilingSupportStatus.SUPPORT_EBPF_PROFILING.value()); + process.setLabelsJson(Const.EMPTY_STRING); + process.setTimeBucket(minute(firstSeenMinute)); + process.setLastPingTimestamp(minute(firstSeenMinute)); + return process; + } + + private static IntList components(final int... ids) { + final IntList components = new IntList(ids.length); + for (final int id : ids) { + components.add(id); + } + return components; + } + + private static ServiceRelationServerSideMetrics serverSide(final String source, final String dest, + final long timeBucket, final int... components) { + final ServiceRelationServerSideMetrics metrics = new ServiceRelationServerSideMetrics(); + metrics.setSourceServiceId(source); + metrics.setDestServiceId(dest); + metrics.setComponentIds(components(components)); + metrics.setEntityId(serviceRelation(source, dest)); + metrics.setTimeBucket(timeBucket); + return metrics; + } + + private static ServiceRelationClientSideMetrics clientSide(final String source, final String dest, + final long timeBucket, final int... components) { + final ServiceRelationClientSideMetrics metrics = new ServiceRelationClientSideMetrics(); + metrics.setSourceServiceId(source); + metrics.setDestServiceId(dest); + metrics.setComponentIds(components(components)); + metrics.setEntityId(serviceRelation(source, dest)); + metrics.setTimeBucket(timeBucket); + return metrics; + } + + private static ServiceInstanceRelationServerSideMetrics instanceServerSide(final String source, final String dest, + final long minute) { + final ServiceInstanceRelationServerSideMetrics metrics = new ServiceInstanceRelationServerSideMetrics(); + metrics.setSourceServiceId(IDManager.ServiceInstanceID.analysisId(source).getServiceId()); + metrics.setSourceServiceInstanceId(source); + metrics.setDestServiceId(IDManager.ServiceInstanceID.analysisId(dest).getServiceId()); + metrics.setDestServiceInstanceId(dest); + metrics.setEntityId(instanceRelation(source, dest)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } + + private static ServiceInstanceRelationClientSideMetrics instanceClientSide(final String source, final String dest, + final long minute) { + final ServiceInstanceRelationClientSideMetrics metrics = new ServiceInstanceRelationClientSideMetrics(); + metrics.setSourceServiceId(IDManager.ServiceInstanceID.analysisId(source).getServiceId()); + metrics.setSourceServiceInstanceId(source); + metrics.setDestServiceId(IDManager.ServiceInstanceID.analysisId(dest).getServiceId()); + metrics.setDestServiceInstanceId(dest); + metrics.setEntityId(instanceRelation(source, dest)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } + + private static EndpointRelationServerSideMetrics endpointRelation(final String sourceService, + final String source, + final String destService, + final String dest, + final long minute) { + final EndpointRelationServerSideMetrics metrics = new EndpointRelationServerSideMetrics(); + final String sourceId = endpoint(sourceService, source); + final String destId = endpoint(destService, dest); + metrics.setSourceEndpoint(sourceId); + metrics.setDestEndpoint(destId); + metrics.setComponentId(HTTP); + metrics.setEntityId(relation( + IDManager.EndpointID.buildRelationId( + new IDManager.EndpointID.EndpointRelationDefine(sourceService, source, destService, dest)), + sourceId, destId)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } + + private static ProcessRelationServerSideMetrics processServerSide(final String instance, final String source, + final String dest, final long minute, + final int component) { + final ProcessRelationServerSideMetrics metrics = new ProcessRelationServerSideMetrics(); + metrics.setServiceInstanceId(instance); + metrics.setSourceProcessId(source); + metrics.setDestProcessId(dest); + metrics.setComponentId(component); + metrics.setEntityId(processRelation(source, dest)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } + + private static ProcessRelationClientSideMetrics processClientSide(final String instance, final String source, + final String dest, final long minute, + final int component) { + final ProcessRelationClientSideMetrics metrics = new ProcessRelationClientSideMetrics(); + metrics.setServiceInstanceId(instance); + metrics.setSourceProcessId(source); + metrics.setDestProcessId(dest); + metrics.setComponentId(component); + metrics.setEntityId(processRelation(source, dest)); + metrics.setTimeBucket(minute(minute)); + return metrics; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryMariaDBIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryMariaDBIT.java new file mode 100644 index 000000000000..d70d7a29df43 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryMariaDBIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCTopologyQueryMariaDBIT extends JDBCTopologyQueryITBase { + @Container + private static final MariaDBContainer SERVER = JDBCQueryITSupport.mariaDB(); + + public JDBCTopologyQueryMariaDBIT() { + super(SERVER, JDBCQueryITSupport.Target.MARIADB); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryPostgreSQLIT.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryPostgreSQLIT.java new file mode 100644 index 000000000000..3fe0e5185862 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTopologyQueryPostgreSQLIT.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Testcontainers +public class JDBCTopologyQueryPostgreSQLIT extends JDBCTopologyQueryITBase { + @Container + private static final PostgreSQLContainer SERVER = JDBCQueryITSupport.postgreSQL(); + + public JDBCTopologyQueryPostgreSQLIT() { + super(SERVER, JDBCQueryITSupport.Target.POSTGRESQL); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTraceQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTraceQueryITBase.java index de74f23947ce..7b87e4578269 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTraceQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCTraceQueryITBase.java @@ -18,55 +18,128 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; import java.util.List; -import java.util.Set; -import java.util.TreeSet; +import java.util.Map; +import java.util.function.Consumer; import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanType; +import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag; import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.config.SearchableTracesTagsWatcher; import org.apache.skywalking.oap.server.core.query.enumeration.Step; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.query.type.BasicTrace; import org.apache.skywalking.oap.server.core.query.type.QueryOrder; import org.apache.skywalking.oap.server.core.query.type.TraceState; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.library.module.ModuleProviderHolder; +import org.apache.skywalking.oap.server.library.module.ModuleServiceHolder; +import org.joda.time.DateTime; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.MethodSource; import org.testcontainers.containers.JdbcDatabaseContainer; import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** - * Reads segments the way the trace list and trace profiling do, on segments written on both sides of midnight, so - * in two day tables. A subclass names the server. + * Reads native segments the way the trace list and trace profiling do on the JDBC storages, through + * {@link JDBCTraceQueryDAO}: the trace search, the segments of one trace by its id, and the two reads of a trace + * profiling task, its profiled segments by id and then the segments of their traces on the profiled instances. A + * subclass names the server. + * + *

The search range is a minute either side of midnight, so it reads two day tables, and a tag condition joins the + * segments' tag table of each, which only the tag keys OAP is configured to search may do. Each search condition is + * tried alone, in both orders, and with all the others. For every condition a near-miss segment matches all the + * others but that one, and the two matching segments lie on the bounds of the range and of the latency. The range is + * read from its first millisecond through its end instant, {@link Duration#getEndTimestamp()}, the first millisecond + * after its last second, so the segments just outside it start a millisecond before and a millisecond after. A segment + * of its own starts on the end instant of a range ending in a day's last second: the next midnight, in the next day's + * table. The search lists segments, not traces: the trace that matches every + * condition has a matching segment either side of midnight, and a third segment, of another service, after the range. + * The reads by trace id and by segment id are not bounded by the range on this storage. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) abstract class JDBCTraceQueryITBase { - private static final String SERVICE_ID = IDManager.ServiceID.buildId("checkout", true); - private static final String INSTANCE_ID = IDManager.ServiceInstanceID.buildId(SERVICE_ID, "checkout-1"); - private static final String ENDPOINT_ID = IDManager.EndpointID.buildId(SERVICE_ID, "POST /orders"); - /** Segment id, offset from midnight in seconds, latency in ms. Three before midnight, two after. */ - private static final Object[][] SEGMENTS = { - {"s1", -30, 300}, - {"s2", -20, 100}, - {"s3", -10, 500}, - {"s4", 10, 200}, - {"s5", 20, 400}, - }; + private static final long START = MIDNIGHT - 60_000L; + /** The range's end instant, the first millisecond after its last second. */ + private static final long END = MIDNIGHT + 60_000L; + private static final Duration AROUND_MIDNIGHT = duration(START, END - 1_000L); + /** Yesterday's midnight. */ + private static final long PREVIOUS_MIDNIGHT = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + + private static final String CHECKOUT = "checkout"; + private static final String CANARY = "checkout-canary"; + private static final String INVENTORY = "inventory"; + private static final String CHECKOUT_1 = "checkout-1"; + private static final String OTHER_INSTANCE = "checkout-2"; + private static final String POST_ORDERS = "POST:/orders"; + private static final String OTHER_ENDPOINT = "GET:/orders"; + private static final int MIN_LATENCY = 100; + private static final int MAX_LATENCY = 900; + private static final Tag METHOD = new Tag("http.method", "POST"); + private static final Tag STATUS = new Tag("http.status_code", "500"); + + /** The trace matching every condition. */ + private static final String TRACE = "4a2b6c8d0e1f.37.17684352000000001"; + /** One digit off the matching trace's id. */ + private static final String OTHER_TRACE = "4a2b6c8d0e1f.37.17684352000000002"; + private static final String UNKNOWN_TRACE = "4a2b6c8d0e1f.37.17684352000000009"; + + /** On the range's start, before midnight, at the least latency. */ + private static final String FIRST_MATCH = "segment-first-match"; + /** On the range's end instant, after midnight, at the most latency. */ + private static final String LAST_MATCH = "segment-last-match"; + private static final String OTHER_SERVICE = "segment-other-service"; + private static final String OTHER_INSTANCE_SEGMENT = "segment-other-instance"; + private static final String OTHER_ENDPOINT_SEGMENT = "segment-other-endpoint"; + private static final String OTHER_TRACE_SEGMENT = "segment-other-trace"; + private static final String BELOW_MIN_LATENCY = "segment-below-min-latency"; + private static final String ABOVE_MAX_LATENCY = "segment-above-max-latency"; + private static final String SUCCEEDED = "segment-succeeded"; + /** Has one of the two tags searched for, and the other with another value. */ + private static final String OTHER_TAG_VALUE = "segment-other-tag-value"; + private static final String BEFORE_START = "segment-before-start"; + private static final String AFTER_END = "segment-after-end"; + private static final String ON_PREVIOUS_MIDNIGHT = "segment-on-previous-midnight"; + /** The matching trace's call to the inventory, after the range. */ + private static final String DOWNSTREAM = "segment-downstream"; + + private static final Map SEGMENTS = fixture(); + /** The segments in the range, latest start first. */ + private static final List IN_RANGE = Arrays.asList( + LAST_MATCH, OTHER_TRACE_SEGMENT, OTHER_TAG_VALUE, SUCCEEDED, ABOVE_MAX_LATENCY, BELOW_MIN_LATENCY, + OTHER_ENDPOINT_SEGMENT, OTHER_INSTANCE_SEGMENT, OTHER_SERVICE, FIRST_MATCH); private final JdbcDatabaseContainer server; private final JDBCQueryITSupport.Target target; private JDBCQueryITSupport support; private JDBCTraceQueryDAO dao; - private Duration duration; JDBCTraceQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { this.server = server; @@ -76,15 +149,19 @@ abstract class JDBCTraceQueryITBase { @BeforeAll public void writeSegments() throws Exception { support = JDBCQueryITSupport.connect(server, target); - final var model = support.install( + final Model model = support.install( SegmentRecord.class, DefaultScopeDefine.SEGMENT, SegmentRecord.INDEX_NAME, DownSampling.Second); - final List segments = new ArrayList<>(); - for (final Object[] segment : SEGMENTS) { - segments.add(segment((String) segment[0], MIDNIGHT + (int) segment[1] * 1000L, (int) segment[2])); + // Neither in start order nor against it. + final List records = new ArrayList<>(); + for (final String id : Arrays.asList( + OTHER_TAG_VALUE, FIRST_MATCH, AFTER_END, OTHER_INSTANCE_SEGMENT, DOWNSTREAM, SUCCEEDED, LAST_MATCH, + BELOW_MIN_LATENCY, OTHER_SERVICE, BEFORE_START, OTHER_TRACE_SEGMENT, ABOVE_MAX_LATENCY, + OTHER_ENDPOINT_SEGMENT, ON_PREVIOUS_MIDNIGHT)) { + records.add(SEGMENTS.get(id).record()); } - support.writeRecords(model, new SegmentRecord.Builder(), segments); - dao = new JDBCTraceQueryDAO(support.getModuleManager(), support.getClient(), support.getTableHelper()); - duration = JDBCQueryITSupport.duration(MIDNIGHT - 60_000L, MIDNIGHT + 60_000L, Step.SECOND); + support.writeRecords(model, new SegmentRecord.Builder(), records); + dao = new JDBCTraceQueryDAO(searchableTags(METHOD.getKey() + "," + STATUS.getKey()), support.getClient(), + support.getTableHelper()); } @AfterAll @@ -94,69 +171,382 @@ public void close() { } } + @ParameterizedTest(name = "{0} {1}") + @MethodSource("conditionsInBothOrders") + public void findsTheSegmentsMatchingOneCondition(final Condition condition, final QueryOrder order) + throws IOException { + final List expected = expected(condition.matching(), order); + assertEquals(expected, search(condition.criterion, order, 0, expected.size())); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(QueryOrder.class) + public void findsOnlyTheSegmentsMatchingEveryCondition(final QueryOrder order) throws IOException { + final Consumer everyCondition = search -> { + for (final Condition condition : Condition.values()) { + if (condition != Condition.SUCCESS) { + condition.criterion.accept(search); + } + } + }; + assertEquals(expected(Arrays.asList(FIRST_MATCH, LAST_MATCH), order), search(everyCondition, order, 0, 10)); + } + + @Test + public void findsNoSegmentWhenNoneMatches() throws IOException { + assertEquals(Collections.emptyList(), + search(search -> search.serviceId = serviceId("unknown"), QueryOrder.BY_START_TIME, 0, 10)); + } + + @Test + public void findsNoSegmentByATagKeyThatIsNotSearchable() throws IOException { + assertEquals(Collections.emptyList(), search( + search -> search.tags = Arrays.asList(METHOD, new Tag("db.instance", "orders")), + QueryOrder.BY_START_TIME, 0, 10)); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(QueryOrder.class) + public void pagesThroughTheSegmentsOfTheRange(final QueryOrder order) throws IOException { + final List all = expected(IN_RANGE, order); + assertEquals(all.subList(0, 4), search(search -> { }, order, 0, 4)); + assertEquals(all.subList(4, 8), search(search -> { }, order, 4, 4)); + assertEquals(all.subList(8, 10), search(search -> { }, order, 8, 4)); + assertEquals(Collections.emptyList(), search(search -> { }, order, 12, 4)); + } + /** - * apache/skywalking#14239: a trace profiling task without a sampled segment has no segment id to read. + * A range ending in a day's last second ends at the next midnight, which is in the next day's table. */ @Test - public void readsNoSegmentForNoSegmentId() throws Exception { - assertEquals(Collections.emptyList(), dao.queryBySegmentIdList(Collections.emptyList(), duration)); + public void findsASegmentOnTheEndInstantOfARangeEndingAtMidnight() throws IOException { + final Duration endingAtMidnight = duration(PREVIOUS_MIDNIGHT - 60_000L, PREVIOUS_MIDNIGHT - 1_000L); + assertEquals(expected(Collections.singletonList(ON_PREVIOUS_MIDNIGHT), QueryOrder.BY_START_TIME), + search(endingAtMidnight, search -> { }, QueryOrder.BY_START_TIME, 0, 10)); + } + + @Test + public void readsAWholeTraceByIdAcrossMidnight() throws IOException { + final List trace = records(FIRST_MATCH, LAST_MATCH, DOWNSTREAM); + assertEquals(trace, sorted(dao.queryByTraceId(TRACE, AROUND_MIDNIGHT))); + assertEquals(trace, sorted(dao.queryByTraceId(TRACE, null))); + assertEquals(Collections.emptyList(), dao.queryByTraceId(UNKNOWN_TRACE, null)); } + /** + * The flexible trace query reads the traces of a third-party format that has no segments, which this storage + * does not keep. + */ @Test - public void readsTheSegmentsOfTheIdsOfBothDays() throws Exception { - assertEquals(new TreeSet<>(Arrays.asList("s1", "s4")), - segmentIds(dao.queryBySegmentIdList(Arrays.asList("s1", "s4"), duration))); + public void readsNoTraceWithoutSegments() throws IOException { + assertEquals(Collections.emptyList(), dao.doFlexibleTraceQuery(TRACE)); } @Test - public void readsNoSegmentForNoTraceOrNoInstance() throws Exception { - assertEquals(Collections.emptyList(), dao.queryByTraceIdWithInstanceId( - Collections.emptyList(), Collections.singletonList(INSTANCE_ID), null)); - assertEquals(Collections.emptyList(), dao.queryByTraceIdWithInstanceId( - Collections.singletonList("trace-s1"), Collections.emptyList(), null)); + public void readsTheProfiledSegmentsById() throws IOException { + assertEquals(records(FIRST_MATCH, LAST_MATCH, BEFORE_START), + sorted(dao.queryBySegmentIdList( + Arrays.asList(LAST_MATCH, "segment-unknown", BEFORE_START, FIRST_MATCH), AROUND_MIDNIGHT))); } /** - * Each page is cut from the segments of both days ranked together, latest start first. + * apache/skywalking#14239: a trace profiling task without a sampled segment has no segment id to read. */ @Test - public void pagesTheLatestTracesAcrossMidnight() throws Exception { - assertEquals(Arrays.asList("s5", "s4"), page(0, 2, QueryOrder.BY_START_TIME)); - assertEquals(Arrays.asList("s3", "s2"), page(2, 2, QueryOrder.BY_START_TIME)); - assertEquals(Collections.singletonList("s1"), page(4, 2, QueryOrder.BY_START_TIME)); + public void readsNoSegmentForNoSegmentId() throws IOException { + assertEquals(Collections.emptyList(), dao.queryBySegmentIdList(Collections.emptyList(), AROUND_MIDNIGHT)); } + /** + * The segments of other traces on the profiled instance stay out, and so do the listed traces' segments on the + * other instances until those are asked for: the matching trace's call to the inventory, and the other listed + * trace's segment on the second checkout instance. + */ @Test - public void pagesTheSlowestTracesAcrossMidnight() throws Exception { - assertEquals(Arrays.asList("s3", "s5", "s1"), page(0, 3, QueryOrder.BY_DURATION)); - assertEquals(Arrays.asList("s4", "s2"), page(3, 3, QueryOrder.BY_DURATION)); - } - - private List page(final int from, final int limit, final QueryOrder order) throws Exception { - return dao.queryBasicTraces(duration, 0, 0, SERVICE_ID, null, null, null, limit, from, TraceState.ALL, order, null) - .getTraces() - .stream() - .map(BasicTrace::getSegmentId) - .collect(Collectors.toList()); - } - - private static Set segmentIds(final List segments) { - return segments.stream().map(SegmentRecord::getSegmentId).collect(Collectors.toCollection(TreeSet::new)); - } - - private static SegmentRecord segment(final String segmentId, final long startTime, final int latency) { - final SegmentRecord segment = new SegmentRecord(); - segment.setSegmentId(segmentId); - segment.setTraceId("trace-" + segmentId); - segment.setServiceId(SERVICE_ID); - segment.setServiceInstanceId(INSTANCE_ID); - segment.setEndpointId(ENDPOINT_ID); - segment.setStartTime(startTime); - segment.setLatency(latency); - segment.setIsError(0); - segment.setDataBinary(new byte[] {1}); - segment.setTags(new ArrayList<>()); - segment.setTimeBucket(TimeBucket.getRecordTimeBucket(startTime)); - return segment; + public void readsTheSegmentsOfTheProfiledTracesOnTheProfiledInstances() throws IOException { + final List traces = Arrays.asList(TRACE, traceOf(OTHER_INSTANCE_SEGMENT)); + assertEquals(records(FIRST_MATCH, LAST_MATCH), sorted(dao.queryByTraceIdWithInstanceId( + traces, Collections.singletonList(instanceId(CHECKOUT, CHECKOUT_1)), null))); + assertEquals(records(FIRST_MATCH, LAST_MATCH, DOWNSTREAM, OTHER_INSTANCE_SEGMENT), + sorted(dao.queryByTraceIdWithInstanceId(traces, Arrays.asList( + instanceId(CHECKOUT, CHECKOUT_1), instanceId(CHECKOUT, OTHER_INSTANCE), + instanceId(INVENTORY, "inventory-1")), null))); + } + + @Test + public void readsNoSegmentForNoTraceOrNoInstance() throws IOException { + assertEquals(Collections.emptyList(), dao.queryByTraceIdWithInstanceId( + Collections.emptyList(), Collections.singletonList(instanceId(CHECKOUT, CHECKOUT_1)), null)); + assertEquals(Collections.emptyList(), dao.queryByTraceIdWithInstanceId( + Collections.singletonList(TRACE), Collections.emptyList(), null)); + } + + /** What the trace search filters by, and the segments in the range each keeps. */ + enum Condition { + TIME_RANGE_ONLY(search -> { }), + SERVICE(search -> search.serviceId = serviceId(CHECKOUT), OTHER_SERVICE), + // The other service's segment is on an instance and an endpoint of that service. + INSTANCE(search -> search.instanceId = instanceId(CHECKOUT, CHECKOUT_1), + OTHER_INSTANCE_SEGMENT, OTHER_SERVICE), + ENDPOINT(search -> search.endpointId = endpointId(CHECKOUT, POST_ORDERS), + OTHER_ENDPOINT_SEGMENT, OTHER_SERVICE), + TRACE_ID(search -> search.traceId = TRACE, only(FIRST_MATCH, LAST_MATCH)), + MIN_DURATION(search -> search.minLatency = MIN_LATENCY, BELOW_MIN_LATENCY), + MAX_DURATION(search -> search.maxLatency = MAX_LATENCY, ABOVE_MAX_LATENCY), + ERROR(search -> search.state = TraceState.ERROR, SUCCEEDED), + SUCCESS(search -> search.state = TraceState.SUCCESS, only(SUCCEEDED)), + TAGS(search -> search.tags = Arrays.asList(METHOD, STATUS), OTHER_TAG_VALUE); + + private final Consumer criterion; + private final List excluded; + private final List matching; + + Condition(final Consumer criterion, final String... excluded) { + this.criterion = criterion; + this.excluded = Arrays.asList(excluded); + this.matching = null; + } + + Condition(final Consumer criterion, final List matching) { + this.criterion = criterion; + this.excluded = Collections.emptyList(); + this.matching = matching; + } + + private List matching() { + if (matching != null) { + return matching; + } + final List kept = new ArrayList<>(IN_RANGE); + kept.removeAll(excluded); + return kept; + } + + private static List only(final String... segments) { + return Arrays.asList(segments); + } + } + + static Stream conditionsInBothOrders() { + return Arrays.stream(Condition.values()).flatMap(condition -> Stream.of( + Arguments.of(condition, QueryOrder.BY_START_TIME), + Arguments.of(condition, QueryOrder.BY_DURATION))); + } + + /** The arguments of a trace search besides the range, the order and the page. */ + private static final class Search { + private long minLatency; + private long maxLatency; + private String serviceId; + private String instanceId; + private String endpointId; + private String traceId; + private TraceState state = TraceState.ALL; + private List tags; + } + + private List search(final Consumer criteria, final QueryOrder order, final int from, + final int limit) throws IOException { + return search(AROUND_MIDNIGHT, criteria, order, from, limit); + } + + private List search(final Duration range, final Consumer criteria, final QueryOrder order, + final int from, final int limit) throws IOException { + final Search search = new Search(); + criteria.accept(search); + final List found = new ArrayList<>(); + for (final BasicTrace trace : dao.queryBasicTraces( + range, search.minLatency, search.maxLatency, search.serviceId, search.instanceId, + search.endpointId, search.traceId, limit, from, search.state, order, search.tags).getTraces()) { + found.add(render(trace)); + } + return found; + } + + /** The segments as the search lists them, in the order asked for. */ + private static List expected(final List segments, final QueryOrder order) { + final Comparator latestFirst = Comparator.comparingLong(segment -> -segment.start); + final Comparator slowestFirst = Comparator.comparingInt(segment -> -segment.latency); + return segments.stream() + .map(SEGMENTS::get) + .sorted(order == QueryOrder.BY_DURATION ? slowestFirst : latestFirst) + .map(StoredSegment::listed) + .collect(Collectors.toList()); + } + + private static String render(final BasicTrace trace) { + return trace.getSegmentId() + " of " + trace.getTraceIds() + " at " + trace.getEndpointNames() + + ", start " + trace.getStart() + ", " + trace.getDuration() + " ms, error " + trace.isError(); + } + + /** Every column of the segment the trace and profiling queries use. */ + private static String render(final SegmentRecord segment) { + return segment.getSegmentId() + " of " + segment.getTraceId() + " on " + segment.getServiceId() + " / " + + segment.getServiceInstanceId() + " / " + segment.getEndpointId() + ", start " + segment.getStartTime() + + ", " + segment.getLatency() + " ms, error " + segment.getIsError() + ", " + + Base64.getEncoder().encodeToString(segment.getDataBinary()); + } + + private static List records(final String... segments) { + return Arrays.stream(segments).sorted().map(id -> render(SEGMENTS.get(id).record())) + .collect(Collectors.toList()); + } + + private static List sorted(final List segments) { + return segments.stream() + .sorted(Comparator.comparing(SegmentRecord::getSegmentId)) + .map(JDBCTraceQueryITBase::render) + .collect(Collectors.toList()); + } + + private static Map fixture() { + final Map segments = new LinkedHashMap<>(); + add(segments, new StoredSegment(FIRST_MATCH, START, MIN_LATENCY).trace(TRACE)); + add(segments, new StoredSegment(OTHER_SERVICE, MIDNIGHT - 50_000L, 150).service(CANARY)); + add(segments, new StoredSegment(OTHER_INSTANCE_SEGMENT, MIDNIGHT - 45_000L, 200).instance(OTHER_INSTANCE)); + add(segments, new StoredSegment(OTHER_ENDPOINT_SEGMENT, MIDNIGHT - 40_000L, 250).endpoint(OTHER_ENDPOINT)); + add(segments, new StoredSegment(BELOW_MIN_LATENCY, MIDNIGHT - 30_000L, MIN_LATENCY - 1)); + add(segments, new StoredSegment(ABOVE_MAX_LATENCY, MIDNIGHT - 20_000L, MAX_LATENCY + 1)); + add(segments, new StoredSegment(SUCCEEDED, MIDNIGHT - 10_000L, 300).succeeded()); + add(segments, new StoredSegment(OTHER_TAG_VALUE, MIDNIGHT + 10_000L, 350) + .tags(METHOD, new Tag(STATUS.getKey(), "200"))); + add(segments, new StoredSegment(OTHER_TRACE_SEGMENT, MIDNIGHT + 20_000L, 400).trace(OTHER_TRACE)); + add(segments, new StoredSegment(LAST_MATCH, END, MAX_LATENCY).trace(TRACE)); + add(segments, new StoredSegment(BEFORE_START, START - 1, 500)); + add(segments, new StoredSegment(AFTER_END, END + 1, 550)); + add(segments, new StoredSegment(ON_PREVIOUS_MIDNIGHT, PREVIOUS_MIDNIGHT, 600)); + add(segments, new StoredSegment(DOWNSTREAM, END + 1_000L, 50) + .trace(TRACE).service(INVENTORY).instance("inventory-1").endpoint("GET:/stock").tags()); + return segments; + } + + private static void add(final Map segments, final StoredSegment segment) { + segments.put(segment.id, segment); + } + + /** A segment of its own trace matching every condition unless told otherwise. */ + private static final class StoredSegment { + private final String id; + private final long start; + private final int latency; + private String traceId; + private String service = CHECKOUT; + private String instance = CHECKOUT_1; + private String endpoint = POST_ORDERS; + private boolean error = true; + private List tags = Arrays.asList(METHOD, STATUS); + + private StoredSegment(final String id, final long start, final int latency) { + this.id = id; + this.start = start; + this.latency = latency; + this.traceId = traceOf(id); + } + + private StoredSegment trace(final String traceId) { + this.traceId = traceId; + return this; + } + + private StoredSegment service(final String service) { + this.service = service; + return this; + } + + private StoredSegment instance(final String instance) { + this.instance = instance; + return this; + } + + private StoredSegment endpoint(final String endpoint) { + this.endpoint = endpoint; + return this; + } + + private StoredSegment succeeded() { + this.error = false; + return this; + } + + private StoredSegment tags(final Tag... tags) { + this.tags = Arrays.asList(tags); + return this; + } + + /** The record {@code SegmentAnalysisListener} stores for the segment, with its searchable tags. */ + private SegmentRecord record() { + final SegmentObject segment = SegmentObject.newBuilder() + .setTraceId(traceId) + .setTraceSegmentId(id) + .setService(service) + .setServiceInstance(instance) + .addSpans(SpanObject.newBuilder() + .setSpanId(0) + .setParentSpanId(-1) + .setStartTime(start) + .setEndTime(start + latency) + .setOperationName(endpoint) + .setSpanType(SpanType.Entry) + .setIsError(error)) + .build(); + final SegmentRecord record = new SegmentRecord(); + record.setSegmentId(id); + record.setTraceId(traceId); + record.setServiceId(serviceId(service)); + record.setServiceInstanceId(instanceId(service, instance)); + record.setEndpointId(endpointId(service, endpoint)); + record.setStartTime(start); + record.setLatency(latency); + record.setIsError(error ? 1 : 0); + record.setDataBinary(segment.toByteArray()); + record.setTags(tags.stream().map(Tag::toString).collect(Collectors.toList())); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(start)); + return record; + } + + /** The segment as the trace search lists it. */ + private String listed() { + final BasicTrace trace = new BasicTrace(); + trace.setSegmentId(id); + trace.getTraceIds().add(traceId); + trace.getEndpointNames().add(endpoint); + trace.setStart(String.valueOf(start)); + trace.setDuration(latency); + trace.setError(error); + return render(trace); + } + } + + private static String traceOf(final String segment) { + return "trace-of-" + segment; + } + + private static String serviceId(final String service) { + return IDManager.ServiceID.buildId(service, true); + } + + private static String instanceId(final String service, final String instance) { + return IDManager.ServiceInstanceID.buildId(serviceId(service), instance); + } + + private static String endpointId(final String service, final String endpoint) { + return IDManager.EndpointID.buildId(serviceId(service), endpoint); + } + + private static Duration duration(final long start, final long end) { + return JDBCQueryITSupport.duration(start, end, Step.SECOND); + } + + /** The trace DAO reads the tag keys a search may use from the core module's config. */ + private static ModuleManager searchableTags(final String keys) { + final ConfigService config = mock(ConfigService.class); + when(config.getSearchableTracesTags()).thenReturn(new SearchableTracesTagsWatcher(keys, null)); + final ModuleManager moduleManager = mock(ModuleManager.class); + final ModuleProviderHolder provider = mock(ModuleProviderHolder.class); + final ModuleServiceHolder services = mock(ModuleServiceHolder.class); + when(moduleManager.find(CoreModule.NAME)).thenReturn(provider); + when(provider.provider()).thenReturn(services); + when(services.getService(ConfigService.class)).thenReturn(config); + return moduleManager; } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryDAOTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryDAOTest.java index ec2111e80ec2..1a097fbb45aa 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryDAOTest.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryDAOTest.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; +import org.apache.skywalking.oap.server.core.query.enumeration.Step; +import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.zipkin.ZipkinSpanRecord; import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCClient; import org.apache.skywalking.oap.server.storage.plugin.jdbc.common.JDBCTableInstaller; @@ -29,11 +31,13 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import zipkin2.storage.QueryRequest; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -84,6 +88,48 @@ void getTraces_shouldUseInClauseForMultipleTraceIds() throws Exception { .contains("abc123", "def456", "ghi789"); } + @Test + void getTraces_shouldRankTheTracesOfEveryTableInOneQuery() throws Exception { + final Duration duration = new Duration(); + duration.setStart("2026-09-17 235900"); + duration.setEnd("2026-09-18 000059"); + duration.setStep(Step.SECOND); + when(tableHelper.getTablesBetweenTimestamps( + ZipkinSpanRecord.INDEX_NAME, duration.getStartTimestamp(), duration.getEndTimestamp())) + .thenReturn(Arrays.asList("zipkin_span_20260917", "zipkin_span_20260918")); + final List capturedSql = new ArrayList<>(); + final List capturedParams = new ArrayList<>(); + doAnswer(invocation -> { + capturedSql.add(invocation.getArgument(0)); + final Object[] allArgs = invocation.getArguments(); + capturedParams.add(Arrays.copyOfRange(allArgs, 2, allArgs.length)); + return null; + }).when(jdbcClient).executeQuery(anyString(), any(), any(Object[].class)); + + dao.getTraces(QueryRequest.newBuilder() + .serviceName("backend") + .parseAnnotationQuery("error") + .endTs(duration.getEndTimestamp()) + .lookback(120_000L) + .limit(10) + .build(), duration); + + // each table ranks its traces by their first match, joining its own query table; one query ranks them all + assertThat(capturedSql).hasSize(1); + assertThat(capturedSql.get(0)) + .startsWith("select trace_id, min(rank_value) as rank_value from (select zipkin_span_20260917.trace_id, " + + "min(timestamp_millis) as rank_value from zipkin_span_20260917 inner join " + + "zipkin_query_20260917 zipkin_query_202609170 on ") + .contains(" group by zipkin_span_20260917.trace_id union all select zipkin_span_20260918.trace_id, " + + "min(timestamp_millis) as rank_value from zipkin_span_20260918 inner join " + + "zipkin_query_20260918 zipkin_query_202609180 on ") + .endsWith(" group by zipkin_span_20260918.trace_id) ranked group by trace_id" + + " order by min(rank_value) desc, trace_id limit 10"); + assertThat(capturedParams.get(0)).containsExactly( + duration.getStartTimestamp(), duration.getEndTimestamp(), "backend", "error", + duration.getStartTimestamp(), duration.getEndTimestamp(), "backend", "error"); + } + @Test void getTraces_shouldReturnEmptyListWhenTraceIdsEmpty() throws Exception { final Set traceIds = Collections.emptySet(); diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryITBase.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryITBase.java index 99c2d0c9279d..1c23689f4223 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryITBase.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/JDBCZipkinQueryITBase.java @@ -21,48 +21,140 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; -import java.util.TreeSet; +import java.util.function.Consumer; import org.apache.skywalking.oap.server.core.analysis.DownSampling; import org.apache.skywalking.oap.server.core.analysis.TimeBucket; import org.apache.skywalking.oap.server.core.query.enumeration.Step; import org.apache.skywalking.oap.server.core.query.input.Duration; import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.zipkin.ZipkinServiceRelationTraffic; +import org.apache.skywalking.oap.server.core.zipkin.ZipkinServiceSpanTraffic; +import org.apache.skywalking.oap.server.core.zipkin.ZipkinServiceTraffic; import org.apache.skywalking.oap.server.core.zipkin.ZipkinSpanRecord; +import org.joda.time.DateTime; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.ValueSource; import org.testcontainers.containers.JdbcDatabaseContainer; +import zipkin2.Endpoint; import zipkin2.Span; import zipkin2.storage.QueryRequest; import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.JDBCQueryITSupport.MIDNIGHT; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Searches Zipkin traces the way the Zipkin query API does, on spans written on both sides of midnight, so in two - * day tables. A search ranks the traces by their first span that matches, latest first, and keeps {@code limit} of - * them; a trace comes back whole, every span of it, whichever day table holds it. A subclass names the server. + * Reads Zipkin data the way the Zipkin query API does, through {@link JDBCZipkinQueryDAO}, from what OAP's own write + * path stored in PostgreSQL or MariaDB. A subclass names the server. + * + *

The search range spans a midnight, so it reads two day tables, and an annotation query joins the spans' query + * table of each. Each search criterion is tried alone and with all the others. For every criterion a near-miss trace + * matches all the others but that one, the matching traces lie on the bounds of the time range and of the duration, + * and a search asks for exactly as many traces as should match, so a near-miss that leaks in pushes a match out. A + * trace found by one span must come back with all of them, from both day tables. + * + *

The Zipkin API promises no order of the traces it returns (zipkin2 {@code SpanStore#getTraces}, + * {@code Traces#getTraces}) and the query handler passes the DAO's order through, so traces are compared regardless + * of order. Which traces a limit keeps is promised: the ones nearest the end of the range (zipkin2 + * {@code QueryRequest}), by their first span that matches. The catalogs' names were first seen on either side of + * midnight, so their rows are in two day tables. */ @TestInstance(TestInstance.Lifecycle.PER_CLASS) abstract class JDBCZipkinQueryITBase { + /** The main search range: a minute either side of midnight. */ + private static final Range AROUND_MIDNIGHT = Range.ending(MIDNIGHT + 59_000L, 119_000L); + private static final long START = AROUND_MIDNIGHT.duration.getStartTimestamp(); + private static final long END = AROUND_MIDNIGHT.duration.getEndTimestamp(); + /** Ten to five minutes before midnight, apart from the main range. */ + private static final Range BEFORE_MIDNIGHT = Range.ending(MIDNIGHT - 301_000L, 299_000L); + private static final String BACKEND = "backend"; + private static final String CANARY = "backend-canary"; private static final String FRONTEND = "frontend"; - /** The backend's traces, latest first by their first backend span. */ - private static final List LATEST_FIRST = Arrays.asList( - "a02", "b07", "b06", "b05", "b04", "b03", "b02", "a01"); + private static final String SCHEDULER = "scheduler"; + private static final String MYSQL = "mysql"; + private static final String REDIS = "redis"; + private static final String KAFKA = "kafka"; + private static final String POSTGRES = "postgres"; + private static final String SELECT_ORDERS = "select orders"; + private static final String SELECT_USERS = "select users"; + private static final String PUBLISH_ORDERS = "publish orders"; + private static final String GET_ORDERS = "get /orders"; + private static final String RUN_JOB = "run job"; + private static final String LOAD_BATCH = "load batch"; + + /** The start of yesterday, the end instant of a range ending in the last second of the day before. */ + private static final long YESTERDAY = new DateTime(MIDNIGHT).minusDays(1).getMillis(); + private static final String NIGHTLY = "nightly"; + /** A span of the nightly service on yesterday's first instant, in yesterday's table. */ + private static final Span ON_THE_END_INSTANT = span( + "08603e4bc22d2e110000000000000d01", "000000000000d011", null, Span.Kind.SERVER, RUN_JOB, NIGHTLY, YESTERDAY) + .duration(1_000L).build(); + /** A millisecond after it. */ + private static final Span AFTER_THE_END_INSTANT = span( + "08603e4bc22d2e110000000000000d02", "000000000000d021", null, Span.Kind.SERVER, RUN_JOB, NIGHTLY, + YESTERDAY + 1).duration(1_000L).build(); + + private static final long MIN_DURATION_US = 1_000L; + private static final long MAX_DURATION_US = 2_000L; + + /** Found by a span on the range's start, whose child is written after midnight. */ + private static final String FIRST_MATCH = "08603e4bc22d2e110000000000000a01"; + /** Found by a span on the range's end, whose parent is written before midnight. */ + private static final String LAST_MATCH = "08603e4bc22d2e110000000000000a02"; + private static final String OTHER_SERVICE = "08603e4bc22d2e110000000000000b01"; + private static final String OTHER_REMOTE_SERVICE = "08603e4bc22d2e110000000000000b02"; + private static final String OTHER_SPAN_NAME = "08603e4bc22d2e110000000000000b03"; + private static final String OTHER_TAG_VALUE = "08603e4bc22d2e110000000000000b04"; + /** Has "error" as a tag value, not as a tag key or an annotation. */ + private static final String NO_ERROR_KEY = "08603e4bc22d2e110000000000000b05"; + private static final String BELOW_MIN_DURATION = "08603e4bc22d2e110000000000000b06"; + private static final String ABOVE_MAX_DURATION = "08603e4bc22d2e110000000000000b07"; + private static final String BEFORE_START = "08603e4bc22d2e110000000000000b08"; + private static final String AFTER_END = "08603e4bc22d2e110000000000000b09"; + /** The scheduler's traces, each of two spans: the later a trace starts, the earlier it ends. */ + private static final String STARTED_FIRST = "08603e4bc22d2e110000000000000c01"; + private static final String STARTED_SECOND = "08603e4bc22d2e110000000000000c02"; + private static final String STARTED_LAST = "08603e4bc22d2e110000000000000c03"; + private static final String UNKNOWN = "08603e4bc22d2e110000000000000fff"; + + private static final List IN_MAIN_RANGE = Arrays.asList( + FIRST_MATCH, LAST_MATCH, OTHER_SERVICE, OTHER_REMOTE_SERVICE, OTHER_SPAN_NAME, OTHER_TAG_VALUE, NO_ERROR_KEY, + BELOW_MIN_DURATION, ABOVE_MAX_DURATION); + private static final Map> TRACES = fixture(); + /** Neither in time order nor against it, and each trace's spans child first. */ + private static final List WRITE_ORDER = Arrays.asList( + STARTED_FIRST, AFTER_END, LAST_MATCH, OTHER_SPAN_NAME, STARTED_LAST, BELOW_MIN_DURATION, FIRST_MATCH, + BEFORE_START, OTHER_TAG_VALUE, OTHER_REMOTE_SERVICE, STARTED_SECOND, ABOVE_MAX_DURATION, OTHER_SERVICE, + NO_ERROR_KEY); + + private static final List SERVICES = Arrays.asList(BACKEND, CANARY, FRONTEND, SCHEDULER); + private static final Map> REMOTE_SERVICES = Map.of( + BACKEND, Arrays.asList(KAFKA, MYSQL, REDIS), + CANARY, Collections.singletonList(MYSQL), + FRONTEND, Collections.emptyList(), + SCHEDULER, Collections.singletonList(POSTGRES)); + private static final Map> SPAN_NAMES = Map.of( + BACKEND, Arrays.asList(PUBLISH_ORDERS, SELECT_ORDERS, SELECT_USERS), + CANARY, Collections.singletonList(SELECT_ORDERS), + FRONTEND, Collections.singletonList(GET_ORDERS), + SCHEDULER, Arrays.asList(LOAD_BATCH, RUN_JOB)); private final JdbcDatabaseContainer server; private final JDBCQueryITSupport.Target target; private JDBCQueryITSupport support; private JDBCZipkinQueryDAO dao; - private Duration duration; JDBCZipkinQueryITBase(final JdbcDatabaseContainer server, final JDBCQueryITSupport.Target target) { this.server = server; @@ -72,24 +164,34 @@ abstract class JDBCZipkinQueryITBase { @BeforeAll public void writeSpans() throws Exception { support = JDBCQueryITSupport.connect(server, target); - final var model = support.install( + final Model spans = support.install( ZipkinSpanRecord.class, DefaultScopeDefine.ZIPKIN_SPAN, ZipkinSpanRecord.INDEX_NAME, DownSampling.Second); - support.writeRecords(model, new ZipkinSpanRecord.Builder(), Arrays.asList( - // a01 starts in the backend before midnight; its child, after midnight, is the backend too. - span("a01", 1, 0, BACKEND, -60), - span("a01", 2, 1, BACKEND, 1), - // a02 starts in the frontend before midnight; only its backend child, after midnight, matches. - span("a02", 1, 0, FRONTEND, -1), - span("a02", 2, 1, BACKEND, 60), - span("b02", 1, 0, BACKEND, -40), - span("b03", 1, 0, BACKEND, -30), - span("b04", 1, 0, BACKEND, -20), - span("b05", 1, 0, BACKEND, 10), - span("b06", 1, 0, BACKEND, 20), - span("b07", 1, 0, BACKEND, 30) - )); + final Model services = support.install( + ZipkinServiceTraffic.class, DefaultScopeDefine.ZIPKIN_SERVICE, ZipkinServiceTraffic.INDEX_NAME, + DownSampling.Minute); + final Model remoteServices = support.install( + ZipkinServiceRelationTraffic.class, DefaultScopeDefine.ZIPKIN_SERVICE_RELATION, + ZipkinServiceRelationTraffic.INDEX_NAME, DownSampling.Minute); + final Model spanNames = support.install( + ZipkinServiceSpanTraffic.class, DefaultScopeDefine.ZIPKIN_SERVICE_SPAN, ZipkinServiceSpanTraffic.INDEX_NAME, + DownSampling.Minute); + + final List records = new ArrayList<>(); + for (final String traceId : WRITE_ORDER) { + final List trace = new ArrayList<>(TRACES.get(traceId)); + Collections.reverse(trace); + for (final Span span : trace) { + records.add(ZipkinSpanRecords.of(span)); + } + } + support.writeRecords(spans, new ZipkinSpanRecord.Builder(), records); + support.writeRecords(spans, new ZipkinSpanRecord.Builder(), Arrays.asList( + ZipkinSpanRecords.of(ON_THE_END_INSTANT), ZipkinSpanRecords.of(AFTER_THE_END_INSTANT))); + support.writeMetrics(services, new ZipkinServiceTraffic.Builder(), serviceTraffic()); + support.writeMetrics(remoteServices, new ZipkinServiceRelationTraffic.Builder(), relationTraffic()); + support.writeMetrics(spanNames, new ZipkinServiceSpanTraffic.Builder(), spanTraffic()); + dao = new JDBCZipkinQueryDAO(support.getClient(), support.getTableHelper()); - duration = JDBCQueryITSupport.duration(MIDNIGHT - 120_000L, MIDNIGHT + 120_000L, Step.SECOND); } @AfterAll @@ -99,73 +201,329 @@ public void close() { } } - @ParameterizedTest(name = "limit {0}") - @ValueSource(ints = {4, 5, 8}) - public void keepsTheLatestTracesAcrossMidnight(final int limit) throws Exception { - final Set> expected = new HashSet<>(); - for (final String trace : LATEST_FIRST.subList(0, limit)) { - expected.add(spanIds(trace)); + @Test + public void listsEveryServiceThatReportedSpans() { + assertEquals(SERVICES, sorted(dao.getServiceNames())); + } + + @Test + public void listsTheRemoteServicesEachServiceCalled() { + for (final String service : SERVICES) { + assertEquals(REMOTE_SERVICES.get(service), sorted(dao.getRemoteServiceNames(service)), service); + } + assertEquals(Collections.emptyList(), dao.getRemoteServiceNames("unknown")); + } + + @Test + public void listsTheSpanNamesOfEachService() { + for (final String service : SERVICES) { + assertEquals(SPAN_NAMES.get(service), sorted(dao.getSpanNames(service)), service); } - final QueryRequest request = QueryRequest.newBuilder() - .serviceName(BACKEND) - .endTs(MIDNIGHT + 120_000L) - .lookback(240_000L) - .limit(limit) - .build(); - assertEquals(expected, traces(dao.getTraces(request, duration))); + assertEquals(Collections.emptyList(), dao.getSpanNames("unknown")); } @Test - public void readsWholeTracesByIdAcrossMidnight() throws Exception { - final Set traceIds = new HashSet<>(Arrays.asList(traceId("a01"), traceId("a02"), traceId("b05"))); - assertEquals(new HashSet<>(Arrays.asList(spanIds("a01"), spanIds("a02"), spanIds("b05"))), - traces(dao.getTraces(traceIds, duration))); + public void readsAWholeTraceById() { + assertEquals(sortedById(TRACES.get(FIRST_MATCH)), sortedById(dao.getTrace(FIRST_MATCH, null))); + assertEquals(sortedById(TRACES.get(LAST_MATCH)), + sortedById(dao.getTrace(LAST_MATCH, AROUND_MIDNIGHT.duration))); + assertEquals(Collections.emptyList(), dao.getTrace(UNKNOWN, null)); + } + + @Test + public void readsWholeTracesByIds() { + final Set ids = new HashSet<>(Arrays.asList(FIRST_MATCH, LAST_MATCH, STARTED_LAST, UNKNOWN)); + assertEquals(traces(FIRST_MATCH, LAST_MATCH, STARTED_LAST), canonical(dao.getTraces(ids, null))); + } + + @ParameterizedTest(name = "{0}") + @EnumSource(Criterion.class) + public void findsEveryTraceMatchingOneCriterion(final Criterion criterion) { + final List matching = new ArrayList<>(IN_MAIN_RANGE); + matching.remove(criterion.nearMiss); + assertEquals(traces(matching.toArray(new String[0])), + canonical(search(AROUND_MIDNIGHT, criterion.criterion, matching.size()))); } /** - * Each trace as its sorted span ids; a trace split into two lists shows up as two entries. + * The two traces matching every criterion lie on the bounds: one has a span on the range's start lasting the + * minimum duration, the other a span on the range's end lasting the maximum. Neither has another span that + * matches. */ - private static Set> traces(final List> traces) { - final Set> result = new HashSet<>(); - for (final List trace : traces) { - final Set ids = new TreeSet<>(); - for (final Span span : trace) { - ids.add(span.id()); + @Test + public void findsOnlyTheTracesMatchingEveryCriterion() { + assertEquals(traces(FIRST_MATCH, LAST_MATCH), canonical(search(AROUND_MIDNIGHT, request -> request + .serviceName(BACKEND) + .remoteServiceName(MYSQL) + .spanName(SELECT_ORDERS) + .parseAnnotationQuery("error and db.instance=orders") + .minDuration(MIN_DURATION_US) + .maxDuration(MAX_DURATION_US), 2))); + } + + @Test + public void findsNoTraceWhenNoSpanMatches() { + assertEquals(Collections.emptyList(), search(AROUND_MIDNIGHT, request -> request.serviceName("unknown"), 10)); + } + + @ParameterizedTest(name = "limit {0}") + @ValueSource(ints = {1, 2, 3}) + public void keepsTheTracesThatStartedLast(final int limit) { + final List latestStartFirst = Arrays.asList(STARTED_LAST, STARTED_SECOND, STARTED_FIRST); + assertEquals(traces(latestStartFirst.subList(0, limit).toArray(new String[0])), + canonical(search(BEFORE_MIDNIGHT, request -> request.serviceName(SCHEDULER), limit))); + } + + /** + * The first matching trace's span after midnight ranks it among the latest backend traces of that day, but the + * trace ranks by its first backend span, before midnight, so it is the last of them. + */ + @ParameterizedTest(name = "limit {0}") + @ValueSource(ints = {1, 2, 3, 4, 5, 6, 7, 8}) + public void keepsTheTracesNearestTheEndOfTheRangeAcrossMidnight(final int limit) { + final List latestFirst = Arrays.asList( + LAST_MATCH, ABOVE_MAX_DURATION, BELOW_MIN_DURATION, NO_ERROR_KEY, OTHER_TAG_VALUE, OTHER_SPAN_NAME, + OTHER_REMOTE_SERVICE, FIRST_MATCH); + assertEquals(traces(latestFirst.subList(0, limit).toArray(new String[0])), + canonical(search(AROUND_MIDNIGHT, request -> request.serviceName(BACKEND), limit))); + } + + /** + * The Zipkin API's range is read through its end instant, the second after its last one, which for a range ending + * in a day's last second is the next day's first instant, in the next day table. + */ + @Test + public void findsATraceOnTheEndInstantOfARangeEndingInADaysLastSecond() { + assertEquals(canonical(Collections.singletonList(Collections.singletonList(ON_THE_END_INSTANT))), + canonical(search(Range.ending(YESTERDAY - 1_000L, 60_000L), + request -> request.serviceName(NIGHTLY), 10))); + } + + /** What a search can filter by, each with the trace in the main range that matches every other criterion. */ + enum Criterion { + TIME_RANGE_ONLY(request -> { }, null), + SERVICE_NAME(request -> request.serviceName(BACKEND), OTHER_SERVICE), + REMOTE_SERVICE_NAME(request -> request.remoteServiceName(MYSQL), OTHER_REMOTE_SERVICE), + SPAN_NAME(request -> request.spanName(SELECT_ORDERS), OTHER_SPAN_NAME), + ANNOTATION_KEY(request -> request.parseAnnotationQuery("error"), NO_ERROR_KEY), + ANNOTATION_KEY_AND_VALUE(request -> request.parseAnnotationQuery("db.instance=orders"), OTHER_TAG_VALUE), + MIN_DURATION(request -> request.minDuration(MIN_DURATION_US), BELOW_MIN_DURATION), + MAX_DURATION(request -> request.minDuration(1L).maxDuration(MAX_DURATION_US), ABOVE_MAX_DURATION); + + private final Consumer criterion; + private final String nearMiss; + + Criterion(final Consumer criterion, final String nearMiss) { + this.criterion = criterion; + this.nearMiss = nearMiss; + } + } + + private List> search(final Range range, final Consumer criteria, + final int limit) { + final QueryRequest.Builder request = QueryRequest.newBuilder() + .endTs(range.endTs) + .lookback(range.lookback) + .limit(limit); + criteria.accept(request); + return dao.getTraces(request.build(), range.duration); + } + + private static Map> fixture() { + final Map> traces = new LinkedHashMap<>(); + traces.put(FIRST_MATCH, Arrays.asList( + matching(FIRST_MATCH, "000000000000a011", START).duration(MIN_DURATION_US).build(), + span(FIRST_MATCH, "000000000000a012", "000000000000a011", Span.Kind.PRODUCER, PUBLISH_ORDERS, BACKEND, + MIDNIGHT + 40_000L).remoteEndpoint(endpoint(KAFKA)).duration(5_000L).debug(true).build() + )); + traces.put(LAST_MATCH, Arrays.asList( + span(LAST_MATCH, "000000000000a021", null, Span.Kind.SERVER, GET_ORDERS, FRONTEND, MIDNIGHT - 1_000L) + .duration(62_000_000L).shared(true).build(), + matching(LAST_MATCH, "000000000000a022", END) + .parentId("000000000000a021") + .duration(MAX_DURATION_US) + .clearTags() + .putTag("db.instance", "orders") + .addAnnotation(END * 1000L, "error") + .build() + )); + traces.put(OTHER_SERVICE, Collections.singletonList( + matching(OTHER_SERVICE, "000000000000b011", MIDNIGHT - 50_000L).localEndpoint(endpoint(CANARY)).build())); + traces.put(OTHER_REMOTE_SERVICE, Collections.singletonList( + matching(OTHER_REMOTE_SERVICE, "000000000000b021", MIDNIGHT - 40_000L) + .remoteEndpoint(endpoint(REDIS)).build())); + traces.put(OTHER_SPAN_NAME, Collections.singletonList( + matching(OTHER_SPAN_NAME, "000000000000b031", MIDNIGHT - 30_000L).name(SELECT_USERS).build())); + traces.put(OTHER_TAG_VALUE, Collections.singletonList( + matching(OTHER_TAG_VALUE, "000000000000b041", MIDNIGHT - 20_000L).putTag("db.instance", "users").build())); + traces.put(NO_ERROR_KEY, Collections.singletonList( + matching(NO_ERROR_KEY, "000000000000b051", MIDNIGHT + 10_000L) + .clearTags() + .putTag("db.instance", "orders") + .putTag("otel.status_code", "error") + .build())); + traces.put(BELOW_MIN_DURATION, Collections.singletonList( + matching(BELOW_MIN_DURATION, "000000000000b061", MIDNIGHT + 20_000L) + .duration(MIN_DURATION_US - 1).build())); + traces.put(ABOVE_MAX_DURATION, Collections.singletonList( + matching(ABOVE_MAX_DURATION, "000000000000b071", MIDNIGHT + 30_000L) + .duration(MAX_DURATION_US + 1).build())); + traces.put(BEFORE_START, Collections.singletonList( + matching(BEFORE_START, "000000000000b081", START - 1).build())); + traces.put(AFTER_END, Collections.singletonList( + matching(AFTER_END, "000000000000b091", END + 1).build())); + traces.put(STARTED_FIRST, scheduled(STARTED_FIRST, "000000000000c011", MIDNIGHT - 540_000L, + MIDNIGHT - 310_000L)); + traces.put(STARTED_SECOND, scheduled(STARTED_SECOND, "000000000000c021", MIDNIGHT - 500_000L, + MIDNIGHT - 350_000L)); + traces.put(STARTED_LAST, scheduled(STARTED_LAST, "000000000000c031", MIDNIGHT - 460_000L, + MIDNIGHT - 390_000L)); + return traces; + } + + /** A backend span matching every criterion of the search in the main range. */ + private static Span.Builder matching(final String traceId, final String spanId, final long timestampMillis) { + return span(traceId, spanId, null, Span.Kind.CLIENT, SELECT_ORDERS, BACKEND, timestampMillis) + .localEndpoint(Endpoint.newBuilder().serviceName(BACKEND).ip("10.0.0.1").port(8080).build()) + .remoteEndpoint(Endpoint.newBuilder().serviceName(MYSQL).ip("2001:db8::1").port(3306).build()) + .duration(1_500L) + .putTag("db.instance", "orders") + .putTag("error", "deadlock"); + } + + /** A scheduler job and the batch it loads, both matching a search by the scheduler's name. */ + private static List scheduled(final String traceId, final String rootSpanId, final long startMillis, + final long loadMillis) { + final String loadSpanId = rootSpanId.substring(0, 15) + "2"; + return Arrays.asList( + span(traceId, rootSpanId, null, Span.Kind.SERVER, RUN_JOB, SCHEDULER, startMillis) + .duration((loadMillis - startMillis + 1_000L) * 1000L).build(), + span(traceId, loadSpanId, rootSpanId, Span.Kind.CLIENT, LOAD_BATCH, SCHEDULER, loadMillis) + .remoteEndpoint(endpoint(POSTGRES)).duration(1_000_000L).build() + ); + } + + private static Span.Builder span(final String traceId, final String spanId, final String parentId, + final Span.Kind kind, final String name, final String service, + final long timestampMillis) { + return Span.newBuilder() + .traceId(traceId) + .id(spanId) + .parentId(parentId) + .kind(kind) + .name(name) + .localEndpoint(endpoint(service)) + .timestamp(timestampMillis * 1000L) + .debug(false) + .shared(false); + } + + private static Endpoint endpoint(final String service) { + return Endpoint.newBuilder().serviceName(service).build(); + } + + /** + * What the Zipkin receiver reports for every span — its service, its span name and the remote service it + * called — each kept once, as the metrics workers merge them, in the minute it was first seen. + */ + private static List serviceTraffic() { + final Map traffic = new LinkedHashMap<>(); + for (final Span span : spansByTime()) { + final ZipkinServiceTraffic service = new ZipkinServiceTraffic(); + service.setServiceName(span.localServiceName()); + service.setTimeBucket(minuteOf(span)); + traffic.putIfAbsent(span.localServiceName(), service); + } + return new ArrayList<>(traffic.values()); + } + + private static List relationTraffic() { + final Map traffic = new LinkedHashMap<>(); + for (final Span span : spansByTime()) { + if (span.remoteServiceName() == null) { + continue; } - assertTrue(result.add(new ArrayList<>(ids)), "trace returned twice: " + ids); + final ZipkinServiceRelationTraffic relation = new ZipkinServiceRelationTraffic(); + relation.setServiceName(span.localServiceName()); + relation.setRemoteServiceName(span.remoteServiceName()); + relation.setTimeBucket(minuteOf(span)); + traffic.putIfAbsent(span.localServiceName() + "/" + span.remoteServiceName(), relation); + } + return new ArrayList<>(traffic.values()); + } + + private static List spanTraffic() { + final Map traffic = new LinkedHashMap<>(); + for (final Span span : spansByTime()) { + final ZipkinServiceSpanTraffic spanName = new ZipkinServiceSpanTraffic(); + spanName.setServiceName(span.localServiceName()); + spanName.setSpanName(span.name()); + spanName.setTimeBucket(minuteOf(span)); + traffic.putIfAbsent(span.localServiceName() + "/" + span.name(), spanName); + } + return new ArrayList<>(traffic.values()); + } + + private static List spansByTime() { + final List spans = new ArrayList<>(); + TRACES.values().forEach(spans::addAll); + spans.sort(Comparator.comparingLong(Span::timestampAsLong)); + return spans; + } + + private static long minuteOf(final Span span) { + return TimeBucket.getMinuteTimeBucket(span.timestampAsLong() / 1000); + } + + private static List> traces(final String... traceIds) { + final List> traces = new ArrayList<>(); + for (final String traceId : traceIds) { + traces.add(TRACES.get(traceId)); } - return result; + return canonical(traces); } - private static List spanIds(final String trace) { - return trace.startsWith("a") - ? Arrays.asList(spanId(trace, 1), spanId(trace, 2)) - : Collections.singletonList(spanId(trace, 1)); + /** The traces sorted by id, each trace's spans sorted by id: the Zipkin API promises neither order. */ + private static List> canonical(final List> traces) { + final List> canonical = new ArrayList<>(); + for (final List trace : traces) { + canonical.add(sortedById(trace)); + } + canonical.sort(Comparator.comparing(trace -> trace.isEmpty() ? "" : trace.get(0).traceId())); + return canonical; } - private static String traceId(final String trace) { - return String.format("%032x", Integer.parseInt(trace.substring(1), 16) + (trace.charAt(0) == 'a' ? 0xa00 : 0xb00)); + private static List sortedById(final List trace) { + final List sorted = new ArrayList<>(trace); + sorted.sort(Comparator.comparing(Span::id)); + return sorted; } - private static String spanId(final String trace, final int span) { - return traceId(trace).substring(20) + String.format("%04x", span); + private static List sorted(final List names) { + final List sorted = new ArrayList<>(names); + Collections.sort(sorted); + return sorted; } - private static ZipkinSpanRecord span(final String trace, final int span, final int parent, - final String service, final int secondsFromMidnight) { - final long timestampMillis = MIDNIGHT + secondsFromMidnight * 1000L; - final ZipkinSpanRecord record = new ZipkinSpanRecord(); - record.setTraceId(traceId(trace)); - record.setSpanId(spanId(trace, span)); - record.setParentId(parent == 0 ? null : spanId(trace, parent)); - record.setKind(parent == 0 ? "SERVER" : "CLIENT"); - record.setName("get /orders"); - record.setLocalEndpointServiceName(service); - record.setTimestamp(timestampMillis * 1000L); - record.setTimestampMillis(timestampMillis); - record.setDuration(1000L); - record.setQuery(new ArrayList<>()); - record.setTimeBucket(TimeBucket.getRecordTimeBucket(timestampMillis)); - return record; + /** A search's time range, from the Zipkin API's {@code endTs} and {@code lookback}. */ + private static final class Range { + private final long endTs; + private final long lookback; + private final Duration duration; + + private Range(final long endTs, final long lookback, final Duration duration) { + this.endTs = endTs; + this.lookback = lookback; + this.duration = duration; + } + + /** The duration {@code ZipkinQueryHandler#buildDuration} passes to the storage for this range. */ + private static Range ending(final long endTs, final long lookback) { + final Duration duration = new Duration(); + duration.setStep(Step.SECOND); + duration.setStart(new DateTime(endTs - lookback).toString("yyyy-MM-dd HHmmss")); + duration.setEnd(new DateTime(endTs).toString("yyyy-MM-dd HHmmss")); + return new Range(endTs, lookback, duration); + } } } diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/NativeTraceFixture.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/NativeTraceFixture.java new file mode 100644 index 000000000000..684ab390f86d --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/NativeTraceFixture.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import org.apache.skywalking.apm.network.common.v3.KeyIntValuePair; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.v3.Log; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentObject; +import org.apache.skywalking.apm.network.language.agent.v3.SegmentReference; +import org.apache.skywalking.apm.network.language.agent.v3.SpanAttachedEvent; +import org.apache.skywalking.apm.network.language.agent.v3.SpanLayer; +import org.apache.skywalking.apm.network.language.agent.v3.SpanObject; +import org.apache.skywalking.apm.network.language.agent.v3.SpanType; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.IDManager; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.query.type.KeyNumericValue; +import org.apache.skywalking.oap.server.core.query.type.KeyValue; +import org.apache.skywalking.oap.server.core.query.type.LogEntity; +import org.apache.skywalking.oap.server.core.query.type.Ref; +import org.apache.skywalking.oap.server.core.query.type.RefType; +import org.apache.skywalking.oap.server.core.query.type.Span; +import org.apache.skywalking.oap.server.core.query.type.Trace; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.END_SECOND; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.MIDNIGHT; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.MIDNIGHT_SECOND; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.START_SECOND; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.event; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * Two native traces and the events rover attaches to them. In the first, the consumer's exit span calls the provider, + * whose entry span refers back to it, and the provider queries a database. Both sides' rover report the HTTP call + * against the trace context it carried, the consumer's exit span: the consumer's request and response belong to that + * span, the provider's to the provider's entry span. + */ +final class NativeTraceFixture { + static final String TRACE = "3a5b7c9d0e1f.31.17368992000000001"; + static final String ONE_SEGMENT_TRACE = "3a5b7c9d0e1f.31.17368992000000002"; + /** Has only events, against the first trace's spans, and no query asks for it. */ + static final String UNQUERIED_TRACE = "3a5b7c9d0e1f.31.17368992000000003"; + /** Has only two events, on two spans of one segment, with the same name and start instant. */ + static final String TWO_SPANS_TRACE = "3a5b7c9d0e1f.31.17368992000000004"; + static final String UNKNOWN_TRACE = "3a5b7c9d0e1f.31.17368992000000009"; + private static final String CONSUMER = "3a5b7c9d0e1f.31.17368992000000011"; + private static final String PROVIDER = "4b6c8d0e1f2a.47.17368992000000012"; + private static final String WORKER = "5c7d9e1f2a3b.52.17368992000000013"; + private static final String TWO_SPANS_SEGMENT = "3a5b7c9d0e1f.31.17368992000000014"; + + static final Map COMPONENTS = Map.of( + 1, "Tomcat", 2, "JdkHttp", 33, "mysql-connector-java", 40, "kafka-producer", 41, "kafka-consumer"); + + private static final SegmentObject CONSUMER_SEGMENT = SegmentObject.newBuilder() + .setTraceId(TRACE).setTraceSegmentId(CONSUMER).setService("consumer").setServiceInstance("consumer-1") + .addSpans(SpanObject.newBuilder() + .setSpanId(0).setParentSpanId(-1) + .setStartTime(MIDNIGHT - 50_000L).setEndTime(MIDNIGHT + 50_000L) + .setOperationName("GET:/consumer").setSpanType(SpanType.Entry) + .setSpanLayer(SpanLayer.Http).setComponentId(1) + .addTags(tag("http.method", "GET"))) + .addSpans(SpanObject.newBuilder() + .setSpanId(1).setParentSpanId(0) + .setStartTime(MIDNIGHT - 40_000L).setEndTime(MIDNIGHT + 40_000L) + .setOperationName("GET:/provider").setPeer("provider:8080").setSpanType(SpanType.Exit) + .setSpanLayer(SpanLayer.Http).setComponentId(2) + .addTags(tag("url", "http://provider:8080/provider"))) + .build(); + private static final SegmentObject PROVIDER_SEGMENT = SegmentObject.newBuilder() + .setTraceId(TRACE).setTraceSegmentId(PROVIDER).setService("provider").setServiceInstance("provider-1") + .addSpans(SpanObject.newBuilder() + .setSpanId(0).setParentSpanId(-1) + .setStartTime(MIDNIGHT - 39_000L).setEndTime(MIDNIGHT + 39_000L) + .setOperationName("GET:/provider").setSpanType(SpanType.Entry) + .setSpanLayer(SpanLayer.Http).setComponentId(1) + .addRefs(SegmentReference.newBuilder() + .setRefType( + org.apache.skywalking.apm.network.language.agent.v3.RefType.CrossProcess) + .setTraceId(TRACE) + .setParentTraceSegmentId(CONSUMER) + .setParentSpanId(1) + .setParentService("consumer") + .setParentServiceInstance("consumer-1") + .setParentEndpoint("GET:/consumer") + .setNetworkAddressUsedAtPeer("provider:8080"))) + .addSpans(SpanObject.newBuilder() + .setSpanId(1).setParentSpanId(0) + .setStartTime(MIDNIGHT - 30_000L).setEndTime(MIDNIGHT + 30_000L) + .setOperationName("Mysql/JDBC/PreparedStatement/execute").setPeer("mysql:3306") + .setSpanType(SpanType.Exit).setSpanLayer(SpanLayer.Database).setComponentId(33) + .setIsError(true) + .addTags(tag("db.statement", "select * from orders where id = ?")) + .addLogs(Log.newBuilder() + .setTime(MIDNIGHT + 29_000L) + .addData(tag("event", "error")) + .addData(tag("message", "Lock wait timeout exceeded")))) + .build(); + private static final SegmentObject WORKER_SEGMENT = SegmentObject.newBuilder() + .setTraceId(ONE_SEGMENT_TRACE).setTraceSegmentId(WORKER).setService("worker").setServiceInstance("worker-1") + .addSpans(SpanObject.newBuilder() + .setSpanId(0).setParentSpanId(-1) + .setStartTime(MIDNIGHT - 20_000L).setEndTime(MIDNIGHT + 20_000L) + .setOperationName("Kafka/orders/Consumer/billing").setSpanType(SpanType.Entry) + .setSpanLayer(SpanLayer.MQ).setComponentId(41)) + .addSpans(SpanObject.newBuilder() + .setSpanId(1).setParentSpanId(0) + .setStartTime(MIDNIGHT - 10_000L).setEndTime(MIDNIGHT + 10_000L) + .setOperationName("Kafka/invoices/Producer").setPeer("kafka:9092") + .setSpanType(SpanType.Exit).setSpanLayer(SpanLayer.MQ).setComponentId(40)) + .build(); + static final List SEGMENTS = Arrays.asList(CONSUMER_SEGMENT, PROVIDER_SEGMENT, WORKER_SEGMENT); + + /** One millisecond before the start of the queries' duration. */ + static final SpanAttachedEvent CLIENT_REQUEST = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, CONSUMER, "1", START_SECOND - 1, 999_000_000, + "HTTP Request Sampling", http("outbound", "request", "client")).build(); + /** On the start of the queries' duration. */ + static final SpanAttachedEvent CLIENT_WRITE = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, CONSUMER, "1", START_SECOND, 0, + "Syscall write", "package_size", "182B", "package_count", "1") + .addSummary(KeyIntValuePair.newBuilder().setKey("package_bytes").setValue(182L)).build(); + static final SpanAttachedEvent SERVER_REQUEST = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, CONSUMER, "1", MIDNIGHT_SECOND, 5, + "HTTP Request Sampling", http("inbound", "request", "server")).build(); + static final SpanAttachedEvent DATABASE_SEND = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, PROVIDER, "1", MIDNIGHT_SECOND, 900_000_000, + "Syscall sendto", "package_size", "96B", "package_count", "1").build(); + static final SpanAttachedEvent DATABASE_REQUEST = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, PROVIDER, "1", MIDNIGHT_SECOND, 950_000_000, + "HTTP Request Sampling", http("outbound", "request", "client")).build(); + /** From the database's host, whose side has no span: it stays on the provider's exit span. */ + static final SpanAttachedEvent DATABASE_SERVER_REQUEST = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, PROVIDER, "1", MIDNIGHT_SECOND + 5, 0, + "HTTP Request Sampling", http("inbound", "request", "server")).build(); + static final SpanAttachedEvent SERVER_RESPONSE = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, CONSUMER, "1", MIDNIGHT_SECOND + 10, 0, + "HTTP Response Sampling", http("outbound", "response", "server")).build(); + static final SpanAttachedEvent UNKNOWN_SEGMENT_WRITE = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, "unknown-segment", "0", MIDNIGHT_SECOND + 20, 0, + "Syscall write", "package_size", "64B", "package_count", "1").build(); + static final SpanAttachedEvent WORKER_SEND = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, ONE_SEGMENT_TRACE, WORKER, "1", MIDNIGHT_SECOND + 30, 0, + "Syscall sendmsg", "package_size", "1KB", "package_count", "2").build(); + /** On the end of the queries' duration. */ + static final SpanAttachedEvent CLIENT_RESPONSE = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, CONSUMER, "1", END_SECOND, 0, + "HTTP Response Sampling", http("inbound", "response", "client")).build(); + /** One millisecond after the end of the queries' duration, against a span id that is not a number. */ + static final SpanAttachedEvent NOT_A_SPAN_ID_READ = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, CONSUMER, "not-a-span-id", END_SECOND, 1_000_000, + "Syscall read", "package_size", "182B", "package_count", "1").build(); + private static final SpanAttachedEvent UNQUERIED_TRACE_REQUEST = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, UNQUERIED_TRACE, CONSUMER, "1", MIDNIGHT_SECOND + 1, 0, + "HTTP Request Sampling", http("outbound", "request", "client")).build(); + /** Stored with the Zipkin events, as its trace context's type says, though it names this trace. */ + private static final SpanAttachedEvent ZIPKIN_TYPED_WRITE = event( + SpanAttachedEvent.SpanReferenceType.ZIPKIN, TRACE, "", "1", MIDNIGHT_SECOND + 2, 0, + "Syscall write", "package_size", "182B", "package_count", "1").build(); + + static final SpanAttachedEvent FIRST_SPAN_SEND = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TWO_SPANS_TRACE, TWO_SPANS_SEGMENT, "1", + MIDNIGHT_SECOND + 40, 0, "Syscall sendto", "package_size", "96B", "package_count", "1").build(); + static final SpanAttachedEvent SECOND_SPAN_SEND = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TWO_SPANS_TRACE, TWO_SPANS_SEGMENT, "2", + MIDNIGHT_SECOND + 40, 0, "Syscall sendto", "package_size", "96B", "package_count", "1").build(); + + /** Neither in start order nor against it. */ + static final List EVENTS = Arrays.asList( + DATABASE_SEND, CLIENT_RESPONSE, SECOND_SPAN_SEND, DATABASE_SERVER_REQUEST, UNQUERIED_TRACE_REQUEST, + CLIENT_REQUEST, WORKER_SEND, ZIPKIN_TYPED_WRITE, SERVER_RESPONSE, FIRST_SPAN_SEND, DATABASE_REQUEST, + NOT_A_SPAN_ID_READ, CLIENT_WRITE, UNKNOWN_SEGMENT_WRITE, SERVER_REQUEST); + static final List EVENTS_OF_THE_TRACE = Arrays.asList( + CLIENT_REQUEST, CLIENT_WRITE, SERVER_REQUEST, DATABASE_SEND, DATABASE_REQUEST, DATABASE_SERVER_REQUEST, + SERVER_RESPONSE, UNKNOWN_SEGMENT_WRITE, CLIENT_RESPONSE, NOT_A_SPAN_ID_READ); + static final List EVENTS_OF_BOTH_TRACES = Arrays.asList( + CLIENT_REQUEST, CLIENT_WRITE, SERVER_REQUEST, DATABASE_SEND, DATABASE_REQUEST, DATABASE_SERVER_REQUEST, + SERVER_RESPONSE, UNKNOWN_SEGMENT_WRITE, WORKER_SEND, CLIENT_RESPONSE, NOT_A_SPAN_ID_READ); + + private static final Gson GSON = new Gson(); + + private NativeTraceFixture() { + } + + /** + * The trace's spans, each with the events that belong to it, keyed by their segment span id. + */ + static Map expectedSpans(final String traceId) { + final List spans = TRACE.equals(traceId) + ? Arrays.asList( + expected(CONSUMER_SEGMENT, 0, CONSUMER + Const.SEGMENT_SPAN_SPLIT + -1, true), + expected(CONSUMER_SEGMENT, 1, CONSUMER + Const.SEGMENT_SPAN_SPLIT + 0, false, + CLIENT_REQUEST, CLIENT_WRITE, CLIENT_RESPONSE), + expected(PROVIDER_SEGMENT, 0, CONSUMER + Const.SEGMENT_SPAN_SPLIT + 1, false, + SERVER_REQUEST, SERVER_RESPONSE), + expected(PROVIDER_SEGMENT, 1, PROVIDER + Const.SEGMENT_SPAN_SPLIT + 0, false, + DATABASE_SEND, DATABASE_REQUEST, DATABASE_SERVER_REQUEST)) + : Arrays.asList( + expected(WORKER_SEGMENT, 0, WORKER + Const.SEGMENT_SPAN_SPLIT + -1, true), + expected(WORKER_SEGMENT, 1, WORKER + Const.SEGMENT_SPAN_SPLIT + 0, false, WORKER_SEND)); + return render(spans); + } + + /** + * Every field of every span, keyed by the segment span id: the spans' order is not what is read here. + */ + static Map render(final Trace trace) { + return render(trace.getSpans()); + } + + private static Map render(final List spans) { + final Map rendered = new TreeMap<>(); + for (final Span span : spans) { + assertNull(rendered.put(span.getSegmentSpanId(), GSON.toJsonTree(span)), span.getSegmentSpanId()); + } + return rendered; + } + + /** + * The record {@code SegmentDispatcher} stores for a segment. + */ + static SegmentRecord record(final SegmentObject segment) { + final SpanObject first = segment.getSpans(0); + final String serviceId = IDManager.ServiceID.buildId(segment.getService(), true); + final SegmentRecord record = new SegmentRecord(); + record.setSegmentId(segment.getTraceSegmentId()); + record.setTraceId(segment.getTraceId()); + record.setServiceId(serviceId); + record.setServiceInstanceId(IDManager.ServiceInstanceID.buildId(serviceId, segment.getServiceInstance())); + record.setEndpointId(IDManager.EndpointID.buildId(serviceId, first.getOperationName())); + record.setStartTime(first.getStartTime()); + record.setLatency((int) (first.getEndTime() - first.getStartTime())); + record.setIsError(segment.getSpansList().stream().anyMatch(SpanObject::getIsError) ? 1 : 0); + record.setDataBinary(segment.toByteArray()); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(first.getStartTime())); + final List tags = new ArrayList<>(); + for (final SpanObject span : segment.getSpansList()) { + for (final KeyStringValuePair tag : span.getTagsList()) { + tags.add(tag.getKey() + "=" + tag.getValue()); + } + } + record.setTags(tags); + return record; + } + + /** + * A reported span as the trace query shows it, with the events attached to it in start order. + */ + private static Span expected(final SegmentObject segment, final int index, final String segmentParentSpanId, + final boolean root, final SpanAttachedEvent... events) { + final SpanObject reported = segment.getSpans(index); + final Span span = new Span(); + span.setTraceId(segment.getTraceId()); + span.setSegmentId(segment.getTraceSegmentId()); + span.setSpanId(reported.getSpanId()); + span.setParentSpanId(reported.getParentSpanId()); + for (final SegmentReference reference : reported.getRefsList()) { + final Ref ref = new Ref(); + ref.setTraceId(reference.getTraceId()); + ref.setParentSegmentId(reference.getParentTraceSegmentId()); + ref.setParentSpanId(reference.getParentSpanId()); + ref.setType(RefType.CROSS_PROCESS); + span.getRefs().add(ref); + } + span.setServiceCode(segment.getService()); + span.setServiceInstanceName(segment.getServiceInstance()); + span.setStartTime(reported.getStartTime()); + span.setEndTime(reported.getEndTime()); + span.setEndpointName(reported.getOperationName()); + span.setType(reported.getSpanType().name()); + span.setPeer(reported.getPeer()); + span.setComponent(COMPONENTS.get(reported.getComponentId())); + span.setError(reported.getIsError()); + span.setLayer(reported.getSpanLayer().name()); + for (final KeyStringValuePair tag : reported.getTagsList()) { + span.getTags().add(new KeyValue(tag.getKey(), tag.getValue())); + } + for (final Log log : reported.getLogsList()) { + final LogEntity entity = new LogEntity(); + entity.setTime(log.getTime()); + for (final KeyStringValuePair data : log.getDataList()) { + entity.getData().add(new KeyValue(data.getKey(), data.getValue())); + } + span.getLogs().add(entity); + } + span.setRoot(root); + span.setSegmentSpanId(segment.getTraceSegmentId() + Const.SEGMENT_SPAN_SPLIT + reported.getSpanId()); + span.setSegmentParentSpanId(segmentParentSpanId); + for (final SpanAttachedEvent event : events) { + final org.apache.skywalking.oap.server.core.query.type.SpanAttachedEvent attached = + new org.apache.skywalking.oap.server.core.query.type.SpanAttachedEvent(); + attached.getStartTime().setSeconds(event.getStartTime().getSeconds()); + attached.getStartTime().setNanos(event.getStartTime().getNanos()); + attached.getEndTime().setSeconds(event.getEndTime().getSeconds()); + attached.getEndTime().setNanos(event.getEndTime().getNanos()); + attached.setEvent(event.getEvent()); + for (final KeyStringValuePair tag : event.getTagsList()) { + attached.getTags().add(new KeyValue(tag.getKey(), tag.getValue())); + } + for (final KeyIntValuePair summary : event.getSummaryList()) { + attached.getSummary().add(new KeyNumericValue(summary.getKey(), summary.getValue())); + } + span.getAttachedEvents().add(attached); + } + return span; + } + + /** The tags rover's HTTP sampling puts on an event. */ + static String[] http(final String direction, final String type, final String role) { + return new String[] { + "data_size", "182B", + "data_direction", direction, + "data_type", type, + "connection_role", role + }; + } + + private static KeyStringValuePair tag(final String key, final String value) { + return KeyStringValuePair.newBuilder().setKey(key).setValue(value).build(); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/OALTestMetrics.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/OALTestMetrics.java new file mode 100644 index 000000000000..aa4bb0bddaaf --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/OALTestMetrics.java @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.io.File; +import java.io.FileReader; +import java.io.Reader; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import javassist.ClassPool; +import org.apache.skywalking.oal.v2.generator.CodeGenModel; +import org.apache.skywalking.oal.v2.generator.MetricDefinitionEnricher; +import org.apache.skywalking.oal.v2.generator.OALClassGeneratorV2; +import org.apache.skywalking.oal.v2.model.MetricDefinition; +import org.apache.skywalking.oal.v2.parser.OALScriptParserV2; +import org.apache.skywalking.oap.server.core.analysis.SourceDecoratorManager; +import org.apache.skywalking.oap.server.core.analysis.Stream; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.oal.rt.OALDefine; +import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine; +import org.apache.skywalking.oap.server.core.storage.StorageBuilderFactory; +import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder; + +/** + * The metrics classes OAP generates from the shipped {@code core.oal}, generated the same way at test time, so an IT + * reads a metric through the schema, the value columns and the name a real OAP has. + */ +final class OALTestMetrics { + private static final String SOURCE_PACKAGE = "org.apache.skywalking.oap.server.core.source."; + private static final String METRICS_PACKAGE = "org.apache.skywalking.oap.server.core.source.oal.rt.metrics."; + private static final String CORE_OAL = "oal/core.oal"; + private static final String[] STARTER_RESOURCES = { + "../../server-starter/src/main/resources/", + "oap-server/server-starter/src/main/resources/", + "../server-starter/src/main/resources/" + }; + /** The sources {@code core.oal} reads. */ + private static final String[] SOURCES = { + "Service", "ServiceInstance", "Endpoint", "ServiceRelation", "ServiceInstanceRelation", "EndpointRelation", + "DatabaseAccess", "CacheAccess", "MQAccess", "MQEndpointAccess" + }; + private static final String[] DECORATORS = {"ServiceDecorator", "EndpointDecorator"}; + + /** Generated once per JVM: a class name can be defined only once in a class loader. */ + private static Map> CLASSES; + + private OALTestMetrics() { + } + + /** The class OAP generates for the {@code core.oal} metric of this name. */ + static synchronized Class metricsClass(String name) throws Exception { + if (CLASSES == null) { + CLASSES = generate(); + } + final Class metricsClass = CLASSES.get(name); + if (metricsClass == null) { + throw new IllegalArgumentException("core.oal has no metric " + name); + } + return metricsClass; + } + + static int scopeId(Class metricsClass) { + return metricsClass.getAnnotation(Stream.class).scopeId(); + } + + @SuppressWarnings("unchecked") + static StorageBuilder builder(Class metricsClass) throws Exception { + return (StorageBuilder) metricsClass.getAnnotation(Stream.class).builder() + .getDeclaredConstructor().newInstance(); + } + + /** A new metrics object with each named property set through its setter. */ + static Metrics newMetrics(Class metricsClass, Object... properties) throws Exception { + final Metrics metrics = metricsClass.getDeclaredConstructor().newInstance(); + for (int i = 0; i < properties.length; i += 2) { + set(metrics, (String) properties[i], properties[i + 1]); + } + return metrics; + } + + private static void set(Object target, String property, Object value) throws Exception { + final String setter = "set" + Character.toUpperCase(property.charAt(0)) + property.substring(1); + for (Class c = target.getClass(); c != null; c = c.getSuperclass()) { + for (final Method method : c.getDeclaredMethods()) { + if (method.getName().equals(setter) && method.getParameterCount() == 1) { + method.setAccessible(true); + method.invoke(target, value); + return; + } + } + } + throw new IllegalArgumentException(target.getClass().getSimpleName() + " has no " + setter); + } + + private static Map> generate() throws Exception { + final DefaultScopeDefine.Listener scopes = new DefaultScopeDefine.Listener(); + for (final String source : SOURCES) { + try { + scopes.notify(Class.forName(SOURCE_PACKAGE + source)); + } catch (Exception alreadyRegistered) { + // Another IT in this JVM registered it first. + } + } + final SourceDecoratorManager decorators = new SourceDecoratorManager(); + for (final String decorator : DECORATORS) { + decorators.addIfAsSourceDecorator(Class.forName(SOURCE_PACKAGE + decorator)); + } + + final OALDefine define = new OALDefine(CORE_OAL, SOURCE_PACKAGE, "") { + }; + final List definitions; + try (Reader reader = new FileReader(coreOAL())) { + definitions = OALScriptParserV2.parse(reader, CORE_OAL).getMetrics(); + } + final MetricDefinitionEnricher enricher = new MetricDefinitionEnricher(SOURCE_PACKAGE, METRICS_PACKAGE); + final List models = new ArrayList<>(); + for (final MetricDefinition definition : definitions) { + models.add(enricher.enrich(definition)); + } + final OALClassGeneratorV2 generator = new OALClassGeneratorV2(define, new ClassPool(true)); + generator.setStorageBuilderFactory(new StorageBuilderFactory.Default()); + final List metricsClasses = new ArrayList<>(); + generator.generateClassAtRuntime(models, new ArrayList<>(), metricsClasses, new ArrayList<>()); + + final Map> classes = new HashMap<>(); + for (final Class metricsClass : metricsClasses) { + classes.put(metricsClass.getAnnotation(Stream.class).name(), metricsClass.asSubclass(Metrics.class)); + } + return classes; + } + + private static File coreOAL() { + for (final String resources : STARTER_RESOURCES) { + final File file = new File(resources + CORE_OAL); + if (file.isFile()) { + return file; + } + } + throw new IllegalStateException( + "Cannot find the shipped " + CORE_OAL + " from " + new File("").getAbsolutePath()); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/TraceRankingAcrossTablesTest.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/TraceRankingAcrossTablesTest.java new file mode 100644 index 000000000000..be1250080903 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/TraceRankingAcrossTablesTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.Arrays; +import java.util.Collections; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class TraceRankingAcrossTablesTest { + @Test + void ranksTheTracesOfOneTableInItsRankingAlone() { + assertThat(TraceRankingAcrossTables.sql( + "trace_id", Collections.singletonList("select t1.trace_id, min(start_time) as rank_value from t1" + + " group by t1.trace_id"), "min", 20)) + .isEqualTo("select t1.trace_id, min(start_time) as rank_value from t1 group by t1.trace_id" + + " order by rank_value desc, trace_id limit 20"); + } + + @Test + void ranksTheTracesOfEveryTableOverTheUnionOfTheirRankings() { + assertThat(TraceRankingAcrossTables.sql( + "trace_id", Arrays.asList( + "select t1.trace_id, min(start_time) as rank_value from t1 group by t1.trace_id", + "select t2.trace_id, min(start_time) as rank_value from t2 group by t2.trace_id"), "min", 5)) + .isEqualTo("select trace_id, min(rank_value) as rank_value from (select t1.trace_id, min(start_time) as" + + " rank_value from t1 group by t1.trace_id union all select t2.trace_id, min(start_time) as" + + " rank_value from t2 group by t2.trace_id) ranked group by trace_id" + + " order by min(rank_value) desc, trace_id limit 5"); + } + + /** + * A trace among the highest by its greatest value is among the highest of the table holding that value, so each + * table keeps only its highest. + */ + @Test + void limitsEachTablesRankingByTheGreatestValue() { + assertThat(TraceRankingAcrossTables.sql( + "trace_id", Arrays.asList( + "select t1.trace_id, max(duration) as rank_value from t1 group by t1.trace_id", + "select t2.trace_id, max(duration) as rank_value from t2 group by t2.trace_id"), "max", 5)) + .isEqualTo("select trace_id, max(rank_value) as rank_value from ((select t1.trace_id, max(duration) as" + + " rank_value from t1 group by t1.trace_id order by rank_value desc, trace_id limit 5)" + + " union all (select t2.trace_id, max(duration) as rank_value from t2 group by" + + " t2.trace_id order by rank_value desc, trace_id limit 5)) ranked group by trace_id" + + " order by max(rank_value) desc, trace_id limit 5"); + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/ZipkinSpanRecords.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/ZipkinSpanRecords.java new file mode 100644 index 000000000000..28a6998aa8a0 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/ZipkinSpanRecords.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import com.google.gson.JsonObject; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.zipkin.ZipkinSpanRecord; +import zipkin2.Annotation; +import zipkin2.Span; + +final class ZipkinSpanRecords { + private ZipkinSpanRecords() { + } + + /** + * The record the Zipkin receiver ({@code SpanForward}) writes for a span. + */ + static ZipkinSpanRecord of(final Span span) { + final ZipkinSpanRecord record = new ZipkinSpanRecord(); + record.setTraceId(span.traceId()); + record.setSpanId(span.id()); + record.setParentId(span.parentId()); + record.setName(span.name()); + record.setKind(span.kind().name()); + record.setDuration(span.durationAsLong()); + record.setLocalEndpointServiceName(span.localServiceName()); + record.setLocalEndpointIPV4(span.localEndpoint().ipv4()); + record.setLocalEndpointIPV6(span.localEndpoint().ipv6()); + record.setLocalEndpointPort(span.localEndpoint().portAsInt()); + if (span.remoteEndpoint() != null) { + record.setRemoteEndpointServiceName(span.remoteServiceName()); + record.setRemoteEndpointIPV4(span.remoteEndpoint().ipv4()); + record.setRemoteEndpointIPV6(span.remoteEndpoint().ipv6()); + record.setRemoteEndpointPort(span.remoteEndpoint().portAsInt()); + } + record.setTimestamp(span.timestampAsLong()); + record.setTimestampMillis(span.timestampAsLong() / 1000); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(record.getTimestampMillis())); + final List query = new ArrayList<>(); + if (!span.tags().isEmpty() || !span.annotations().isEmpty()) { + final JsonObject annotations = new JsonObject(); + for (final Annotation annotation : span.annotations()) { + annotations.addProperty(Long.toString(annotation.timestamp()), annotation.value()); + query.add(annotation.value()); + } + final JsonObject tags = new JsonObject(); + for (final Map.Entry tag : span.tags().entrySet()) { + tags.addProperty(tag.getKey(), tag.getValue()); + query.add(tag.getKey()); + query.add(tag.getKey() + "=" + tag.getValue()); + } + record.setAnnotations(annotations); + record.setTags(tags); + } + record.setQuery(query); + record.setDebug(Boolean.TRUE.equals(span.debug()) ? 1 : 0); + record.setShared(Boolean.TRUE.equals(span.shared()) ? 1 : 0); + return record; + } +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/ZipkinTraceFixture.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/ZipkinTraceFixture.java new file mode 100644 index 000000000000..2b31c45db8c3 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/test/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/common/dao/ZipkinTraceFixture.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.apm.network.common.v3.Instant; +import org.apache.skywalking.apm.network.common.v3.KeyIntValuePair; +import org.apache.skywalking.apm.network.common.v3.KeyStringValuePair; +import org.apache.skywalking.apm.network.language.agent.v3.SpanAttachedEvent; +import org.yaml.snakeyaml.Yaml; +import zipkin2.Endpoint; +import zipkin2.Span; + +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.MIDNIGHT; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.MIDNIGHT_SECOND; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.AttachedEventFixture.event; +import static org.apache.skywalking.oap.server.storage.plugin.jdbc.common.dao.NativeTraceFixture.http; + +/** + * Two Zipkin traces and the events rover attaches to them. In the first, the frontend's client span calls the + * backend, whose server span is its child (B3 without shared span ids), and the backend queries a database. Both + * sides' rover report the HTTP call against the B3 context it carried, the client span: the frontend's request and + * response belong to that span, the backend's to the backend's server span. Rover reports a Zipkin context without a + * segment id. The backend's server span also calls the inventory with its own span id, as a service traced on its + * server side only does, so it carries the client side of that call besides the server side of the frontend's. + */ +final class ZipkinTraceFixture { + static final String TRACE = "463ac35c9f6413ad48485a3953bb6124"; + static final String ONE_SERVICE_TRACE = "463ac35c9f6413ad48485a3953bb6125"; + /** Has only events, against the first trace's spans, and no query asks for it. */ + static final String UNQUERIED_TRACE = "463ac35c9f6413ad48485a3953bb6126"; + /** Each has only an event, on the same span id as the other's, with the same name and start instant. */ + static final String SAME_SPAN_ID_TRACE = "463ac35c9f6413ad48485a3953bb6127"; + static final String OTHER_SAME_SPAN_ID_TRACE = "463ac35c9f6413ad48485a3953bb6128"; + /** Has only two events, on two spans, with the same name and start instant. */ + static final String TWO_SPANS_TRACE = "463ac35c9f6413ad48485a3953bb6129"; + static final String UNKNOWN_TRACE = "463ac35c9f6413ad48485a3953bb6fff"; + private static final String ROOT = "1000000000000001"; + private static final String CLIENT = "1000000000000002"; + private static final String SERVER = "1000000000000003"; + private static final String DATABASE = "1000000000000004"; + private static final String INVENTORY = "1000000000000005"; + private static final String JOB = "2000000000000001"; + private static final String LOAD = "2000000000000002"; + + private static final Span ROOT_SPAN = span(TRACE, ROOT, null, Span.Kind.SERVER, "get /", "frontend", + MIDNIGHT - 35_000L, 30_000_000L).build(); + private static final Span CLIENT_SPAN = span(TRACE, CLIENT, ROOT, Span.Kind.CLIENT, "get /api", "frontend", + MIDNIGHT - 31_000L, 20_000_000L) + .remoteEndpoint(Endpoint.newBuilder().serviceName("backend").ip("10.0.0.2").port(8080).build()).build(); + private static final Span SERVER_SPAN = span(TRACE, SERVER, CLIENT, Span.Kind.SERVER, "get /api", "backend", + MIDNIGHT - 30_500L, 19_000_000L).build(); + private static final Span DATABASE_SPAN = span(TRACE, DATABASE, SERVER, Span.Kind.CLIENT, "select orders", + "backend", MIDNIGHT - 26_000L, 1_000_000L) + .remoteEndpoint(Endpoint.newBuilder().serviceName("mysql").build()) + .putTag("sql.query", "select * from orders").build(); + /** Called by the backend's server span with that span's id, without a client span between them. */ + private static final Span INVENTORY_SPAN = span(TRACE, INVENTORY, SERVER, Span.Kind.SERVER, "get /stock", + "inventory", MIDNIGHT - 22_500L, 2_000_000L).build(); + private static final Span JOB_SPAN = span(ONE_SERVICE_TRACE, JOB, null, Span.Kind.SERVER, "run job", "batch", + MIDNIGHT - 20_000L, 5_000_000L).build(); + private static final Span LOAD_SPAN = span(ONE_SERVICE_TRACE, LOAD, JOB, Span.Kind.CLIENT, "load batch", "batch", + MIDNIGHT - 19_000L, 1_000_000L) + .remoteEndpoint(Endpoint.newBuilder().serviceName("postgres").build()).build(); + static final List SPANS = Arrays.asList( + ROOT_SPAN, CLIENT_SPAN, SERVER_SPAN, DATABASE_SPAN, INVENTORY_SPAN, JOB_SPAN, LOAD_SPAN); + + static final SpanAttachedEvent CLIENT_REQUEST = zipkinEvent( + TRACE, CLIENT, MIDNIGHT_SECOND - 30, 100, "HTTP Request Sampling", http("outbound", "request", "client")) + .build(); + static final SpanAttachedEvent CLIENT_WRITE = zipkinEvent( + TRACE, CLIENT, MIDNIGHT_SECOND - 29, 0, "Syscall write", "package_size", "182B", "package_count", "1") + .addSummary(KeyIntValuePair.newBuilder().setKey("package_bytes").setValue(182L)).build(); + static final SpanAttachedEvent SERVER_REQUEST = zipkinEvent( + TRACE, CLIENT, MIDNIGHT_SECOND - 28, 0, "HTTP Request Sampling", http("inbound", "request", "server")) + .build(); + static final SpanAttachedEvent DATABASE_SEND = zipkinEvent( + TRACE, DATABASE, MIDNIGHT_SECOND - 25, 0, "Syscall sendto", "package_size", "96B", "package_count", "1") + .build(); + static final SpanAttachedEvent UNKNOWN_SPAN_READ = zipkinEvent( + TRACE, "ffffffffffffffff", MIDNIGHT_SECOND - 24, 0, "Syscall read", "package_size", "64B", + "package_count", "1").build(); + static final SpanAttachedEvent BACKEND_REQUEST = zipkinEvent( + TRACE, SERVER, MIDNIGHT_SECOND - 23, 0, "HTTP Request Sampling", http("outbound", "request", "client")) + .build(); + static final SpanAttachedEvent INVENTORY_REQUEST = zipkinEvent( + TRACE, SERVER, MIDNIGHT_SECOND - 22, 0, "HTTP Request Sampling", http("inbound", "request", "server")) + .build(); + static final SpanAttachedEvent INVENTORY_RESPONSE = zipkinEvent( + TRACE, SERVER, MIDNIGHT_SECOND - 21, 0, "HTTP Response Sampling", http("outbound", "response", "server")) + .build(); + static final SpanAttachedEvent BACKEND_RESPONSE = zipkinEvent( + TRACE, SERVER, MIDNIGHT_SECOND - 21, 500_000_000, "HTTP Response Sampling", + http("inbound", "response", "client")).build(); + static final SpanAttachedEvent SERVER_RESPONSE = zipkinEvent( + TRACE, CLIENT, MIDNIGHT_SECOND - 20, 0, "HTTP Response Sampling", http("outbound", "response", "server")) + .build(); + static final SpanAttachedEvent CLIENT_RESPONSE = zipkinEvent( + TRACE, CLIENT, MIDNIGHT_SECOND - 19, 0, "HTTP Response Sampling", http("inbound", "response", "client")) + .build(); + static final SpanAttachedEvent LOAD_WRITE = zipkinEvent( + ONE_SERVICE_TRACE, LOAD, MIDNIGHT_SECOND - 18, 0, "Syscall write", "package_size", "1KB", + "package_count", "2").build(); + private static final SpanAttachedEvent UNQUERIED_TRACE_REQUEST = zipkinEvent( + UNQUERIED_TRACE, CLIENT, MIDNIGHT_SECOND - 27, 0, "HTTP Request Sampling", + http("outbound", "request", "client")).build(); + static final SpanAttachedEvent SAME_SPAN_ID_REQUEST = zipkinEvent( + SAME_SPAN_ID_TRACE, "3000000000000001", MIDNIGHT_SECOND - 40, 0, "HTTP Request Sampling", + http("outbound", "request", "client")).build(); + static final SpanAttachedEvent OTHER_SAME_SPAN_ID_REQUEST = zipkinEvent( + OTHER_SAME_SPAN_ID_TRACE, "3000000000000001", MIDNIGHT_SECOND - 40, 0, "HTTP Request Sampling", + http("outbound", "request", "client")).build(); + static final SpanAttachedEvent FIRST_SPAN_REQUEST = zipkinEvent( + TWO_SPANS_TRACE, "5000000000000001", MIDNIGHT_SECOND - 41, 0, "HTTP Request Sampling", + http("outbound", "request", "client")).build(); + static final SpanAttachedEvent SECOND_SPAN_REQUEST = zipkinEvent( + TWO_SPANS_TRACE, "5000000000000002", MIDNIGHT_SECOND - 41, 0, "HTTP Request Sampling", + http("outbound", "request", "client")).build(); + /** Stored with the SkyWalking events, as its trace context's type says, though it names this trace. */ + private static final SpanAttachedEvent SKYWALKING_TYPED_WRITE = event( + SpanAttachedEvent.SpanReferenceType.SKYWALKING, TRACE, "3a5b7c9d0e1f.31.17368992000000021", "0", + MIDNIGHT_SECOND - 26, 0, "Syscall write", "package_size", "182B", "package_count", "1").build(); + + /** Neither in start order nor against it. */ + static final List EVENTS = Arrays.asList( + SERVER_RESPONSE, LOAD_WRITE, INVENTORY_REQUEST, SAME_SPAN_ID_REQUEST, SECOND_SPAN_REQUEST, CLIENT_WRITE, + BACKEND_RESPONSE, UNKNOWN_SPAN_READ, SKYWALKING_TYPED_WRITE, CLIENT_REQUEST, OTHER_SAME_SPAN_ID_REQUEST, + FIRST_SPAN_REQUEST, BACKEND_REQUEST, + UNQUERIED_TRACE_REQUEST, CLIENT_RESPONSE, INVENTORY_RESPONSE, DATABASE_SEND, SERVER_REQUEST); + static final List EVENTS_OF_BOTH_TRACES = Arrays.asList( + CLIENT_REQUEST, CLIENT_WRITE, SERVER_REQUEST, DATABASE_SEND, UNKNOWN_SPAN_READ, BACKEND_REQUEST, + INVENTORY_REQUEST, INVENTORY_RESPONSE, BACKEND_RESPONSE, SERVER_RESPONSE, CLIENT_RESPONSE, LOAD_WRITE); + + private ZipkinTraceFixture() { + } + + /** + * The traces with each event attached to its span as {@code ZipkinQueryService} shows it: a start and a finish + * annotation, and the event's tags and summary as tags. An event name that occurs more than once in a trace is + * numbered in start order. + */ + static List> expectedTraces(final String... traceIds) { + final List> traces = new ArrayList<>(); + for (final String traceId : traceIds) { + if (TRACE.equals(traceId)) { + traces.add(Arrays.asList( + ROOT_SPAN, + attach(attach(attach(CLIENT_SPAN.toBuilder(), + "HTTP Request Sampling-1", CLIENT_REQUEST), + "Syscall write", CLIENT_WRITE), + "HTTP Response Sampling-4", CLIENT_RESPONSE).build(), + // The server side of the frontend's call, and the client side of its own call to the inventory. + attach(attach(attach(attach(SERVER_SPAN.toBuilder(), + "HTTP Request Sampling-2", SERVER_REQUEST), + "HTTP Request Sampling-3", BACKEND_REQUEST), + "HTTP Response Sampling-2", BACKEND_RESPONSE), + "HTTP Response Sampling-3", SERVER_RESPONSE).build(), + attach(DATABASE_SPAN.toBuilder(), "Syscall sendto", DATABASE_SEND).build(), + attach(attach(INVENTORY_SPAN.toBuilder(), + "HTTP Request Sampling-4", INVENTORY_REQUEST), + "HTTP Response Sampling-1", INVENTORY_RESPONSE).build())); + } else { + traces.add(Arrays.asList( + JOB_SPAN, + attach(LOAD_SPAN.toBuilder(), "Syscall write", LOAD_WRITE).build())); + } + } + return canonical(traces); + } + + /** + * The traces sorted by id, each trace's spans sorted by id, and every event's YAML tag read as a map: the Zipkin + * API promises no order of traces or spans, and YAML has more than one way to write a map. + */ + static List> canonical(final List> traces) { + final List> canonical = new ArrayList<>(); + for (final List trace : traces) { + final List spans = new ArrayList<>(); + for (final Span span : trace) { + final Span.Builder builder = span.toBuilder().clearTags(); + for (final Map.Entry tag : span.tags().entrySet()) { + final boolean eventMap = tag.getKey().endsWith(".tags") || tag.getKey().endsWith(".summary"); + builder.putTag(tag.getKey(), eventMap ? sortedMap(new Yaml().load(tag.getValue())) : tag.getValue()); + } + spans.add(builder.build()); + } + spans.sort(Comparator.comparing(Span::id)); + canonical.add(spans); + } + canonical.sort(Comparator.comparing(trace -> trace.isEmpty() ? "" : trace.get(0).traceId())); + return canonical; + } + + private static Span.Builder attach(final Span.Builder span, final String name, final SpanAttachedEvent event) { + span.addAnnotation(micros(event.getStartTime()), "Start " + name); + span.addAnnotation(micros(event.getEndTime()), "Finished " + name); + if (event.getSummaryCount() > 0) { + final Map summary = new TreeMap<>(); + for (final KeyIntValuePair pair : event.getSummaryList()) { + summary.put(pair.getKey(), pair.getValue()); + } + span.putTag(eventTagKey(name, "summary"), new Yaml().dump(summary)); + } + if (event.getTagsCount() > 0) { + final Map tags = new TreeMap<>(); + for (final KeyStringValuePair tag : event.getTagsList()) { + tags.put(tag.getKey(), tag.getValue()); + } + span.putTag(eventTagKey(name, "tags"), new Yaml().dump(tags)); + } + return span; + } + + private static String eventTagKey(final String eventName, final String suffix) { + return (eventName + "." + suffix).replace(' ', '.').toLowerCase(Locale.ROOT); + } + + private static long micros(final Instant instant) { + return TimeUnit.SECONDS.toMicros(instant.getSeconds()) + TimeUnit.NANOSECONDS.toMicros(instant.getNanos()); + } + + private static String sortedMap(final Map map) { + final Map sorted = new TreeMap<>(); + map.forEach((key, value) -> sorted.put(String.valueOf(key), String.valueOf(value))); + return sorted.toString(); + } + + private static SpanAttachedEvent.Builder zipkinEvent(final String traceId, final String spanId, + final long startSecond, final int startNanos, + final String name, final String... tags) { + return event(SpanAttachedEvent.SpanReferenceType.ZIPKIN, traceId, "", spanId, startSecond, startNanos, + name, tags); + } + + private static Span.Builder span(final String traceId, final String spanId, final String parentId, + final Span.Kind kind, final String name, final String service, + final long timestampMillis, final long durationMicros) { + return Span.newBuilder() + .traceId(traceId) + .id(spanId) + .parentId(parentId) + .kind(kind) + .name(name) + .localEndpoint(Endpoint.newBuilder().serviceName(service).build()) + .timestamp(timestampMillis * 1000L) + .duration(durationMicros) + .debug(false) + .shared(false); + } +} diff --git a/test/e2e-v2/cases/ai-agent/es/docker-compose.yml b/test/e2e-v2/cases/ai-agent/es/docker-compose.yml index 5d6f82e98273..242f3d84c359 100644 --- a/test/e2e-v2/cases/ai-agent/es/docker-compose.yml +++ b/test/e2e-v2/cases/ai-agent/es/docker-compose.yml @@ -52,7 +52,7 @@ x-asz: &asz services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/alarm/es/docker-compose.yml b/test/e2e-v2/cases/alarm/es/docker-compose.yml index 5799ca339f1b..275c81364b8c 100644 --- a/test/e2e-v2/cases/alarm/es/docker-compose.yml +++ b/test/e2e-v2/cases/alarm/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml index bf2f13490736..c64706e1c7f4 100644 --- a/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml +++ b/test/e2e-v2/cases/alarm/es/es-sharding/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/baseline/es/docker-compose.yml b/test/e2e-v2/cases/baseline/es/docker-compose.yml index 8e48eebb81d5..4d461786c598 100644 --- a/test/e2e-v2/cases/baseline/es/docker-compose.yml +++ b/test/e2e-v2/cases/baseline/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/baseline/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/baseline/es/es-sharding/docker-compose.yml index 001e80973851..9af7cf100b7b 100644 --- a/test/e2e-v2/cases/baseline/es/es-sharding/docker-compose.yml +++ b/test/e2e-v2/cases/baseline/es/es-sharding/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/browser/es/docker-compose.yml b/test/e2e-v2/cases/browser/es/docker-compose.yml index 66667af3ab36..1e42e69aa2c1 100644 --- a/test/e2e-v2/cases/browser/es/docker-compose.yml +++ b/test/e2e-v2/cases/browser/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/browser/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/browser/es/es-sharding/docker-compose.yml index c7554d610542..d7867deb6455 100644 --- a/test/e2e-v2/cases/browser/es/es-sharding/docker-compose.yml +++ b/test/e2e-v2/cases/browser/es/es-sharding/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/cluster/zk/es/docker-compose.yml b/test/e2e-v2/cases/cluster/zk/es/docker-compose.yml index 79c019f8e7b0..be33eece69fa 100644 --- a/test/e2e-v2/cases/cluster/zk/es/docker-compose.yml +++ b/test/e2e-v2/cases/cluster/zk/es/docker-compose.yml @@ -27,7 +27,7 @@ services: retries: 120 es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/event/es/docker-compose.yml b/test/e2e-v2/cases/event/es/docker-compose.yml index fe5062446169..efd2b625b2d5 100644 --- a/test/e2e-v2/cases/event/es/docker-compose.yml +++ b/test/e2e-v2/cases/event/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/inspect/elasticsearch/docker-compose.yml b/test/e2e-v2/cases/inspect/elasticsearch/docker-compose.yml index d408ac3928d1..0d8bd105a261 100644 --- a/test/e2e-v2/cases/inspect/elasticsearch/docker-compose.yml +++ b/test/e2e-v2/cases/inspect/elasticsearch/docker-compose.yml @@ -18,7 +18,7 @@ # so that metric is foreign to it (read via the merged metrics-all index). services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/log/es/docker-compose.yml b/test/e2e-v2/cases/log/es/docker-compose.yml index 801ee6ac3f77..27154ce93eed 100644 --- a/test/e2e-v2/cases/log/es/docker-compose.yml +++ b/test/e2e-v2/cases/log/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/log/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/log/es/es-sharding/docker-compose.yml index aedc13ce4da6..b09268c3abd9 100644 --- a/test/e2e-v2/cases/log/es/es-sharding/docker-compose.yml +++ b/test/e2e-v2/cases/log/es/es-sharding/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml b/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml index 7a38376c4cc5..a5c87330229e 100644 --- a/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml +++ b/test/e2e-v2/cases/log/fluent-bit/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/logql/docker-compose.yml b/test/e2e-v2/cases/logql/docker-compose.yml index 8ad7d15cf2b5..4f078356c9f3 100644 --- a/test/e2e-v2/cases/logql/docker-compose.yml +++ b/test/e2e-v2/cases/logql/docker-compose.yml @@ -68,7 +68,7 @@ services: condition: service_healthy es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} ports: - 9200 networks: diff --git a/test/e2e-v2/cases/miniprogram/alipay/expected/logs.yml b/test/e2e-v2/cases/miniprogram/alipay/expected/logs.yml index 4eb3d0357ad2..11fd5ea8c9e0 100644 --- a/test/e2e-v2/cases/miniprogram/alipay/expected/logs.yml +++ b/test/e2e-v2/cases/miniprogram/alipay/expected/logs.yml @@ -26,7 +26,7 @@ logs: traceid: null timestamp: {{ gt .timestamp 0 }} contenttype: TEXT - content: '{{ notEmpty .content }}' + content: {{ printf "%q" (notEmpty .content) }} tags: {{- contains .tags }} - key: platform diff --git a/test/e2e-v2/cases/miniprogram/wechat/expected/logs.yml b/test/e2e-v2/cases/miniprogram/wechat/expected/logs.yml index 94464ed39a75..955dafed9ecb 100644 --- a/test/e2e-v2/cases/miniprogram/wechat/expected/logs.yml +++ b/test/e2e-v2/cases/miniprogram/wechat/expected/logs.yml @@ -26,7 +26,7 @@ logs: traceid: null timestamp: {{ gt .timestamp 0 }} contenttype: TEXT - content: '{{ notEmpty .content }}' + content: {{ printf "%q" (notEmpty .content) }} tags: {{- contains .tags }} - key: platform diff --git a/test/e2e-v2/cases/otlp-traces/native/es/docker-compose.yml b/test/e2e-v2/cases/otlp-traces/native/es/docker-compose.yml index b8527a512665..64b411c1e3c2 100644 --- a/test/e2e-v2/cases/otlp-traces/native/es/docker-compose.yml +++ b/test/e2e-v2/cases/otlp-traces/native/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:7.15.0 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/profiling/async-profiler/es/docker-compose.yml b/test/e2e-v2/cases/profiling/async-profiler/es/docker-compose.yml index 33a48a7161d8..6a89eaf327e2 100644 --- a/test/e2e-v2/cases/profiling/async-profiler/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/async-profiler/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml index 1b78a6114a57..90c133d7fa8b 100644 --- a/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/pprof/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/profiling/trace/es/docker-compose.yml b/test/e2e-v2/cases/profiling/trace/es/docker-compose.yml index e5fd1315dda9..553b370ee4ad 100644 --- a/test/e2e-v2/cases/profiling/trace/es/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/trace/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 ports: diff --git a/test/e2e-v2/cases/profiling/trace/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/profiling/trace/es/es-sharding/docker-compose.yml index 30c6ea94a37c..efa45bbe4a26 100644 --- a/test/e2e-v2/cases/profiling/trace/es/es-sharding/docker-compose.yml +++ b/test/e2e-v2/cases/profiling/trace/es/es-sharding/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/runtime-rule/mal-storage/elasticsearch/docker-compose.yml b/test/e2e-v2/cases/runtime-rule/mal-storage/elasticsearch/docker-compose.yml index 54391f663268..dfb6ab9fb1e7 100644 --- a/test/e2e-v2/cases/runtime-rule/mal-storage/elasticsearch/docker-compose.yml +++ b/test/e2e-v2/cases/runtime-rule/mal-storage/elasticsearch/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} networks: - e2e ports: diff --git a/test/e2e-v2/cases/storage/es/docker-compose.yml b/test/e2e-v2/cases/storage/es/docker-compose.yml index 2d87b79dd607..60b775e79147 100644 --- a/test/e2e-v2/cases/storage/es/docker-compose.yml +++ b/test/e2e-v2/cases/storage/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml index dfacace5da82..5cdba0fc15a5 100644 --- a/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml +++ b/test/e2e-v2/cases/storage/es/es-sharding/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/ttl/es/docker-compose.yml b/test/e2e-v2/cases/ttl/es/docker-compose.yml index 774342257d72..da3d607770bf 100644 --- a/test/e2e-v2/cases/ttl/es/docker-compose.yml +++ b/test/e2e-v2/cases/ttl/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:${ES_VERSION} + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/zipkin/es/docker-compose.yml b/test/e2e-v2/cases/zipkin/es/docker-compose.yml index 31be0aeebb03..6b045a3e7db7 100644 --- a/test/e2e-v2/cases/zipkin/es/docker-compose.yml +++ b/test/e2e-v2/cases/zipkin/es/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: diff --git a/test/e2e-v2/cases/zipkin/es/es-sharding/docker-compose.yml b/test/e2e-v2/cases/zipkin/es/es-sharding/docker-compose.yml index d36d5f43227f..6ff900430c3e 100644 --- a/test/e2e-v2/cases/zipkin/es/es-sharding/docker-compose.yml +++ b/test/e2e-v2/cases/zipkin/es/es-sharding/docker-compose.yml @@ -15,7 +15,7 @@ services: es: - image: elastic/elasticsearch:8.18.8 + image: elastic/elasticsearch:${ES_VERSION:-8.18.8} expose: - 9200 networks: