diff --git a/api/src/org/labkey/api/action/ApiQueryResponse.java b/api/src/org/labkey/api/action/ApiQueryResponse.java index bbcaba1b7b9..742447b1653 100644 --- a/api/src/org/labkey/api/action/ApiQueryResponse.java +++ b/api/src/org/labkey/api/action/ApiQueryResponse.java @@ -174,6 +174,9 @@ public void render(ApiResponseWriter writer) throws Exception long rowCount = _rowCount > 0 ? _rowCount : _offset + _numRespRows; writer.writeProperty("rowCount", rowCount); + if (_dataRegion.isTotalRowsCapped()) + writer.writeProperty("rowCountCapped", true); + if (_includeMetaData) { // messages, but only if metadata is requested diff --git a/api/src/org/labkey/api/data/DataRegion.java b/api/src/org/labkey/api/data/DataRegion.java index 0aeb5ec9a1b..e79b9abe1f6 100644 --- a/api/src/org/labkey/api/data/DataRegion.java +++ b/api/src/org/labkey/api/data/DataRegion.java @@ -189,6 +189,7 @@ public class DataRegion extends DisplayElement private boolean _horizontalGroups = true; private boolean _errorCreatingResults = false; private Long _totalRows = null; // total rows in the query or null if unknown + private boolean _totalRowsCapped = false; // true when _totalRows was capped at maxCount rather than counted exactly private Integer _rowCount = null; // number of rows in the result set or null if unknown private boolean _complete = false; // true if all rows are in the ResultSet private boolean _buttonBarRendered = false; @@ -860,6 +861,14 @@ public Map> getAggregateResults(RenderContext ctx _totalRows = 0L; if (countStarResult.getValue() instanceof Number) _totalRows = ((Number) countStarResult.getValue()).longValue(); + + // The cap only takes effect when count star is the sole aggregate (same precondition as the SELECT 1 optimization), so it never truncates a summary-stat query. + int maxCount = getSettings() != null ? getSettings().getMaxCount() : 0; + if (baseAggregates.isEmpty() && maxCount > 0 && _totalRows > maxCount) + { + _totalRows = (long) maxCount; + _totalRowsCapped = true; + } } } } @@ -892,6 +901,11 @@ public Long getTotalRows() return _totalRows; } + public boolean isTotalRowsCapped() + { + return _totalRowsCapped; + } + public void setTotalRows(Long totalRows) { if (_totalRows == null) diff --git a/api/src/org/labkey/api/data/RenderContext.java b/api/src/org/labkey/api/data/RenderContext.java index db90b930c62..d073bb516b4 100644 --- a/api/src/org/labkey/api/data/RenderContext.java +++ b/api/src/org/labkey/api/data/RenderContext.java @@ -434,10 +434,12 @@ protected void close(@Nullable ResultSet rs, @Nullable Connection conn) selector.setNamedParameters(parameters); + // GitHub Issue 1534: Cap row counts for React grid pagination + int maxCount = settings != null ? settings.getMaxCount() : 0; if (async) - return selector.getAggregatesAsync(aggregates, getViewContext().getResponse()); + return selector.getAggregatesAsync(aggregates, getViewContext().getResponse(), maxCount); else - return selector.getAggregates(aggregates); + return selector.getAggregates(aggregates, maxCount); } return Collections.emptyMap(); diff --git a/api/src/org/labkey/api/data/TableSelector.java b/api/src/org/labkey/api/data/TableSelector.java index ad2ab32e015..898773d4b5f 100644 --- a/api/src/org/labkey/api/data/TableSelector.java +++ b/api/src/org/labkey/api/data/TableSelector.java @@ -554,15 +554,32 @@ public boolean exists() // TODO: Convert to return Map> public Map> getAggregates(final List aggregates) + { + return getAggregates(aggregates, 0); + } + + /** + * @param maxCount when > 0 and the only aggregate is COUNT(*), bounds the inner select to maxCount + 1 rows so the database can stop early. + */ + public Map> getAggregates(final List aggregates, int maxCount) { // If we are only asking for the COUNT(*) aggregate, then we don't need to include all of the table columns in the subselect. // This can make a big performance difference for Sample Type and Data Class tables as they can then skip // the join between the exp schema base table and the materialized table for the given table. - Collection aggColumns = aggregates.size() == 1 && aggregates.getFirst().isCountStar() ? getRowCountingSelectColumns(_table) : _columns; + boolean countStarOnly = aggregates.size() == 1 && aggregates.getFirst().isCountStar(); + Collection aggColumns = countStarOnly ? getRowCountingSelectColumns(_table) : _columns; final AggregateSqlFactory sqlFactory = new AggregateSqlFactory(_filter, aggregates, aggColumns); ResultSetFactory resultSetFactory = new ExecutingResultSetFactory(sqlFactory); + // Setting _maxRows threads LIMIT maxCount + 1 through TableSqlFactory.getSql() into the inner select; restore it after. + boolean cap = maxCount > 0 && countStarOnly; + var maxRows = _maxRows; + if (cap) + _maxRows = maxCount + 1; + + try + { return resultSetFactory.handleResultSet((rs, conn) -> { Map> results = new CaseInsensitiveHashMap<>(); @@ -589,9 +606,20 @@ public Map> getAggregates(final List aggregates) return results; }); + } + finally + { + if (cap) + _maxRows = maxRows; + } } public Map> getAggregatesAsync(final List aggregates, HttpServletResponse response) + { + return getAggregatesAsync(aggregates, response, 0); + } + + public Map> getAggregatesAsync(final List aggregates, HttpServletResponse response, int maxCount) { setLogger(ConnectionWrapper.getConnectionLogger()); AsyncQueryRequest>> asyncRequest = new AsyncQueryRequest<>(response, getAsyncResourceName("getAggregates"), getAsyncSpanTags()); @@ -599,7 +627,7 @@ public Map> getAggregatesAsync(final List aggreg try { - return asyncRequest.waitForResult(() -> getAggregates(aggregates)); + return asyncRequest.waitForResult(() -> getAggregates(aggregates, maxCount)); } catch (SQLException e) { diff --git a/api/src/org/labkey/api/query/QueryParam.java b/api/src/org/labkey/api/query/QueryParam.java index 71e62364ffb..69055017618 100644 --- a/api/src/org/labkey/api/query/QueryParam.java +++ b/api/src/org/labkey/api/query/QueryParam.java @@ -29,6 +29,7 @@ public enum QueryParam implements SafeToRenderEnum offset, maxRows, + maxCount, showRows, ignoreFilter, diff --git a/api/src/org/labkey/api/query/QuerySettings.java b/api/src/org/labkey/api/query/QuerySettings.java index 3677bd11971..d474e82e3b7 100644 --- a/api/src/org/labkey/api/query/QuerySettings.java +++ b/api/src/org/labkey/api/query/QuerySettings.java @@ -76,6 +76,7 @@ public class QuerySettings private boolean _ignoreViewFilter; private int _maxRows = 100; private boolean _maxRowsSet = false; // Explicitly track setting maxRows, allows for different defaults + private int _maxCount = 0; // 0 = count exactly (unbounded); >0 caps the pagination COUNT(*) at this many rows private long _offset = 0; private String _selectionKey = null; @@ -286,6 +287,21 @@ public void init(PropertyValues pvs) throwParameterParseException(QueryParam.maxRows); } } + + String maxCountParam = _getParameter(param(QueryParam.maxCount)); + if (maxCountParam != null) + { + try + { + int maxCount = Integer.parseInt(maxCountParam); + if (maxCount > 0) + setMaxCount(maxCount); + } + catch (NumberFormatException nfe) + { + throwParameterParseException(QueryParam.maxCount); + } + } } String containerFilterNameParam = _getParameter(param(QueryParam.containerFilterName)); @@ -632,6 +648,18 @@ public boolean isMaxRowsSet() return _maxRowsSet; } + /** @return The cap on the pagination COUNT(*), or 0 for an exact (unbounded) count. */ + public int getMaxCount() + { + return _maxCount; + } + + /** @param maxCount count no further than this many rows; 0 restores an exact count. */ + public void setMaxCount(int maxCount) + { + _maxCount = maxCount; + } + /** @return The offset parameter when {@link ShowRows#PAGINATED}, otherwise 0. */ public long getOffset() { diff --git a/core/package-lock.json b/core/package-lock.json index 17b93f8333c..5861828d5ef 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.62.3", + "@labkey/components": "7.62.4-fb-limitMaxCount.2", "@labkey/themes": "1.9.6" }, "devDependencies": { @@ -2897,9 +2897,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.52.4", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.52.4.tgz", - "integrity": "sha512-a6kLuI7Z33nbkH1MF+0vfn7secPrGyh9Qig3QH1esmsOGVE0EfggI+hZ8RgVBXxcbq5UKP/ti2ypHD3cvNE+zQ==", + "version": "1.52.5-fb-limitMaxCount.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.52.5-fb-limitMaxCount.1.tgz", + "integrity": "sha512-X320LbNjGkpStkwWVudCWPS5pK0DfkRYeMBScWL9QBtIaLKkRyiOMHem6JomXM5IBC64X5qTo7W7uy+zge8Wag==", "license": "Apache-2.0" }, "node_modules/@labkey/build": { @@ -2931,13 +2931,13 @@ } }, "node_modules/@labkey/components": { - "version": "7.62.3", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.62.3.tgz", - "integrity": "sha512-M8ZinIt7fk8F0/GW5ARYLRyyzwwAZ6snozEhup1aqTCxRWCQTYJhlCoOZwplYPdxEGG7ak85q83fRnyQBellFg==", + "version": "7.62.4-fb-limitMaxCount.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.62.4-fb-limitMaxCount.2.tgz", + "integrity": "sha512-qu9F+xZ16CsFsmkNZjCRubI5pzdMEDDeiRTgntZttykiXoOku4Q1mB9+Z7gF1blCvA7Uy3GK3yENupZAxv2UJw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.52.4", + "@labkey/api": "1.52.5-fb-limitMaxCount.1", "@testing-library/dom": "~10.4.1", "@testing-library/jest-dom": "~7.0.1", "@testing-library/react": "~16.3.2", diff --git a/core/package.json b/core/package.json index 8babe09ba57..87f5bce954f 100644 --- a/core/package.json +++ b/core/package.json @@ -20,7 +20,7 @@ "lint-branch-fix": "node lint.diff.mjs --currentBranch --fix" }, "dependencies": { - "@labkey/components": "7.62.3", + "@labkey/components": "7.62.4-fb-limitMaxCount.2", "@labkey/themes": "1.9.6" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index 4a8f90988c4..440fc72c867 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.62.3" + "@labkey/components": "7.62.4-fb-limitMaxCount.0" }, "devDependencies": { "@labkey/build": "10.1.2", @@ -2906,9 +2906,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.52.4", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.52.4.tgz", - "integrity": "sha512-a6kLuI7Z33nbkH1MF+0vfn7secPrGyh9Qig3QH1esmsOGVE0EfggI+hZ8RgVBXxcbq5UKP/ti2ypHD3cvNE+zQ==", + "version": "1.52.5-fb-limitMaxCount.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.52.5-fb-limitMaxCount.1.tgz", + "integrity": "sha512-X320LbNjGkpStkwWVudCWPS5pK0DfkRYeMBScWL9QBtIaLKkRyiOMHem6JomXM5IBC64X5qTo7W7uy+zge8Wag==", "license": "Apache-2.0" }, "node_modules/@labkey/build": { @@ -2940,13 +2940,13 @@ } }, "node_modules/@labkey/components": { - "version": "7.62.3", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.62.3.tgz", - "integrity": "sha512-M8ZinIt7fk8F0/GW5ARYLRyyzwwAZ6snozEhup1aqTCxRWCQTYJhlCoOZwplYPdxEGG7ak85q83fRnyQBellFg==", + "version": "7.62.4-fb-limitMaxCount.0", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.62.4-fb-limitMaxCount.0.tgz", + "integrity": "sha512-yDzEWOByQAIzLET2HkI5LofK0Tm2y9qoxWSIgFObiDWIUL3OPAMBnCX3AyuDk3xh6UKBkWEZEAjeenC8mAejaQ==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.52.4", + "@labkey/api": "1.52.5-fb-limitMaxCount.1", "@testing-library/dom": "~10.4.1", "@testing-library/jest-dom": "~7.0.1", "@testing-library/react": "~16.3.2", diff --git a/experiment/package.json b/experiment/package.json index 9fb8b975d2e..bd8561f5cb1 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.62.3" + "@labkey/components": "7.62.4-fb-limitMaxCount.0" }, "devDependencies": { "@labkey/build": "10.1.2", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index cb8e0f77b07..11caa695ce6 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.62.3" + "@labkey/components": "7.62.4-fb-limitMaxCount.2" }, "devDependencies": { "@labkey/build": "10.1.2", @@ -1435,9 +1435,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.52.4", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.52.4.tgz", - "integrity": "sha512-a6kLuI7Z33nbkH1MF+0vfn7secPrGyh9Qig3QH1esmsOGVE0EfggI+hZ8RgVBXxcbq5UKP/ti2ypHD3cvNE+zQ==", + "version": "1.52.5-fb-limitMaxCount.1", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.52.5-fb-limitMaxCount.1.tgz", + "integrity": "sha512-X320LbNjGkpStkwWVudCWPS5pK0DfkRYeMBScWL9QBtIaLKkRyiOMHem6JomXM5IBC64X5qTo7W7uy+zge8Wag==", "license": "Apache-2.0" }, "node_modules/@labkey/build": { @@ -1469,13 +1469,13 @@ } }, "node_modules/@labkey/components": { - "version": "7.62.3", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.62.3.tgz", - "integrity": "sha512-M8ZinIt7fk8F0/GW5ARYLRyyzwwAZ6snozEhup1aqTCxRWCQTYJhlCoOZwplYPdxEGG7ak85q83fRnyQBellFg==", + "version": "7.62.4-fb-limitMaxCount.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.62.4-fb-limitMaxCount.2.tgz", + "integrity": "sha512-qu9F+xZ16CsFsmkNZjCRubI5pzdMEDDeiRTgntZttykiXoOku4Q1mB9+Z7gF1blCvA7Uy3GK3yENupZAxv2UJw==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.52.4", + "@labkey/api": "1.52.5-fb-limitMaxCount.1", "@testing-library/dom": "~10.4.1", "@testing-library/jest-dom": "~7.0.1", "@testing-library/react": "~16.3.2", diff --git a/pipeline/package.json b/pipeline/package.json index 291b4da2f76..dcd9bac66ec 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production rspack build --config node_modules/@labkey/build/configs/prod.config.js" }, "dependencies": { - "@labkey/components": "7.62.3" + "@labkey/components": "7.62.4-fb-limitMaxCount.2" }, "devDependencies": { "@labkey/build": "10.1.2", diff --git a/query/src/org/labkey/query/controllers/QueryController.java b/query/src/org/labkey/query/controllers/QueryController.java index 8b80c5ee0d1..d222143fd72 100644 --- a/query/src/org/labkey/query/controllers/QueryController.java +++ b/query/src/org/labkey/query/controllers/QueryController.java @@ -3480,6 +3480,7 @@ public static class APIQueryForm extends ContainerFilterQueryForm { private Integer _start; private Integer _limit; + private Integer _maxCount; private boolean _includeDetailsColumn = false; private boolean _includeUpdateColumn = false; private boolean _includeTotalCount = true; @@ -3508,6 +3509,16 @@ public void setLimit(Integer limit) _limit = limit; } + public Integer getMaxCount() + { + return _maxCount; + } + + public void setMaxCount(Integer maxCount) + { + _maxCount = maxCount; + } + public boolean isIncludeTotalCount() { return _includeTotalCount; @@ -3598,6 +3609,8 @@ protected QuerySettings createQuerySettings(UserSchema schema) } if (getStart() != null) results.setOffset(getStart()); + if (getMaxCount() != null) + results.setMaxCount(getMaxCount()); return results; }