diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml new file mode 100644 index 00000000..63b823a4 --- /dev/null +++ b/.github/workflows/cla.yml @@ -0,0 +1,94 @@ +# Contributor License Agreement (CLA.md). +# +# On a contributor's first pull request the bot comments with a link to +# CLA.md and the phrase to sign with. Commenting the phrase once records +# the signature in signatures/cla.json on the `cla-signatures` branch of +# this repository, and it covers every later pull request here. +# +# `pull_request_target` rather than `pull_request` because a PR from a +# fork gets a read-only token under the latter, and this workflow has to +# comment and write the signature file. That trigger runs with the base +# repository's token, so this workflow **must never check out or execute +# PR code**. It doesn't: no actions/checkout, no build; the action reads +# PR metadata and comments through the API. +# +# Two consequences of the triggers worth knowing: +# - `issue_comment` always runs the copy of this file on the default +# branch (`main`), so signing by comment takes effect immediately only +# once this file is on `main`. Until then a signature comment is picked +# up by the next run that `pull_request_target` starts (a push to the PR). +# - Only `GITHUB_TOKEN` is used: no personal access token, no +# cross-repository token. That is why signatures live in this repository. +name: CLA + +on: + pull_request_target: + types: [opened, synchronize, reopened, closed] + issue_comment: + types: [created] + +# What the action calls: comments and the post-merge lock (pull-requests), +# the signature file (contents), and re-running the failed check after a +# signature (actions). It never uses the commit status API, so no +# `statuses` here. +permissions: + actions: write + contents: write + pull-requests: write + +jobs: + cla: + runs-on: ubuntu-latest + # Pull request events, except a close without a merge (nothing to lock), + # and comments on pull requests that are exactly the signature or + # `recheck`. Expression comparisons ignore case. + if: >- + (github.event_name == 'pull_request_target' && + (github.event.action != 'closed' || github.event.pull_request.merged)) || + (github.event_name == 'issue_comment' && github.event.issue.pull_request && + (github.event.comment.body == 'recheck' || + github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA')) + steps: + # The action cannot start the record itself: it compares the 404 it + # gets for a missing file with the string "404", so a missing branch + # or file fails every run. Create the branch once, as an orphan that + # holds only the signature file. No workflow lives on it, and writes + # made with GITHUB_TOKEN start no workflow runs anyway, so recording + # a signature triggers no CI. + - name: Create the signature branch if missing + if: github.event.action != 'closed' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + BRANCH: cla-signatures + run: | + set -euo pipefail + if gh api "repos/$REPO/branches/$BRANCH" --silent 2>/dev/null; then + exit 0 + fi + tree=$(jq -n '{tree: [{path: "signatures/cla.json", mode: "100644", type: "blob", + content: "{\n \"signedContributors\": []\n}\n"}]}' | + gh api "repos/$REPO/git/trees" --input - --jq .sha) + commit=$(jq -n --arg tree "$tree" \ + '{message: "Start the CLA signature record", tree: $tree, parents: []}' | + gh api "repos/$REPO/git/commits" --input - --jq .sha) + # A concurrent run may have created it in the meantime; that is fine. + gh api "repos/$REPO/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$commit" --silent || + gh api "repos/$REPO/branches/$BRANCH" --silent + + - name: Check signatures + # v2.6.1 + uses: contributor-assistant/github-action@ca4a40a7d1004f18d9960b404b97e5f30a505a08 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + path-to-document: https://github.com/ThinkWatchProject/ThinkWatch/blob/main/CLA.md + branch: cla-signatures + path-to-signatures: signatures/cla.json + custom-pr-sign-comment: I have read the CLA Document and I hereby sign the CLA + # The maintainer, and bots, which cannot sign. `*[bot]` is a + # wildcard the action supports; the named bots are listed anyway. + allowlist: fylorn,dependabot[bot],renovate[bot],github-actions[bot],*[bot] + # Lock the conversation after a merge, as the action recommends, + # so the signature comments on it stay as they were. + lock-pullrequest-aftermerge: true diff --git a/CHANGELOG.md b/CHANGELOG.md index c270990c..9d46b912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,103 @@ target. ## [Unreleased] +## [3.7.0] — 2026-10-12 + +Outbound redaction keeps away from the structure of a request: it no longer +looks inside field names, and custom rules leave alone the identifiers a +protocol needs as they are, so a rule can no longer break the ids a client +sends back with its history. A custom redaction rule now says whether it +looks for its text as written or for a regular expression, and whether case +matters. The thinkwatch-core crates move from v0.70.0 to v0.71.0. + +### Read before upgrading + +- **Custom redaction rules saved before 3.7.0 now match their text as + written, ignoring case.** A custom rule in `security.redact` takes two new + optional keys, `match` (`contains` or `regex`) and `case_sensitive`. A + rule without `match` is looked for as plain text, in which `.`, `*`, `\d` + and the like have no special meaning; a rule without `case_sensitive` + ignores case. Until now every custom redaction rule was a case-sensitive + regular expression, and no rule saved by 3.0 to 3.6 carries either key, + including the rules an earlier upgrade converted from the old + `security.pii_redactor_patterns` setting. A rule written as a regular + expression still loads and nothing reports it, but it stops matching: + what it used to replace is sent upstream as it is and, with + `audit.body_redact_pii` on, stored unmasked. Nothing is converted + automatically. **Once every replica runs 3.7.0, open each custom + redaction rule on the console's security page, set Match by to Regex + for one written as a regular expression, and tick Case-sensitive where + case mattered.** A policy written by a script or through + `PATCH /api/admin/settings` needs `"match": "regex"` and + `"case_sensitive": true` on each such rule. Settings still in the format + from before 3.0.0, converted on the first start of 3.7.0, become + case-sensitive regex rules and keep matching as before. +- **Upgrade every replica before editing redaction rules, and do not roll + back afterwards.** 3.6 refuses a redaction policy with keys it does not + know. A 3.6 instance that reads a policy in which any custom rule has + `match` or `case_sensitive` logs `Unreadable guard policy` and runs + redaction on the factory policy: observe mode, the built-in rules as + shipped, no custom rules, so nothing is replaced on the wire. Any change + to outbound redaction saved from the 3.7.0 console, even a mode switch, + writes `match` on every custom rule. During a rolling upgrade, + edit redaction rules only once no 3.6 instance is left; to roll back to + 3.6 after editing them, first remove `match` and `case_sensitive` from + every custom rule in `security.redact`. +- **Redaction no longer searches field names, and custom rules skip + protocol identifiers.** No rule looks inside a JSON key any more. Custom + rules also leave alone the values of `id`, `call_id`, `item_id`, + `tool_use_id`, `tool_call_id`, `toolUseId`, `previous_response_id`, + `response_id`, `type`, `role`, `name`, `namespace`, `model`, `status`, + `object` and `phase`. Built-in rules still search those values, and every + other value is searched by every rule as before, including fields such as + `user_id` or `aws_access_key_id` inside tool arguments. A custom rule + meant to catch a value under one of those keys, such as a tool name or a + model name, no longer does. The same holds for masking stored bodies + (`audit.body_redact_pii`). +- **The private-key rule replaces less.** The header of a key block and its + end line are now looked for within the same text (see Fixed). A header + in one message and an end line in another are no longer replaced, along + with everything between them, as one key. + +### Added + +- **Match by and Case-sensitive for custom redaction rules.** The custom + rule dialog for outbound redaction offers Contains and Regex, as the + content filter's does, and a Case-sensitive checkbox. Contains looks for + the text exactly as written; leading and trailing spaces count. The rule + list says how each rule matches, such as "Contains `x`, ignoring case" or + "Regex `x`, case-sensitive". Trying a sample in the dialog uses the same + settings: `POST /api/admin/security/redact/test` takes `match` and + `case_sensitive`. Code points remain a content filter matcher and are + refused for redaction. + +### Fixed + +- **Custom redaction rules no longer break the ids a client sends back.** + Rules ran over the whole request body, field names and identifiers + included, so a custom rule could match inside an id: Codex sends ids such + as `fco_` followed by a hexadecimal UUID, and a rule whose text can be + spelled in hexadecimal turned one into `fco_…<>…`. The + upstream rejected the request, and since the history is sent again on + every turn, every later turn of that conversation as well. See Read + before upgrading for what is no longer searched. +- **Private keys stay within one text.** The private-key rule looked for + the end of a key block anywhere after its header. A header without an + end in one message, such as a file read halfway or code that spells the + marker, joined up with an end line in a later message, and everything in + between, earlier turns included, was replaced by a single placeholder. + +### Changed + +- thinkwatch-core crates (tw-bedrock, tw-breaker, tw-dialect, tw-guard) + v0.70.0 → v0.71.0. +- The Licensor in `LICENSE` is now the project's owner, named in the file, + instead of "ThinkWatch Contributors"; the Business Source License 1.1 + terms are unchanged. +- A first pull request to this repository asks its author to sign the + Contributor License Agreement (`CLA.md`) once, by a comment on the pull + request; `CONTRIBUTING.md` explains how. + ## [3.6.0] — 2026-10-12 A key shown once can go straight into ThinkWatch Lite: the console's key @@ -2022,7 +2119,8 @@ unreleased builds should: stop the gateway, run `db/schema.sql` against PostgreSQL, restart against this tag. The schema is idempotent end-to-end, so the apply is safe to repeat. -[Unreleased]: https://github.com/ThinkWatchProject/ThinkWatch/compare/v3.6.0...HEAD +[Unreleased]: https://github.com/ThinkWatchProject/ThinkWatch/compare/v3.7.0...HEAD +[3.7.0]: https://github.com/ThinkWatchProject/ThinkWatch/releases/tag/v3.7.0 [3.6.0]: https://github.com/ThinkWatchProject/ThinkWatch/releases/tag/v3.6.0 [3.5.0]: https://github.com/ThinkWatchProject/ThinkWatch/releases/tag/v3.5.0 [3.4.0]: https://github.com/ThinkWatchProject/ThinkWatch/releases/tag/v3.4.0 diff --git a/CLA.md b/CLA.md new file mode 100644 index 00000000..712617e8 --- /dev/null +++ b/CLA.md @@ -0,0 +1,140 @@ +# ThinkWatch Contributor License Agreement + +Version 1.0 + +This agreement (the "Agreement") sets out the terms on which You contribute to +the ThinkWatch Project. Please read it before You sign. A Chinese translation is +in [CLA.zh-CN.md](CLA.zh-CN.md); the English text governs (section 12). + +## 1. Definitions + +- **"Licensor"** means the holder of the GitHub account @fylorn + (https://github.com/fylorn), and that holder's heirs, successors and assigns. +- **"You"** means the individual who accepts this Agreement. +- **"Covered Repositories"** means these repositories of the ThinkWatch Project, + including any repository one of them is later moved or renamed to: + - ThinkWatch: https://github.com/ThinkWatchProject/ThinkWatch + - ThinkWatch Lite: https://github.com/ThinkWatchProject/ThinkWatch-Lite + - ThinkWatch Core: https://github.com/ThinkWatchProject/ThinkWatch-Core +- **"Project"** means the software, documentation and other material in the + Covered Repositories, in every edition and version, and any product or + service the Licensor makes from them. +- **"Submit"** means to send material to a Covered Repository or to the + Licensor for inclusion in the Project, for example by opening a pull + request, pushing commits to a pull request, or posting a patch or proposed + text in an issue or review comment. +- **"Contribution"** means any work of authorship that You Submit, such as + code, documentation, images or translations, including changes to existing + material. Material that You clearly mark as "Not a Contribution" when You + Submit it is not a Contribution. + +## 2. You keep Your copyright + +You keep the copyright and all other rights in Your Contributions, except for +the licences You grant in this Agreement. Nothing in this Agreement transfers +ownership of Your copyright to the Licensor. + +## 3. Copyright licence + +You grant the Licensor a perpetual, worldwide, non-exclusive, royalty-free, +irrevocable licence to use, reproduce, modify, prepare derivative works of, +publicly display, publicly perform, distribute and sublicense Your +Contributions and derivative works of them. + +This licence includes the right to license and relicense Your Contributions, +on their own or as part of the Project, under any terms the Licensor chooses. +Those terms may be open source licences such as the MIT License, +source-available licences such as the Business Source License, or proprietary +and commercial licences. The Licensor may move Your Contributions between +editions of the Project, may charge for licences to the Project, and does not +owe You any part of the income. + +## 4. Patent licence + +You grant the Licensor, and everyone who receives the Project or any part of it +from the Licensor or under a licence granted by the Licensor, a perpetual, +worldwide, non-exclusive, royalty-free patent licence to make, have made, use, +offer to sell, sell, import and otherwise transfer Your Contributions and the +Project. This licence is irrevocable, except as stated in the last paragraph of +this section. + +The patent licence covers only those patent claims that You are able to +license and that are necessarily infringed by Your Contribution on its own, or +by Your Contribution combined with the Project as it stood when You Submitted +it. + +If anyone starts patent litigation, including a cross-claim or counterclaim, +alleging that Your Contribution or the Project directly or indirectly infringes +a patent, every patent licence granted to that party under this Agreement for +that Contribution or the Project ends on the date the litigation is filed. + +## 5. What You confirm + +By accepting this Agreement, You confirm that: + +1. **You are entitled to grant these licences.** If Your employer, a client or + anyone else has rights in work You create, including Your Contributions, + You have their permission to make Your Contributions under this Agreement, + or they have waived those rights in Your Contributions. +2. **Each Contribution is Your original work,** apart from material You + identify as coming from someone else under point 3. +3. **You identify material from others.** If You Submit anything that is not + entirely Your original work, You say so when You Submit it (for example in + the pull request description), and You name its source, its licence and + any other restriction on it that You know of, including related patents + and trademarks. + +## 6. The Licensor may transfer these rights + +The Licensor may assign or transfer the rights granted under this Agreement, in +whole or in part, to any person or entity, for example a company formed later +to run the Project. The grant then benefits the assignee in the same way it +benefited the Licensor. You do not need to be asked or notified. + +## 7. No obligation to use Your Contribution + +The Licensor is not required to use, merge, distribute or keep any +Contribution, and may change or remove it at any time. + +## 8. No warranty + +Except for what You confirm in section 5, and unless applicable law requires +otherwise or You agree otherwise in writing, You provide Your Contributions +"as is", without warranties or conditions of any kind, express or implied, +including warranties or conditions of title, non-infringement, merchantability +or fitness for a particular purpose. You are not expected to provide support +for Your Contributions. + +## 9. Telling the Licensor about inaccuracies + +If You learn of any fact or circumstance that would make anything You confirm +in section 5 untrue, You agree to tell the Licensor promptly, for example by +commenting on the pull request concerned or by opening an issue in the Covered +Repository. + +## 10. How You accept this Agreement + +You accept this Agreement by posting this comment, exactly, on a pull request +to a Covered Repository: + +> I have read the CLA Document and I hereby sign the CLA + +Your acceptance covers the Contributions in that pull request, including those +Submitted before You posted the comment, and everything You Submit to any +Covered Repository afterwards. Each Covered Repository keeps its own record of +acceptances, so its automated check may ask You to accept again on Your first +pull request there; accepting again does not change these terms. + +To record Your acceptance, the Covered Repository stores Your GitHub username, +Your GitHub user ID, the pull request number, the comment ID and the time of +the comment in a public file in that repository. + +## 11. Later versions + +The Licensor may publish a new version of this Agreement. A new version applies +only to Contributions You Submit after You accept it. + +## 12. Language + +This Agreement is written in English. The Chinese translation is provided for +convenience only; if the two differ, the English text governs. diff --git a/CLA.zh-CN.md b/CLA.zh-CN.md new file mode 100644 index 00000000..2e6fc378 --- /dev/null +++ b/CLA.zh-CN.md @@ -0,0 +1,80 @@ +# ThinkWatch 贡献者许可协议 + +第 1.0 版 + +> 本文是 [CLA.md](CLA.md) 的中文译本,仅供参考。两者如有不一致,以英文原文为准。 +> 签署时同意的是英文原文。 + +本协议(「本协议」)规定您向 ThinkWatch 项目作出贡献的条件。签署前请先阅读。 + +## 1. 定义 + +- **「许可方」**指 GitHub 账号 @fylorn(https://github.com/fylorn)的持有人,及其继承人、承继者和受让人。 +- **「您」**指接受本协议的个人。 +- **「适用仓库」**指 ThinkWatch 项目的下列仓库,包括其中任何一个日后迁移或改名后的仓库: + - ThinkWatch:https://github.com/ThinkWatchProject/ThinkWatch + - ThinkWatch Lite:https://github.com/ThinkWatchProject/ThinkWatch-Lite + - ThinkWatch Core:https://github.com/ThinkWatchProject/ThinkWatch-Core +- **「本项目」**指适用仓库中的软件、文档和其他材料,涵盖各个产品版本(如企业版和桌面版)和各个发布版本,也包括许可方以其制作的任何产品或服务。 +- **「提交」**指为纳入本项目而把材料发送到适用仓库或发送给许可方,例如发起拉取请求、向拉取请求推送提交,或在议题、评审评论中贴出补丁或建议文字。 +- **「贡献」**指您提交的任何作品,例如代码、文档、图片或译文,包括对已有材料的修改。您在提交时明确标注为「Not a Contribution」(非贡献)的材料不属于贡献。 + +## 2. 版权仍归您所有 + +除本协议授予的许可外,您的贡献的版权及其他一切权利仍归您所有。本协议不会把您的版权转让给许可方。 + +## 3. 版权许可 + +您授予许可方一项永久、全球范围、非独占、免许可费、不可撤销的许可,可以使用、复制、修改、制作演绎作品、公开展示、公开表演、分发和再许可您的贡献及其演绎作品。 + +该许可包括:以许可方选定的任何条款,单独或作为本项目的一部分,对您的贡献进行许可和重新许可。这些条款可以是开源许可证(如 MIT 许可证)、源码可见许可证(如 Business Source License),也可以是专有许可或商业许可。许可方可以在本项目的不同产品版本之间转移您的贡献,可以对本项目的许可收费,且无需与您分享任何收入。 + +## 4. 专利许可 + +您授予许可方,以及从许可方处或依许可方授予的许可取得本项目或其任何部分的每一方,一项永久、全球范围、非独占、免许可费的专利许可,可以制造、委托制造、使用、许诺销售、销售、进口以及以其他方式转让您的贡献和本项目。除本节最后一段所述情形外,该许可不可撤销。 + +该专利许可仅涵盖您有权许可、且仅因您的贡献本身,或因您的贡献与您提交时的本项目相结合而必然被侵犯的专利权利要求。 + +如果任何一方提起专利诉讼(包括交叉请求或反诉),主张您的贡献或本项目直接或间接侵犯专利,则依本协议就该贡献或本项目授予该方的一切专利许可,自诉讼提起之日起终止。 + +## 5. 您确认的事项 + +接受本协议即表示您确认: + +1. **您有权授予上述许可。**如果您的雇主、客户或其他任何人对您创作的作品(包括您的贡献)享有权利,您已取得其许可,可以依本协议作出贡献,或其已放弃对您的贡献享有的这些权利。 +2. **每项贡献都是您的原创作品,**但您依第 3 点标明来自他人的材料除外。 +3. **您会标明来自他人的材料。**如果您提交的内容并非完全是您的原创作品,您会在提交时说明(例如写在拉取请求的描述里),并写明其来源、许可证,以及您所知道的任何其他限制,包括相关的专利和商标。 + +## 6. 许可方可以转让这些权利 + +许可方可以将本协议授予的权利全部或部分转让给任何个人或实体,例如日后为运营本项目而成立的公司。届时,本协议的授权以与此前惠及许可方相同的方式惠及受让人。转让无需征得您的同意,也无需通知您。 + +## 7. 无义务使用您的贡献 + +许可方没有义务使用、合并、分发或保留任何贡献,并可随时修改或删除。 + +## 8. 不提供担保 + +除您在第 5 节确认的事项外,除非适用法律另有要求或您另行书面同意,您的贡献按「现状」提供,不附带任何明示或默示的担保或条件,包括权利、不侵权、适销性或特定用途适用性方面的担保或条件。您无需为您的贡献提供支持。 + +## 9. 告知许可方不实之处 + +如果您得知任何事实或情况,使您在第 5 节确认的任何事项不再属实,您同意及时告知许可方,例如在相关拉取请求下评论,或在该适用仓库中开一个议题。 + +## 10. 如何接受本协议 + +在适用仓库的拉取请求下原样发表下面这条评论,即表示您接受本协议: + +> I have read the CLA Document and I hereby sign the CLA + +您的接受涵盖该拉取请求中的贡献(包括您发表评论之前提交的部分),以及您此后向任何适用仓库提交的全部内容。每个适用仓库各自记录接受情况,因此您在另一个适用仓库首次发起拉取请求时,自动检查可能会请您再接受一次;再次接受不会改变本协议的条款。 + +为记录您的接受,适用仓库会把您的 GitHub 用户名、GitHub 用户 ID、拉取请求编号、评论 ID 和评论时间保存在该仓库的一个公开文件中。 + +## 11. 后续版本 + +许可方可以发布本协议的新版本。新版本只适用于您在接受新版本之后提交的贡献。 + +## 12. 语言 + +本协议以英文写成。中文译本仅为方便阅读而提供;两者如有不一致,以英文原文为准。 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2308a33b..82a4e588 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,6 +28,25 @@ diverged too far to carry a fix cleanly. The full branch contract is in Automation follows the same rule — Renovate is pinned to `dev` via `baseBranches` in `renovate.json`. +## Sign the Contributor License Agreement once + +Your first pull request to this repository gets a comment from a bot +asking you to sign the [Contributor License Agreement](CLA.md) +([Chinese translation](CLA.zh-CN.md)). Read it, then reply on the pull +request with exactly: + +```text +I have read the CLA Document and I hereby sign the CLA +``` + +That is the only time. The signature is recorded in this repository and +covers all your later pull requests here. If the check stays red after +you sign, comment `recheck`. + +You keep the copyright in your contribution; the agreement grants the +project the right to license it under any terms, including commercial +ones. + ## Commit messages Conventional Commits (`fix(scope): subject`), because `git-cliff` diff --git a/Cargo.lock b/Cargo.lock index 1029fd31..d32b6b86 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4189,7 +4189,7 @@ dependencies = [ [[package]] name = "think-watch-auth" -version = "3.6.0" +version = "3.7.0" dependencies = [ "anyhow", "argon2", @@ -4219,7 +4219,7 @@ dependencies = [ [[package]] name = "think-watch-common" -version = "3.6.0" +version = "3.7.0" dependencies = [ "aes-gcm", "anyhow", @@ -4262,7 +4262,7 @@ dependencies = [ [[package]] name = "think-watch-gateway" -version = "3.6.0" +version = "3.7.0" dependencies = [ "anyhow", "arc-swap", @@ -4301,7 +4301,7 @@ dependencies = [ [[package]] name = "think-watch-mcp-gateway" -version = "3.6.0" +version = "3.7.0" dependencies = [ "anyhow", "arc-swap", @@ -4330,7 +4330,7 @@ dependencies = [ [[package]] name = "think-watch-server" -version = "3.6.0" +version = "3.7.0" dependencies = [ "anyhow", "arc-swap", @@ -4381,7 +4381,7 @@ dependencies = [ [[package]] name = "think-watch-test-support" -version = "3.6.0" +version = "3.7.0" dependencies = [ "anyhow", "async-stream", @@ -4837,8 +4837,8 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.70.0" -source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.70.0#205390ee6400126286a86b09c8b751f56368c439" +version = "0.71.0" +source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.71.0#911f9373bdbc4e39c37f25408aa802f3c4a32028" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -4852,16 +4852,16 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.70.0" -source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.70.0#205390ee6400126286a86b09c8b751f56368c439" +version = "0.71.0" +source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.71.0#911f9373bdbc4e39c37f25408aa802f3c4a32028" dependencies = [ "serde", ] [[package]] name = "tw-dialect" -version = "0.70.0" -source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.70.0#205390ee6400126286a86b09c8b751f56368c439" +version = "0.71.0" +source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.71.0#911f9373bdbc4e39c37f25408aa802f3c4a32028" dependencies = [ "memchr", "serde", @@ -4870,8 +4870,8 @@ dependencies = [ [[package]] name = "tw-guard" -version = "0.70.0" -source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.70.0#205390ee6400126286a86b09c8b751f56368c439" +version = "0.71.0" +source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.71.0#911f9373bdbc4e39c37f25408aa802f3c4a32028" dependencies = [ "base64 0.22.1", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 93d2705b..e5ece7da 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ ] [workspace.package] -version = "3.6.0" +version = "3.7.0" edition = "2024" # The MSRV: the newest `rust-version` among the locked dependencies # (sqlx 0.9 and the aws-smithy crates under aws-sigv4). Without it, @@ -70,10 +70,10 @@ opt-level = 3 # never re-exported through a local shim. And the reverse: something only # this side uses (the at-rest crypto, IMDSv2 credentials, the gateway error) # lives here, not in core. -tw-bedrock = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.70.0" } -tw-breaker = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.70.0" } -tw-dialect = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.70.0" } -tw-guard = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.70.0" } +tw-bedrock = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.71.0" } +tw-breaker = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.71.0" } +tw-dialect = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.71.0" } +tw-guard = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.71.0" } # Web framework axum = { version = "0.8", features = ["macros", "ws"] } diff --git a/LICENSE b/LICENSE index fc9b309f..88b8957a 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,8 @@ Business Source License 1.1 Parameters -Licensor: ThinkWatch Contributors +Licensor: The holder of the GitHub account @fylorn (https://github.com/fylorn), +and that holder's heirs, successors and assigns Licensed Work: ThinkWatch, including all source code, build scripts, and documentation in this repository, each version as released by the Licensor. diff --git a/README.md b/README.md index a3c083c9..7e757b8c 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ The gateway (port `3000`) is the only part that clients need to reach. The conso **Security guards** - There are three guards, each with three modes: off, observe, and one named for what it does — replace (outbound redaction), cut off (tool-call inspection) and enforce (content filter). A new installation starts all three in observe mode: hits go to the audit log and nothing is changed until a guard is switched to its third mode. - Every rule is listed on the console's security page, built-in and custom. Built-in rules can be switched on or off, tool-call and content rules can take another action, custom rules can be added, and a sample can be tried against one rule or a whole guard first. -- Outbound redaction searches the whole request, system prompt and earlier answers included, but not base64 payloads. A match becomes `<>` — `SECRET` for credentials, `ID_NUMBER`, `CARD_NUMBER`, `EMAIL` and `PHONE` for personal data, a label of its own for a custom rule — and is restored in the answer. E-mail addresses and phone numbers ship switched off. +- Outbound redaction searches the whole request, system prompt and earlier answers included, but not field names or base64 payloads. Custom rules also skip the values a protocol needs exactly as they are, such as ids, call ids, types, roles, tool names and the model name; built-in rules still search them. A custom rule looks for its text as written or for a regular expression, ignoring case unless it is set to be case-sensitive. A match becomes `<>` — `SECRET` for credentials, `ID_NUMBER`, `CARD_NUMBER`, `EMAIL` and `PHONE` for personal data, a label of its own for a custom rule — and is restored in the answer. E-mail addresses and phone numbers ship switched off. - A content rule matches a phrase, a regular expression or code points (`U+200B`, `U+E0000–U+E007F`), and either refuses the request, deletes what it matched from the caller's messages and tool results, or records only. Hidden characters are content rules: Unicode tag characters and bidirectional controls ship on, zero-width and private-use characters off. - A tool-call rule cuts the response at the call or records it. Besides the dangerous-command rules, two built-in rules catch a credential sent to an unknown host and a local file uploaded to an external host. - A model's maximum output tokens, set on the Models page, caps `max_tokens` on every request to that model; it replaces the old output length guardrail. @@ -123,6 +123,8 @@ ThinkWatch uses four crates from [ThinkWatch Core](https://github.com/ThinkWatch Contributions are welcome. Please open an issue to discuss before submitting a PR for major changes. +Your first pull request asks you to sign the [Contributor License Agreement](CLA.md) once; [CONTRIBUTING.md](CONTRIBUTING.md) explains how. + ## License ThinkWatch is source-available under the [Business Source License 1.1](LICENSE). diff --git a/README.zh-CN.md b/README.zh-CN.md index 8a103072..9cc41b82 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -86,7 +86,7 @@ cd web && pnpm install && pnpm dev **安全防护** - 三项防护各有三档:关闭、观察,以及按其作用命名的第三档——出站脱敏为「替换」,工具调用审查为「切断」,内容过滤为「处置」。新安装时三项都处于观察档:命中只写入审计日志,切换到第三档之前不改动任何请求。 - 控制台的安全页列出每一条规则,内置规则和自定义规则都在其中。内置规则可以启用或停用,工具调用规则和内容规则可以改变处置,也可以添加自定义规则;切换之前可先用一段文本测试单条规则或整项防护。 -- 出站脱敏查找整个请求,包括系统提示和之前的回答,但不查 base64 载荷。命中的内容替换为 `<>`(凭据为 `SECRET`,个人信息为 `ID_NUMBER`、`CARD_NUMBER`、`EMAIL`、`PHONE`,自定义规则使用自己的标签),并在回答中还原。邮箱地址和手机号规则出厂关闭。 +- 出站脱敏查找整个请求,包括系统提示和之前的回答,但不查字段名和 base64 载荷。自定义规则还会跳过协议要求原样保留的值,如各类 id、调用 id、类型、角色、工具名和模型名;内置规则照常查找这些值。自定义规则按原文或按正则表达式查找,默认不区分大小写,也可设为区分大小写。命中的内容替换为 `<>`(凭据为 `SECRET`,个人信息为 `ID_NUMBER`、`CARD_NUMBER`、`EMAIL`、`PHONE`,自定义规则使用自己的标签),并在回答中还原。邮箱地址和手机号规则出厂关闭。 - 内容规则按关键词、正则表达式或码位(`U+200B`、`U+E0000–U+E007F`)匹配,命中后拒绝请求、从调用方消息和工具结果中删除命中的文字,或仅记录。隐藏字符属于内容规则:Unicode 标签字符和双向控制符出厂开启,零宽字符和私用区字符出厂关闭。 - 工具调用规则在该调用处切断响应,或仅记录。除危险命令规则外,另有「发送凭据到陌生主机」和「上传本地文件到外部主机」两条内置规则。 - 模型的「最大输出 token」在模型页设置,限制发往该模型的每个请求的 `max_tokens`,取代原来的输出长度护栏。 @@ -123,6 +123,8 @@ ThinkWatch 使用 [ThinkWatch Core](https://github.com/ThinkWatchProject/ThinkWa 欢迎贡献。提交大型变更前请先开 Issue 讨论。 +首次提交 PR 时需签署一次[贡献者许可协议](CLA.zh-CN.md),签署方法见 [CONTRIBUTING.md](CONTRIBUTING.md)。 + ## 授权协议 ThinkWatch 采用 [Business Source License 1.1](LICENSE) 进行源码可见分发。 diff --git a/crates/common/src/guard_policy/legacy.rs b/crates/common/src/guard_policy/legacy.rs index 568fdcd8..5620cfd3 100644 --- a/crates/common/src/guard_policy/legacy.rs +++ b/crates/common/src/guard_policy/legacy.rs @@ -493,6 +493,10 @@ pub fn redact(patterns: Option<&Value>) -> RedactPolicy { policy.custom.push(CustomRedactRule { name: names.unique(&name), pattern: p.regex.clone(), + // The old runtime compiled each pattern as a + // case-sensitive regex. + matching: ContentMatch::Regex, + case_sensitive: true, label: label_of(&p.placeholder_prefix), disabled: false, }); @@ -973,6 +977,10 @@ mod tests { .map(|c| (c.name.as_str(), c.label.as_deref())) .collect(); assert_eq!(custom, [("phone_us", Some("PHONE")), ("ipv4", Some("IP"))]); + // Matched the way the old runtime did: a case-sensitive regex. + for c in &p.custom { + assert_eq!((c.matching, c.case_sensitive), (ContentMatch::Regex, true)); + } let rules = p.rules().unwrap(); for id in ["cn-resident-id", "bank-card", "email", "cn-mobile-phone"] { diff --git a/crates/common/src/guard_policy/mod.rs b/crates/common/src/guard_policy/mod.rs index 408fb89d..7530c876 100644 --- a/crates/common/src/guard_policy/mod.rs +++ b/crates/common/src/guard_policy/mod.rs @@ -374,8 +374,8 @@ mod tests { assert!(ids.contains(&"curl-pipe-sh".to_string())); let redact: RedactPolicy = serde_json::from_value(json!({"custom": [ - {"name": "bad", "pattern": "("}, - {"name": "good", "pattern": "PRJ-\\d+"} + {"name": "bad", "pattern": "(", "match": "regex"}, + {"name": "good", "pattern": "PRJ-\\d+", "match": "regex"} ]})) .unwrap(); let rules = redact_rules(&redact); diff --git a/crates/common/src/pii.rs b/crates/common/src/pii.rs index bd37fd0b..7f01cce9 100644 --- a/crates/common/src/pii.rs +++ b/crates/common/src/pii.rs @@ -225,6 +225,8 @@ mod tests { custom: vec![CustomRedactRule { name: "project".into(), pattern: r"PRJ-\d{6}".into(), + matching: tw_guard::policy::ContentMatch::Regex, + case_sensitive: true, label: None, disabled: false, }], diff --git a/crates/gateway/src/redaction.rs b/crates/gateway/src/redaction.rs index 50a1b7a3..4ce35dfe 100644 --- a/crates/gateway/src/redaction.rs +++ b/crates/gateway/src/redaction.rs @@ -321,6 +321,8 @@ mod tests { custom: vec![tw_guard::policy::CustomRedactRule { name: "ssn".into(), pattern: r"\d{3}-\d{2}-\d{4}".into(), + matching: tw_guard::policy::ContentMatch::Regex, + case_sensitive: true, label: None, disabled: false, }], diff --git a/crates/server/src/handlers/admin/security.rs b/crates/server/src/handlers/admin/security.rs index d45c0495..6d63abc5 100644 --- a/crates/server/src/handlers/admin/security.rs +++ b/crates/server/src/handlers/admin/security.rs @@ -78,7 +78,7 @@ pub async fn get_security( ), request_body( content = inline(serde_json::Value), - description = "sample: string; pattern?, match?, rule?, label?, action? (thinkwatch-core's SecurityTestRequest)", + description = "sample: string; pattern?, match?, case_sensitive?, rule?, label?, action? (thinkwatch-core's SecurityTestRequest)", ), responses( (status = 200, description = "Hits, output and refusal (thinkwatch-core's SecurityTestResult)", body = serde_json::Value), diff --git a/crates/test-support/tests/content_filter_pii.rs b/crates/test-support/tests/content_filter_pii.rs index 71e083bb..004e1a13 100644 --- a/crates/test-support/tests/content_filter_pii.rs +++ b/crates/test-support/tests/content_filter_pii.rs @@ -364,7 +364,7 @@ async fn redaction_replaces_values_in_the_whole_request_and_restores_the_answer( app.set_setting( "security.redact", json!({"mode": "enforce", "custom": [ - {"name": "ssn", "pattern": "\\d{3}-\\d{2}-\\d{4}", "label": "SSN"} + {"name": "ssn", "pattern": "\\d{3}-\\d{2}-\\d{4}", "match": "regex", "label": "SSN"} ]}), ) .await; @@ -438,7 +438,7 @@ async fn a_custom_redaction_rule_records_one_event_with_counts_and_no_values() { let app = TestApp::spawn_with_clickhouse().await; app.set_setting( "security.redact", - json!({"custom": [{"name": "ssn", "pattern": "\\d{3}-\\d{2}-\\d{4}"}]}), + json!({"custom": [{"name": "ssn", "pattern": "\\d{3}-\\d{2}-\\d{4}", "match": "regex"}]}), ) .await; let upstream = echo_upstream().await; diff --git a/crates/test-support/tests/guard_settings_upgrade.rs b/crates/test-support/tests/guard_settings_upgrade.rs index 796da651..e3b819a0 100644 --- a/crates/test-support/tests/guard_settings_upgrade.rs +++ b/crates/test-support/tests/guard_settings_upgrade.rs @@ -221,8 +221,10 @@ async fn the_old_settings_are_converted_once_and_behave_as_before() { "mode": "enforce", "enable": ["email", "cn-mobile-phone"], "custom": [ - {"name": "phone_us", "pattern": "\\b\\d{3}[-.]?\\d{3}[-.]?\\d{4}\\b", "label": "PHONE"}, - {"name": "ipv4", "pattern": "\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\b", "label": "IP"} + {"name": "phone_us", "pattern": "\\b\\d{3}[-.]?\\d{3}[-.]?\\d{4}\\b", + "match": "regex", "case_sensitive": true, "label": "PHONE"}, + {"name": "ipv4", "pattern": "\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\b", + "match": "regex", "case_sensitive": true, "label": "IP"} ] }) ); diff --git a/deploy/helm/think-watch/Chart.yaml b/deploy/helm/think-watch/Chart.yaml index f89b3004..030e055b 100644 --- a/deploy/helm/think-watch/Chart.yaml +++ b/deploy/helm/think-watch/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: think-watch description: Enterprise AI API Gateway & MCP Management Platform type: application -version: 3.6.0 -appVersion: "3.6.0" +version: 3.7.0 +appVersion: "3.7.0" keywords: - ai - gateway diff --git a/web/package.json b/web/package.json index 063ae0da..7cebe3ee 100644 --- a/web/package.json +++ b/web/package.json @@ -1,7 +1,7 @@ { "name": "web", "private": true, - "version": "3.6.0", + "version": "3.7.0", "type": "module", "packageManager": "pnpm@12.9.1", "scripts": { diff --git a/web/src/i18n/en.json b/web/src/i18n/en.json index c2c86fa8..404fa3b6 100644 --- a/web/src/i18n/en.json +++ b/web/src/i18n/en.json @@ -235,10 +235,12 @@ "regex": "Regex", "codepoints": "Code points" }, + "caseSensitive": "Case-sensitive", "patternContains": "Text to find", "patternRegex": "Regular expression", "patternCodepoints": "Code points", "containsHint": "Searched for in user messages and tool results, ignoring case. Leading and trailing spaces count.", + "redactContainsHint": "Every occurrence of this text is replaced with a placeholder. It is found as written: characters such as . and * have no special meaning, and leading and trailing spaces count.", "codepointsHint": "A code point or a range of them; separate several with commas, as in U+200B, U+E0000–U+E007F", "regexHint": { "redact": "Everything the pattern matches is replaced with a placeholder.", @@ -293,8 +295,10 @@ "bankCard": "A {{networks}} card number whose prefix and length match and that passes the Luhn check; public test card numbers excepted", "email": "Email addresses, such as {{example}}", "cnMobilePhone": "11-digit mobile numbers starting with 13–19, not run together with other digits", - "regex": "Regex {{pattern}}", + "regex": "Regex {{pattern}}, ignoring case", + "regexCaseSensitive": "Regex {{pattern}}, case-sensitive", "contains": "Contains {{text}}, ignoring case", + "containsCaseSensitive": "Contains {{text}}, case-sensitive", "codepoints": "Code points {{ranges}}", "other": "Built-in check" }, diff --git a/web/src/i18n/zh.json b/web/src/i18n/zh.json index d2fdfcea..294a1fb6 100644 --- a/web/src/i18n/zh.json +++ b/web/src/i18n/zh.json @@ -235,10 +235,12 @@ "regex": "正则", "codepoints": "码位" }, + "caseSensitive": "区分大小写", "patternContains": "匹配内容", "patternRegex": "正则表达式", "patternCodepoints": "码位", "containsHint": "在用户消息和工具结果中查找这段文字,不区分大小写,首尾空格也参与匹配。", + "redactContainsHint": "这段文字出现的每一处都替换为占位符。按原样查找:. 和 * 等字符没有特殊含义,首尾空格也参与匹配。", "codepointsHint": "码位或码位范围,多个之间用逗号分隔,如 U+200B, U+E0000–U+E007F", "regexHint": { "redact": "匹配到的整段内容替换为占位符。", @@ -293,8 +295,10 @@ "bankCard": "{{networks}} 的卡号:号段、位数对得上并通过 Luhn 校验;公开的测试卡号除外", "email": "邮箱地址,如 {{example}}", "cnMobilePhone": "11 位手机号,以 13–19 开头,前后不紧挨其他数字", - "regex": "正则 {{pattern}}", + "regex": "正则 {{pattern}},不区分大小写", + "regexCaseSensitive": "正则 {{pattern}},区分大小写", "contains": "包含 {{text}},不区分大小写", + "containsCaseSensitive": "包含 {{text}},区分大小写", "codepoints": "码位 {{ranges}}", "other": "内置判据" }, diff --git a/web/src/lib/security-types.ts b/web/src/lib/security-types.ts index 36029820..bac65286 100644 --- a/web/src/lib/security-types.ts +++ b/web/src/lib/security-types.ts @@ -37,6 +37,9 @@ export type ContentAction = 'block' | 'strip' | 'record'; /** How a content rule matches. */ export type ContentMatch = 'contains' | 'regex' | 'codepoints'; +/** How a custom redaction rule matches: the text as written, or a regex. */ +export type RedactMatch = Exclude; + export interface CardPrefix { from: number; to: number; @@ -51,7 +54,9 @@ export interface CardNetwork { /** * What a rule matches, for display. Built-in rules use the specific kinds; - * custom rules are `regex`, `contains` or `codepoints`. A built-in rule's + * custom rules are `regex`, `contains` or `codepoints`. `case_sensitive` + * is left out when false: tool-call regexes are case-sensitive, content + * rules are not, a custom redaction rule is when it says so. A built-in rule's * `codepoints.ranges` is the canonical spelling (`U+200B`, * `U+E0000–U+E007F`); a custom rule's is the items as written, so joining * them with `, ` gives back what is stored. `builtin` is a check @@ -70,8 +75,8 @@ export type Matcher = | { kind: 'bank-card'; networks: CardNetwork[] } | { kind: 'email' } | { kind: 'cn-mobile-phone' } - | { kind: 'regex'; pattern: string } - | { kind: 'contains'; text: string } + | { kind: 'regex'; pattern: string; case_sensitive?: boolean } + | { kind: 'contains'; text: string; case_sensitive?: boolean } | { kind: 'codepoints'; ranges: string[] } | { kind: 'builtin'; check: string }; @@ -117,14 +122,18 @@ export interface SecurityDetail { /** * `POST /api/admin/security/{guard}/test`. With `pattern` only that pattern - * is tried (content: matched by `match`, redaction: replaced under `label`); + * is tried (content and redaction: matched by `match`, redaction also by + * `case_sensitive` and replaced under `label`; tool calls: a regex); * with `rule` only that built-in rule, even when it is off; with neither, * every rule that is on. */ export interface SecurityTestRequest { sample: string; pattern?: string | null; + /** Left out: `contains`. Redaction takes `contains` or `regex` only. */ match?: ContentMatch | null; + /** Redaction only. Left out: ignoring case. */ + case_sensitive?: boolean | null; rule?: string | null; label?: string | null; /** @@ -173,8 +182,12 @@ export interface SecurityTestResult { export interface CustomRedactRule { /** Also the rule's identity: unique within the guard. */ name: string; - /** Regular expression. */ + /** The text to find, or a regular expression, as `match` says. */ pattern: string; + /** Left out: `contains` — the text as written, `.` and `*` included. */ + match?: RedactMatch; + /** Left out: ignoring case. */ + case_sensitive?: boolean; /** Placeholder label, `^[A-Z][A-Z0-9_]{0,23}$`. Left out: `SECRET`. */ label?: string; disabled?: boolean; diff --git a/web/src/routes/gateway/security/matcher-text.tsx b/web/src/routes/gateway/security/matcher-text.tsx index 91e920a6..cd557e40 100644 --- a/web/src/routes/gateway/security/matcher-text.tsx +++ b/web/src/routes/gateway/security/matcher-text.tsx @@ -110,9 +110,18 @@ export function MatcherText({ m }: { m: Matcher }) { case 'cn-mobile-phone': return t('contentSecurity.matcher.cnMobilePhone'); case 'regex': - return fill(t('contentSecurity.matcher.regex', RAW), { pattern: {m.pattern} }); + return fill( + t(m.case_sensitive ? 'contentSecurity.matcher.regexCaseSensitive' : 'contentSecurity.matcher.regex', RAW), + { pattern: {m.pattern} }, + ); case 'contains': - return fill(t('contentSecurity.matcher.contains', RAW), { text: {m.text} }); + return fill( + t( + m.case_sensitive ? 'contentSecurity.matcher.containsCaseSensitive' : 'contentSecurity.matcher.contains', + RAW, + ), + { text: {m.text} }, + ); case 'codepoints': return fill(t('contentSecurity.matcher.codepoints', RAW), { ranges: , diff --git a/web/src/routes/gateway/security/policy.test.ts b/web/src/routes/gateway/security/policy.test.ts index a50adcf6..c399ab29 100644 --- a/web/src/routes/gateway/security/policy.test.ts +++ b/web/src/routes/gateway/security/policy.test.ts @@ -4,6 +4,7 @@ import { checkCodepoints, customRuleView, isLabel, + patternOf, placeholder, policyOf, withAction, @@ -41,6 +42,7 @@ const content: GuardDetail = { name: 'Project X', pattern: 'project-x', match: 'contains', + caseSensitive: false, action: 'strip', label: null, enabled: true, @@ -49,6 +51,7 @@ const content: GuardDetail = { name: 'Tags', pattern: 'U+E0000–U+E007F, U+200B', match: 'codepoints', + caseSensitive: false, action: 'block', label: null, enabled: false, @@ -88,8 +91,9 @@ describe('policyOf', () => { rules: [ builtin({ id: 'internal-ip', kind: 'internal', matcher: { kind: 'private-ip' }, on_by_default: false, label: 'SECRET' }), builtin({ id: 'jwt', kind: 'jwt', matcher: { kind: 'jwt' }, enabled: false, label: 'SECRET' }), - customRuleView({ name: 'Project', pattern: 'PRJ-\\d{6}', match: 'regex', action: null, label: 'PROJECT', enabled: true }), - { ...customRuleView({ name: 'Ticket', pattern: 'T-\\d+', match: 'regex', action: null, label: null, enabled: true }), label: null }, + customRuleView({ name: 'Project', pattern: 'PRJ-\\d{6}', match: 'regex', caseSensitive: true, action: null, label: 'PROJECT', enabled: true }), + { ...customRuleView({ name: 'Ticket', pattern: 'T-\\d+', match: 'regex', caseSensitive: false, action: null, label: null, enabled: true }), label: null }, + customRuleView({ name: 'Codename', pattern: 'blue.fox', match: 'contains', caseSensitive: false, action: null, label: null, enabled: false }), ], } expect(policyOf('redact', d)).toEqual({ @@ -97,19 +101,38 @@ describe('policyOf', () => { enable: ['internal-ip'], disable: ['jwt'], custom: [ - { name: 'Project', pattern: 'PRJ-\\d{6}', label: 'PROJECT' }, - { name: 'Ticket', pattern: 'T-\\d+', label: 'SECRET' }, + { name: 'Project', pattern: 'PRJ-\\d{6}', match: 'regex', case_sensitive: true, label: 'PROJECT' }, + { name: 'Ticket', pattern: 'T-\\d+', match: 'regex', label: 'SECRET' }, + { name: 'Codename', pattern: 'blue.fox', match: 'contains', label: 'SECRET', disabled: true }, ], }) }) + it('reads how a custom redaction rule matches back from the view', () => { + const view = (matcher: SecurityRuleView['matcher']): SecurityRuleView => ({ + ...customRuleView({ name: 'r', pattern: '', match: 'contains', caseSensitive: false, action: null, label: null, enabled: true }), + matcher, + }) + expect(patternOf(view({ kind: 'contains', text: 'a.b' }))).toEqual({ pattern: 'a.b', match: 'contains', caseSensitive: false }) + expect(patternOf(view({ kind: 'regex', pattern: 'a+', case_sensitive: true }))).toEqual({ + pattern: 'a+', + match: 'regex', + caseSensitive: true, + }) + expect(customRuleView({ name: 'r', pattern: 'x', match: 'regex', caseSensitive: true, action: null, label: null, enabled: true }).matcher).toEqual({ + kind: 'regex', + pattern: 'x', + case_sensitive: true, + }) + }) + it('writes tool-call actions and custom rules', () => { const d: GuardDetail = { mode: 'observe', rules: [ builtin({ id: 'rm-rf-root', kind: 'command', matcher: { kind: 'regex', pattern: 'rm' }, action: 'record', default_action: 'cut' }), builtin({ id: 'chmod-777', kind: 'command', matcher: { kind: 'regex', pattern: 'chmod' }, action: 'cut', default_action: 'cut' }), - customRuleView({ name: 'kubectl', pattern: 'kubectl\\s+delete', match: 'regex', action: 'cut', label: null, enabled: true }), + customRuleView({ name: 'kubectl', pattern: 'kubectl\\s+delete', match: 'regex', caseSensitive: false, action: 'cut', label: null, enabled: true }), ], } expect(policyOf('inspect_tools', d)).toEqual({ @@ -137,7 +160,7 @@ describe('edits', () => { mode: 'observe', rules: [ builtin({ id: 'dan', action: 'block', default_action: 'block' }), - customRuleView({ name: 'dan', pattern: 'dan', match: 'contains', action: 'record', label: null, enabled: true }), + customRuleView({ name: 'dan', pattern: 'dan', match: 'contains', caseSensitive: false, action: 'record', label: null, enabled: true }), ], } const next = withEnabled(d, { id: 'dan', custom: true }, false) @@ -148,14 +171,14 @@ describe('edits', () => { const renamed = withCustom( content, 'Project X', - customRuleView({ name: 'Project Y', pattern: 'project-y', match: 'regex', action: 'block', label: null, enabled: true }), + customRuleView({ name: 'Project Y', pattern: 'project-y', match: 'regex', caseSensitive: false, action: 'block', label: null, enabled: true }), ) expect(renamed.rules.filter((r) => r.custom).map((r) => r.id)).toEqual(['Project Y', 'Tags']) const added = withCustom( content, null, - customRuleView({ name: 'New', pattern: 'x', match: 'contains', action: 'record', label: null, enabled: true }), + customRuleView({ name: 'New', pattern: 'x', match: 'contains', caseSensitive: false, action: 'record', label: null, enabled: true }), ) expect(added.rules.at(-1)?.id).toBe('New') expect(withoutCustom(added, 'New').rules).toHaveLength(content.rules.length) diff --git a/web/src/routes/gateway/security/policy.ts b/web/src/routes/gateway/security/policy.ts index 333b588c..6d6881c0 100644 --- a/web/src/routes/gateway/security/policy.ts +++ b/web/src/routes/gateway/security/policy.ts @@ -20,6 +20,7 @@ import type { GuardMode, GuardPolicies, Matcher, + RedactMatch, RedactPolicy, RuleAction, SecurityRuleView, @@ -111,37 +112,56 @@ export function checkCodepoints(text: string): CodepointProblem | null { // Matchers // --------------------------------------------------------------------------- +/** What a custom rule writes. */ +export interface WrittenPattern { + pattern: string; + match: ContentMatch; + /** Only redaction rules choose; the view says it for every regex and text. */ + caseSensitive: boolean; +} + /** What a rule's matcher says, as a custom rule writes it. `null` for the built-in kinds. */ -export function patternOf(r: SecurityRuleView): { pattern: string; match: ContentMatch } | null { - switch (r.matcher.kind) { +export function patternOf(r: SecurityRuleView): WrittenPattern | null { + const m = r.matcher; + switch (m.kind) { case 'regex': - return { pattern: r.matcher.pattern, match: 'regex' }; + return { pattern: m.pattern, match: 'regex', caseSensitive: m.case_sensitive === true }; case 'contains': - return { pattern: r.matcher.text, match: 'contains' }; + return { pattern: m.text, match: 'contains', caseSensitive: m.case_sensitive === true }; case 'codepoints': - return { pattern: r.matcher.ranges.join(', '), match: 'codepoints' }; + return { pattern: m.ranges.join(', '), match: 'codepoints', caseSensitive: false }; default: return null; } } -export function matcherOf(match: ContentMatch, pattern: string): Matcher { +export function matcherOf(match: ContentMatch, pattern: string, caseSensitive = false): Matcher { + const exact = caseSensitive ? { case_sensitive: true } : {}; switch (match) { case 'regex': - return { kind: 'regex', pattern }; + return { kind: 'regex', pattern, ...exact }; case 'contains': - return { kind: 'contains', text: pattern }; + return { kind: 'contains', text: pattern, ...exact }; case 'codepoints': return { kind: 'codepoints', ranges: codepointItems(pattern) }; } } +/** How each guard's custom rules can match, the default first. */ +export const MATCHES: Record = { + redact: ['contains', 'regex'], + inspect_tools: ['regex'], + content: ['contains', 'regex', 'codepoints'], +}; + /** A custom rule as the dialog saves it. */ export interface CustomRuleInput { name: string; pattern: string; - /** Content rules choose; the other two guards' custom rules are regexes. */ + /** Content and redaction rules choose; tool rules are regexes. */ match: ContentMatch; + /** Redaction only; the other guards' rules have a fixed case rule. */ + caseSensitive: boolean; /** Tool and content rules only. */ action: RuleAction | null; /** Redaction only. */ @@ -156,7 +176,7 @@ export function customRuleView(input: CustomRuleInput): SecurityRuleView { custom: true, name: input.name, kind: 'custom', - matcher: matcherOf(input.match, input.pattern), + matcher: matcherOf(input.match, input.pattern, input.caseSensitive), enabled: input.enabled, on_by_default: true, action: input.action, @@ -224,20 +244,27 @@ function changedActions(builtins: SecurityRuleView[], allo return out; } -function customPattern(r: SecurityRuleView): { pattern: string; match: ContentMatch } { - return patternOf(r) ?? { pattern: '', match: 'regex' }; +function customPattern(r: SecurityRuleView): WrittenPattern { + return patternOf(r) ?? { pattern: '', match: 'regex', caseSensitive: false }; } function redactPolicy(d: GuardDetail): RedactPolicy { const { enable, disable } = builtinSwitches(d); const custom: CustomRedactRule[] = d.rules .filter((r) => r.custom) - .map((r) => ({ - name: r.id, - pattern: customPattern(r).pattern, - label: r.label || DEFAULT_LABEL, - ...(r.enabled ? {} : { disabled: true }), - })); + .map((r) => { + const { pattern, match, caseSensitive } = customPattern(r); + // Code points never reach a redaction rule; written as text if one did. + const how: RedactMatch = match === 'regex' ? 'regex' : 'contains'; + return { + name: r.id, + pattern, + match: how, + ...(caseSensitive ? { case_sensitive: true } : {}), + label: r.label || DEFAULT_LABEL, + ...(r.enabled ? {} : { disabled: true }), + }; + }); return { mode: d.mode, ...(enable.length ? { enable } : {}), diff --git a/web/src/routes/gateway/security/rule-dialogs.tsx b/web/src/routes/gateway/security/rule-dialogs.tsx index f736548d..4ba4277b 100644 --- a/web/src/routes/gateway/security/rule-dialogs.tsx +++ b/web/src/routes/gateway/security/rule-dialogs.tsx @@ -4,6 +4,7 @@ import { AlertCircle, Copy, Loader2 } from 'lucide-react'; import { Alert, AlertDescription } from '@/components/ui/alert'; import { Badge } from '@/components/ui/badge'; import { Button } from '@/components/ui/button'; +import { Checkbox } from '@/components/ui/checkbox'; import { Dialog, DialogContent, @@ -47,6 +48,7 @@ import { import { ACTIONS, DEFAULT_LABEL, + MATCHES, MAX_CODEPOINT_ITEMS, NEW_RULE_ACTION, checkCodepoints, @@ -67,8 +69,6 @@ export interface RuleSeed { action: RuleAction | null; } -const MATCHES: readonly ContentMatch[] = ['contains', 'regex', 'codepoints']; - function errorText(err: unknown, fallback: string): string { return err instanceof Error ? err.message : fallback; } @@ -218,10 +218,11 @@ function TrialBox({ /** * Create or edit a custom rule. * - * Content rules match by text, regex or code points; redaction rules name - * the placeholder a hit becomes; tool and content rules choose what a hit - * does. Code points and the placeholder name are checked as they are typed; - * a regex is checked by the server, which marks the sample as it changes. + * Content rules match by text, regex or code points; redaction rules match + * by text or regex, case-sensitive or not, and name the placeholder a hit + * becomes; tool and content rules choose what a hit does. Code points and + * the placeholder name are checked as they are typed; a regex is checked by + * the server, which marks the sample as it changes. */ export function CustomRuleDialog({ guard, @@ -249,10 +250,13 @@ export function CustomRuleDialog({ const was = editing ? patternOf(editing) : null; const [name, setName] = useState(editing?.id ?? seed?.name ?? ''); - // Only content rules choose; the other guards' custom rules are regexes. - const [match, setMatch] = useState( - content ? (was?.match ?? seed?.match ?? 'contains') : 'regex', - ); + // Content and redaction rules choose; tool rules are regexes. + const matches = MATCHES[guard]; + const [match, setMatch] = useState(() => { + const written = was?.match ?? seed?.match; + return written && matches.includes(written) ? written : matches[0]; + }); + const [caseSensitive, setCaseSensitive] = useState(redact && (was?.caseSensitive ?? false)); const [pattern, setPattern] = useState(was?.pattern ?? seed?.pattern ?? ''); const [label, setLabel] = useState(editing?.label || DEFAULT_LABEL); const [action, setAction] = useState( @@ -287,7 +291,8 @@ export function CustomRuleDialog({ ? { sample, pattern, - match: content ? match : undefined, + match: matches.length > 1 ? match : undefined, + case_sensitive: redact ? caseSensitive : undefined, label: redact ? label : undefined, // Redaction rules have no action; the server refuses one there. action: acts ? action : undefined, @@ -295,19 +300,19 @@ export function CustomRuleDialog({ : null; const trial = useTrial(guard, request); - const patternLabel = !content - ? t('contentSecurity.dialog.patternRegex') - : match === 'contains' + const patternLabel = + match === 'contains' ? t('contentSecurity.dialog.patternContains') : match === 'regex' ? t('contentSecurity.dialog.patternRegex') : t('contentSecurity.dialog.patternCodepoints'); - const patternHint = !content - ? t(`contentSecurity.dialog.regexHint.${guard}`) - : match === 'contains' - ? t('contentSecurity.dialog.containsHint') + const patternHint = + match === 'contains' + ? redact + ? t('contentSecurity.dialog.redactContainsHint') + : t('contentSecurity.dialog.containsHint') : match === 'regex' - ? t('contentSecurity.dialog.regexHint.content') + ? t(`contentSecurity.dialog.regexHint.${guard}`) : t('contentSecurity.dialog.codepointsHint'); async function save() { @@ -318,6 +323,7 @@ export function CustomRuleDialog({ name: trimmed, pattern, match, + caseSensitive: redact && caseSensitive, action: acts ? action : null, label: redact ? label : null, enabled: editing?.enabled ?? true, @@ -360,13 +366,13 @@ export function CustomRuleDialog({ /> - {content && ( + {matches.length > 1 && ( -
+
({ + options={matches.map((m) => ({ value: m, label: t(`contentSecurity.dialog.match.${m}`), }))} @@ -376,6 +382,19 @@ export function CustomRuleDialog({ }} disabled={readOnly} /> + {redact && ( + + )}
)} diff --git a/web/src/routes/gateway/security/security.test.tsx b/web/src/routes/gateway/security/security.test.tsx index 155adb69..8baf6521 100644 --- a/web/src/routes/gateway/security/security.test.tsx +++ b/web/src/routes/gateway/security/security.test.tsx @@ -207,16 +207,32 @@ describe('GatewaySecurityPage', () => { await user.click(await screen.findByRole('button', { name: 'New rule' })) const dialog = await screen.findByRole('dialog') - await user.type(within(dialog).getByLabelText('Regular expression'), 'PRJ-\\d+') + // Text as written, ignoring case, unless the rule says otherwise. + expect(within(dialog).getByRole('radio', { name: 'Contains' })).toBeChecked() + expect(within(dialog).queryByRole('radio', { name: 'Code points' })).not.toBeInTheDocument() + expect(within(dialog).getByRole('checkbox', { name: 'Case-sensitive' })).not.toBeChecked() + await user.type(within(dialog).getByLabelText('Text to find'), 'PRJ-') + await user.type(within(dialog).getByLabelText('Test text'), 'see prj-12') + await waitFor(() => + expect(api).toHaveBeenCalledWith( + '/api/admin/security/redact/test', + expect.objectContaining({ + body: { sample: 'see prj-12', pattern: 'PRJ-', match: 'contains', case_sensitive: false, label: 'SECRET' }, + }), + ), + ) + + await user.click(within(dialog).getByRole('radio', { name: 'Regex' })) + await user.click(within(dialog).getByRole('checkbox', { name: 'Case-sensitive' })) + await user.type(within(dialog).getByLabelText('Regular expression'), '\\d+') await user.clear(within(dialog).getByLabelText('Placeholder name')) await user.type(within(dialog).getByLabelText('Placeholder name'), 'PROJECT') - await user.type(within(dialog).getByLabelText('Test text'), 'see PRJ-12') await waitFor(() => expect(api).toHaveBeenCalledWith( '/api/admin/security/redact/test', expect.objectContaining({ - body: { sample: 'see PRJ-12', pattern: 'PRJ-\\d+', label: 'PROJECT' }, + body: { sample: 'see prj-12', pattern: 'PRJ-\\d+', match: 'regex', case_sensitive: true, label: 'PROJECT' }, }), ), ) @@ -230,6 +246,7 @@ describe('GatewaySecurityPage', () => { await user.click(await screen.findByRole('button', { name: 'New rule' })) const dialog = await screen.findByRole('dialog') await user.type(within(dialog).getByLabelText('Name'), 'Broken') + await user.click(within(dialog).getByRole('radio', { name: 'Regex' })) await user.type(within(dialog).getByLabelText('Regular expression'), 'PRJ-(') await user.click(within(dialog).getByRole('button', { name: 'Create' })) @@ -238,7 +255,7 @@ describe('GatewaySecurityPage', () => { settings: { 'security.redact': { mode: 'observe', - custom: [{ name: 'Broken', pattern: 'PRJ-(', label: 'SECRET' }], + custom: [{ name: 'Broken', pattern: 'PRJ-(', match: 'regex', label: 'SECRET' }], }, }, })