diff --git a/README.md b/README.md index 206e361..0860d07 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,30 @@ metorial-admin files upload ./logo.png --purpose organization_image metorial-admin files upload ./build-context --purpose sandbox_image_build_context ``` +## Skills + +Both CLIs can edit Metorial skills. `metorial` works with the skills you can access in your portal; `metorial-admin skills` works with every skill in the instance. + +```bash +# Work on a skill as a local folder +metorial skills list +metorial skills pull release-notes ./release-notes +metorial skills status ./release-notes +metorial skills push ./release-notes # --dry-run to preview, --force to overwrite remote changes + +# Edit single files +metorial skills files ls release-notes +metorial skills files cat release-notes /SKILL.md +metorial skills files put release-notes /references/api.md -f ./api.md +metorial skills files rm release-notes /references/old.md +``` + +`pull`, `push` and `status` work the same way in both CLIs. For single files, `metorial-admin` has the generated `skills files list/get/put/delete/move/batch` commands instead of `ls/cat/put -f/rm`. + +`pull` records the version of every file in `.metorial/skill.json`. Before each batch of up to 100 changes, `push` checks that none of the files it touches changed remotely, and every change also carries the recorded version as a precondition, so edits someone made in the dashboard in the meantime are not overwritten. Batches are not atomic: if the server rejects a change, the changes before it stay applied and are recorded locally, so `status` shows what is left. `push` never deletes `/SKILL.md`. Markdown files become collaboratively editable documents; other files are uploaded as-is. + +These paths are never pushed or pulled: `.git`, `.metorial`, `.DS_Store`, `node_modules`, `.idea`, `.vscode`, `.env`, `.env.*`, `*.swp`, `*~` and `.metorial-*` temporary files. Add up to 30 more patterns to a `.metorialignore` file in the skill folder, one per line; a pattern without `/` matches any file or folder name, one with `/` (including a leading `/`, as in `/drafts`) is matched from the skill root, and a matching folder ignores everything under it (`path.Match` syntax, no `**` or negation; a trailing `/` is ignored). A file with more patterns makes `pull`, `push` and `status` fail until you trim it. `.metorialignore` itself is synced like any other file, so everyone who pulls the skill gets the same rules; you can create it in an empty folder before the first `pull`. + ## Help ```bash diff --git a/generator/src/emit/catalog.go.ts b/generator/src/emit/catalog.go.ts index 9d91a96..6d08b58 100644 --- a/generator/src/emit/catalog.go.ts +++ b/generator/src/emit/catalog.go.ts @@ -1,4 +1,4 @@ -import type { MappedArg, MappedFlag, MappedOperation, MappedResource } from './join'; +import type { MappedArg, MappedFlag, MappedOperation, MappedResource } from '../join'; let flagTypeName: Record = { string: 'resourcecmd.FlagString', @@ -105,6 +105,6 @@ let emitArg = (arg: MappedArg) => `{Name: ${goString(arg.name)}, Target: ${goString(arg.target)}, Required: ${arg.required}, Description: ${goString(arg.description)}}`; let emitFlag = (flag: MappedFlag) => - `{Name: ${goString(flag.name)}, Type: ${flagTypeName[flag.type]}, Target: ${goString(flag.target)}, Usage: ${goString(flag.usage)}, Required: ${flag.required}, Repeated: ${flag.repeated}}`; + `{Name: ${goString(flag.name)}, Type: ${flagTypeName[flag.type]}, Target: ${goString(flag.target)}, Usage: ${goString(flag.usage)}, Required: ${flag.required}, Repeated: ${flag.repeated}${flag.nullable ? ', Nullable: true' : ''}}`; let goString = (value: string) => JSON.stringify(value || ''); diff --git a/generator/src/emit/slate.ts b/generator/src/emit/slate.ts new file mode 100644 index 0000000..8252a05 --- /dev/null +++ b/generator/src/emit/slate.ts @@ -0,0 +1,301 @@ +import { camelToKebab, pathPlaceholders } from '../case'; +import type { IntrospectedType } from '../fetch'; +import { extractProperties, type MappedOperation, type MappedResource } from '../join'; + +type JsonSchema = Record; + +type SlateParam = { in: 'path' | 'query' | 'body'; name: string }; + +type SlateTool = { + key: string; + name: string; + description: string; + category: string; + method: string; + path: string; + instanceScoped: boolean; + readOnly: boolean; + destructive: boolean; + params: Record; + inputSchema: JsonSchema; +}; + +type SlateCategory = { key: string; name: string; description: string }; + +let maxKeyLength = 44; +let destructiveMethods = /^(delete|archive|revoke|remove|terminate|destroy|purge)\b/; +let destructiveMethodNames = new Set([ + 'disable', + 'exec', + 'revoke', + 'rollback', + 'rotate-webhook-secret', + 'set' +]); + +export let emitSlateContract = (input: { apiVersion: string; resources: MappedResource[] }) => { + let categories: SlateCategory[] = []; + let tools: SlateTool[] = []; + let errors: string[] = []; + + for (let root of input.resources) { + categories.push({ + key: root.path[0], + name: root.short, + description: root.long || `${root.short} resources.` + }); + + for (let resource of flatten(root)) { + let operationNames = new Set(resource.operations.map(operation => operation.name)); + for (let name of Object.keys(resource.slateKeys)) { + if (!operationNames.has(name)) { + errors.push(`slateKeys of ${resource.path.join(' ')} names unknown operation ${name}`); + } + } + + for (let name of resource.slateExclude) { + if (!operationNames.has(name)) { + errors.push(`slateExclude of ${resource.path.join(' ')} names unknown operation ${name}`); + } + } + + for (let operation of resource.operations) { + if (operation.source.confidential) continue; + if (resource.slateExclude.includes(operation.name)) continue; + + try { + tools.push(mapTool(root.path[0], resource, operation)); + } catch (error) { + errors.push((error as Error).message); + } + } + } + } + + let keys = new Set(); + for (let tool of tools) { + if (keys.has(tool.key)) errors.push(`duplicate tool key ${tool.key}`); + keys.add(tool.key); + } + + if (errors.length) { + throw new Error(`Cannot emit slate contract:\n ${errors.join('\n ')}`); + } + + let usedCategories = new Set(tools.map(tool => tool.category)); + + return `${JSON.stringify( + { + generatedFrom: { apiVersion: input.apiVersion }, + categories: categories.filter(category => usedCategories.has(category.key)), + tools + }, + null, + 2 + )}\n`; +}; + +let flatten = (resource: MappedResource): MappedResource[] => [ + resource, + ...resource.children.flatMap(flatten) +]; + +let snake = (value: string) => camelToKebab(value).replace(/-/g, '_'); + +let mapTool = ( + category: string, + resource: MappedResource, + operation: MappedOperation +): SlateTool => { + let { query, body, instanceScoped } = operation.source; + let key = + resource.slateKeys[operation.name] || [...resource.path, operation.name].map(snake).join('_'); + if (key.length > maxKeyLength) { + throw new Error(`tool key ${key} is ${key.length} chars (max ${maxKeyLength})`); + } + + let params: Record = {}; + let properties: Record = {}; + let required: string[] = []; + + let add = (field: string, param: SlateParam, schema: JsonSchema, isRequired: boolean) => { + if (params[field]) { + throw new Error( + `tool ${key} has colliding input ${field} (${params[field].in}.${params[field].name} and ${param.in}.${param.name})` + ); + } + params[field] = param; + properties[field] = schema; + if (isRequired) required.push(field); + }; + + let keptPathParams = new Set( + operation.args.filter(arg => arg.target.startsWith('path.')).map(arg => arg.target.slice(5)) + ); + for (let placeholder of pathPlaceholders(operation.path)) { + if (!keptPathParams.has(placeholder)) { + throw new Error(`tool ${key} path ${operation.path} has unmapped placeholder :${placeholder}`); + } + add( + snake(placeholder), + { in: 'path', name: placeholder }, + { type: 'string', description: camelToKebab(placeholder).replace(/-/g, ' ') }, + true + ); + } + + for (let [location, type] of [ + ['query', query], + ['body', body] + ] as const) { + for (let [field, schema] of Object.entries(inputProperties(type))) { + add(field, { in: location, name: field }, toJsonSchema(schema), !schema.optional); + } + } + + let description = + operation.long !== operation.short + ? operation.long + : [operation.short, resource.long].filter(Boolean).join('. '); + + return { + key, + name: operation.short, + description, + category, + method: operation.method, + path: operation.path, + instanceScoped, + readOnly: operation.method === 'GET', + destructive: + operation.method === 'DELETE' || + destructiveMethods.test(operation.name) || + destructiveMethodNames.has(operation.name), + params, + inputSchema: { + $schema: 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties, + ...(required.length ? { required } : {}) + } + }; +}; + +let isEnumLike = (type: IntrospectedType) => + (type.type === 'literal' || type.type === 'enum') && type.examples.length > 0; + +// Unlike extractProperties, merges literal values of keys shared across variants (e.g. a discriminator). +let inputProperties = (type?: IntrospectedType): Record => { + if (type?.type !== 'union' || !type.items) return extractProperties(type); + + let properties: Record = {}; + for (let item of type.items) { + for (let [key, property] of Object.entries(extractProperties(item))) { + let existing = properties[key]; + if (!existing) { + properties[key] = { ...property, optional: true }; + } else if (isEnumLike(existing) && isEnumLike(property)) { + properties[key] = { + ...existing, + type: 'enum', + examples: [...new Set([...existing.examples, ...property.examples])], + nullable: existing.nullable || property.nullable + }; + } + } + } + + return properties; +}; + +let toJsonSchema = (type: IntrospectedType): JsonSchema => { + let schema = baseSchema(type); + if (type.type === 'date') { + schema.description = type.description + ? `${type.description} (ISO 8601 date-time)` + : 'ISO 8601 date-time.'; + } else if (type.description) { + schema.description = type.description; + } + return type.nullable ? withNull(schema) : schema; +}; + +let enumSchema = (values: any[]): JsonSchema => { + let types = new Set(values.map(value => typeof value)); + return types.size === 1 ? { type: typeof values[0], enum: values } : { enum: values }; +}; + +let baseSchema = (type: IntrospectedType): JsonSchema => { + let items = type.items || []; + + switch (type.type) { + case 'string': + return { type: 'string' }; + case 'date': + return { type: 'string' }; + case 'null': + return { type: 'null' }; + case 'bigint': + return { type: 'integer' }; + case 'number': + return { type: 'number' }; + case 'boolean': + return { type: 'boolean' }; + case 'enum': + return type.examples.length ? enumSchema(type.examples) : { type: 'string' }; + case 'literal': + return { type: typeof type.examples[0], const: type.examples[0] }; + case 'array': + return { type: 'array', items: items[0] ? toJsonSchema(items[0]) : {} }; + case 'record': + return { + type: 'object', + additionalProperties: items[0] && items[0].type !== 'any' ? toJsonSchema(items[0]) : {} + }; + case 'object': + return objectSchema(type.properties || {}); + case 'intersection': + if (items.every(item => ['object', 'intersection'].includes(item.type))) { + return objectSchema(extractProperties(type)); + } + return { allOf: items.map(toJsonSchema) }; + case 'union': + if (items.every(item => item.type === 'literal' || item.type === 'enum')) { + let schema = enumSchema(items.flatMap(item => item.examples)); + return items.some(item => item.nullable) ? withNull(schema) : schema; + } + return { anyOf: items.map(toJsonSchema) }; + default: + return {}; + } +}; + +let objectSchema = (properties: Record): JsonSchema => { + let required = Object.entries(properties) + .filter(([, value]) => !value.optional) + .map(([key]) => key); + + return { + type: 'object', + properties: Object.fromEntries( + Object.entries(properties).map(([key, value]) => [key, toJsonSchema(value)]) + ), + ...(required.length ? { required } : {}) + }; +}; + +let withNull = (schema: JsonSchema): JsonSchema => { + if (schema.type === 'null' || schema.enum?.includes(null)) return schema; + if (schema.enum && schema.type === undefined) return { ...schema, enum: [...schema.enum, null] }; + if (schema.anyOf) return { ...schema, anyOf: [...schema.anyOf, { type: 'null' }] }; + if (schema.const !== undefined) { + let { const: value, type, ...rest } = schema; + return { ...rest, type: [type, 'null'], enum: [value, null] }; + } + if (typeof schema.type !== 'string') return schema; + return { + ...schema, + type: [schema.type, 'null'], + ...(schema.enum ? { enum: [...schema.enum, null] } : {}) + }; +}; diff --git a/generator/src/fetch.ts b/generator/src/fetch.ts index 2d7ec33..df81525 100644 --- a/generator/src/fetch.ts +++ b/generator/src/fetch.ts @@ -18,6 +18,7 @@ export type Endpoint = { bodyId?: string | null; outputId: string; controllerId: string; + confidential?: boolean; }; export type Controller = { diff --git a/generator/src/index.ts b/generator/src/index.ts index ee12efb..5ffc67e 100644 --- a/generator/src/index.ts +++ b/generator/src/index.ts @@ -1,8 +1,10 @@ import path from 'node:path'; +import { parseArgs } from 'node:util'; import { $ } from 'bun'; import { emitCatalogGo } from './emit/catalog.go'; +import { emitSlateContract } from './emit/slate'; import { getEndpoints, getEndpointVersions, type IntrospectionSnapshot } from './fetch'; -import { joinManifest } from './join'; +import { joinManifest, type MappedResource } from './join'; import { loadManifest } from './manifest'; let repoRoot = path.resolve(import.meta.dir, '../..'); @@ -12,12 +14,43 @@ let defaultUrls = [ 'http://localhost:4310' ].filter(Boolean) as string[]; -let args = process.argv.slice(2); -let urlArg = args.find(arg => !arg.startsWith('-')); +let { positionals, values } = parseArgs({ + args: process.argv.slice(2), + allowPositionals: true, + options: { + target: { type: 'string', default: 'go' }, + out: { type: 'string' }, + snapshot: { type: 'string' } + } +}); + +let emitters: Record Promise> = { + go: async catalog => { + let outputPath = path.join(repoRoot, 'internal/generated/admin/catalog.go'); + await Bun.write(outputPath, emitCatalogGo(catalog)); + await $`gofmt -w ${outputPath}`; + return outputPath; + }, + slate: async (catalog, apiVersion) => { + let outputPath = path.resolve(values.out!); + await Bun.write(outputPath, emitSlateContract({ apiVersion, resources: catalog })); + return outputPath; + } +}; + +let emit = emitters[values.target!]; +if (!emit) { + throw new Error(`Unknown --target ${values.target}, expected ${Object.keys(emitters).join(' or ')}`); +} +if (values.target === 'slate' && !values.out) { + throw new Error('--out is required for --target slate'); +} + +let urlArg = positionals[0]; let urls = urlArg ? urlArg.split(',') : defaultUrls; // An explicit URL (argument or API_URL) wins over --snapshot so a running instance can be used. -let snapshotPath = args.find(arg => arg.startsWith('--snapshot='))?.slice('--snapshot='.length); +let snapshotPath = values.snapshot; let snapshot = snapshotPath && !urlArg && !process.env.API_URL ? ((await Bun.file(path.resolve(snapshotPath)).json()) as IntrospectionSnapshot) @@ -47,9 +80,7 @@ let catalog = joinManifest({ types: payload.types }); -let outputPath = path.join(repoRoot, 'internal/generated/admin/catalog.go'); -await Bun.write(outputPath, emitCatalogGo(catalog)); -await $`gofmt -w ${outputPath}`; +let outputPath = await emit(catalog, version.version); console.log(`Wrote ${outputPath} from ${workingUrl} (${version.version})`); diff --git a/generator/src/join.ts b/generator/src/join.ts index 8599267..2dabe1c 100644 --- a/generator/src/join.ts +++ b/generator/src/join.ts @@ -24,6 +24,7 @@ export type MappedFlag = { usage: string; required: boolean; repeated: boolean; + nullable: boolean; }; export type MappedOperation = { @@ -35,6 +36,12 @@ export type MappedOperation = { args: MappedArg[]; flags: MappedFlag[]; sdkMapping: string; + source: { + query?: IntrospectedType; + body?: IntrospectedType; + confidential: boolean; + instanceScoped: boolean; + }; }; export type MappedResource = { @@ -46,6 +53,8 @@ export type MappedResource = { aliases: string[]; operations: MappedOperation[]; shortcuts: { path: string[]; method: string }[]; + slateKeys: Record; + slateExclude: string[]; children: MappedResource[]; }; @@ -81,6 +90,8 @@ export let joinManifest = (input: { aliases: entry.aliases || [], operations: [], shortcuts: [], + slateKeys: {}, + slateExclude: [], children: [] } }); @@ -146,6 +157,8 @@ export let joinManifest = (input: { path: shortcut.path, method: camelToKebab(shortcut.method) })), + slateKeys: entry.slateKeys || {}, + slateExclude: entry.slateExclude || [], children: [] } }); @@ -266,7 +279,8 @@ let mapOperation = (input: { target: `query.${key}`, usage: schema.description || key, required: override?.required ?? false, - repeated: flagType === 'string-slice' + repeated: flagType === 'string-slice', + nullable: false }); } @@ -293,7 +307,8 @@ let mapOperation = (input: { target: `body.${key}`, usage: schema.description || key, required: override?.required ?? !schema.optional, - repeated: flagType === 'string-slice' + repeated: flagType === 'string-slice', + nullable: flagType === 'string' && schema.nullable }); if (flagType === 'json') { @@ -303,7 +318,8 @@ let mapOperation = (input: { target: `body.${key}`, usage: `Read ${key} JSON from a file`, required: false, - repeated: false + repeated: false, + nullable: false }); } } @@ -316,7 +332,15 @@ let mapOperation = (input: { long: input.endpoint.description || input.endpoint.name, args, flags, - sdkMapping: input.endpoint.sdkPath + sdkMapping: input.endpoint.sdkPath, + source: { + query: queryType, + body: bodyType, + confidential: input.endpoint.confidential === true, + instanceScoped: input.endpoint.allPaths.some(item => + item.sdkPath.startsWith('management.instance.') + ) + } }; }; diff --git a/generator/src/manifest.ts b/generator/src/manifest.ts index 8953356..c038b1a 100644 --- a/generator/src/manifest.ts +++ b/generator/src/manifest.ts @@ -23,6 +23,8 @@ export type CommandEntry = { long?: string; args?: Record>; shortcuts?: ShortcutEntry[]; + slateKeys?: Record; + slateExclude?: string[]; }; export type Manifest = { diff --git a/internal/apiclient/client.go b/internal/apiclient/client.go new file mode 100644 index 0000000..6a979a8 --- /dev/null +++ b/internal/apiclient/client.go @@ -0,0 +1,119 @@ +package apiclient + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/metorial/cli/internal/config" + "github.com/metorial/cli/internal/fetch" + metorial "github.com/metorial/metorial-go/v1" +) + +type Pagination struct { + HasMoreBefore bool `json:"has_more_before"` + HasMoreAfter bool `json:"has_more_after"` + After string `json:"after,omitempty"` +} + +type ListOptions struct { + Limit float64 + After string + Before string + Cursor string +} + +type Client struct { + Runtime config.Runtime + Headers map[string]string + Describe func(target string, err error) error +} + +func (c Client) Do(method string, target string, body any, output any) error { + var payload []byte + if body != nil { + encoded, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("metorial: failed to encode request body: %w", err) + } + payload = encoded + } + + headers := map[string]string{} + if payload != nil { + headers["Content-Type"] = "application/json" + } + + return c.send(fetch.Request{Method: method, Target: target, Headers: headers, Body: payload}, output) +} + +func (c Client) send(request fetch.Request, output any) error { + headers := map[string]string{} + for key, value := range c.Headers { + headers[key] = value + } + for key, value := range request.Headers { + headers[key] = value + } + request.Headers = headers + + response, err := fetch.Do(c.Runtime, request) + if err != nil { + return c.describe(request.Target, err) + } + + if output == nil || len(response.Body) == 0 { + return nil + } + if err := json.Unmarshal(response.Body, output); err != nil { + return fmt.Errorf("metorial: failed to decode response from %s: %w", request.Target, err) + } + + return nil +} + +func (c Client) describe(target string, err error) error { + if c.Describe == nil { + return err + } + return c.Describe(target, err) +} + +func IsStatus(err error, statusCode int) bool { + apiErr := &metorial.Error{} + return errors.As(err, &apiErr) && apiErr.StatusCode == statusCode +} + +func ListQuery(options ListOptions, extra url.Values) string { + values := url.Values{} + for key, items := range extra { + for _, item := range items { + values.Add(key, item) + } + } + + if options.Limit > 0 { + values.Set("limit", strconv.FormatFloat(options.Limit, 'f', -1, 64)) + } + if after := strings.TrimSpace(options.After); after != "" { + values.Set("after", after) + } + if before := strings.TrimSpace(options.Before); before != "" { + values.Set("before", before) + } + if cursor := strings.TrimSpace(options.Cursor); cursor != "" { + values.Set("cursor", cursor) + } + + if len(values) == 0 { + return "" + } + return "?" + values.Encode() +} + +func PathSegment(value string) string { + return url.PathEscape(strings.TrimSpace(value)) +} diff --git a/internal/cli/admin.go b/internal/cli/admin.go index 36d9703..4c6c640 100644 --- a/internal/cli/admin.go +++ b/internal/cli/admin.go @@ -12,6 +12,7 @@ import ( onboardingcmd "github.com/metorial/cli/internal/commands/onboarding" resourcescmd "github.com/metorial/cli/internal/commands/resources" sandboxcmd "github.com/metorial/cli/internal/commands/sandbox" + skillscmd "github.com/metorial/cli/internal/commands/skills" systemcmd "github.com/metorial/cli/internal/commands/system" "github.com/metorial/cli/internal/commandutil" "github.com/metorial/cli/internal/config" @@ -59,6 +60,9 @@ func newAdminRoot(application *app.App) (*cobra.Command, error) { if err := filescmd.Attach(command, ctx); err != nil { return nil, err } + if err := skillscmd.Attach(command, ctx); err != nil { + return nil, err + } } return command, nil diff --git a/internal/commands/consumer/catalog.go b/internal/commands/consumer/catalog.go index 5568fbc..50a5733 100644 --- a/internal/commands/consumer/catalog.go +++ b/internal/commands/consumer/catalog.go @@ -5,6 +5,7 @@ import ( "io" "strings" + "github.com/metorial/cli/internal/commandutil" "github.com/metorial/cli/internal/consumerapi" "github.com/metorial/cli/internal/output" "github.com/metorial/cli/internal/terminal" @@ -29,7 +30,7 @@ func newCatalogCommands(env commandEnv) []*cobra.Command { return err } - search := strings.TrimSpace(optionalArg(args, 0)) + search := strings.TrimSpace(commandutil.OptionalArg(args, 0)) result, err := client.ListProviders(pagination.list(), search, groupID) if err != nil { return err @@ -42,7 +43,7 @@ func newCatalogCommands(env commandEnv) []*cobra.Command { Name: item.Name, Type: catalogTypeLabel(item), Availability: availabilityLabel(item), - Description: optionalString(item.Description), + Description: commandutil.OptionalString(item.Description), }) } @@ -57,14 +58,14 @@ func newCatalogCommands(env commandEnv) []*cobra.Command { base := command.CommandPath() if search != "" { - base += " " + shellQuote(search) + base += " " + commandutil.ShellQuote(search) } if groupID != "" { - base += " --group " + shellQuote(groupID) + base += " --group " + commandutil.ShellQuote(groupID) } tips = append(tips, paginationTips(base, pagination, firstID, lastID, result.Pagination)...) - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "items": result.Items, "rows": rows, "pagination": result.Pagination, @@ -125,17 +126,17 @@ func newCatalogGroupsCommand(env commandEnv) *cobra.Command { tips = append(tips, fmt.Sprintf("metorial catalog list --group %s", result.Items[0].ID)) } - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "items": result.Items, "pagination": result.Pagination, "tips": tips, }, func(writer io.Writer, features terminal.Features) error { colors := terminal.NewColorizer(features) - renderTitle(writer, features, "Catalog Groups", "Groups your portal uses to organize integrations.") + commandutil.RenderTitle(writer, features, "Catalog Groups", "Groups your portal uses to organize integrations.") if len(result.Items) == 0 { _, _ = fmt.Fprintln(writer, "(no groups)") - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } table := output.Table{ @@ -146,13 +147,13 @@ func newCatalogGroupsCommand(env commandEnv) *cobra.Command { for _, group := range result.Items { table.Rows = append(table.Rows, []string{ colors.Bold(group.Name) + "\n" + colors.Muted(group.ID) + "\n", - optionalString(group.Description), + commandutil.OptionalString(group.Description), }) } if err := table.Render(writer); err != nil { return err } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) }) }, } @@ -199,7 +200,7 @@ func newCatalogGetCommand(env commandEnv) *cobra.Command { tips = append(tips, fmt.Sprintf("metorial reconnect %s %s", item.ID, owned[0].ID)) } - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "item": item, "tools": tools, "integrations": owned, @@ -240,7 +241,7 @@ func newRequestAccessCommand(env commandEnv) *cobra.Command { return err } - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "item": item, "access_request": request, }, func(writer io.Writer, features terminal.Features) error { @@ -266,11 +267,11 @@ func newRequestAccessCommand(env commandEnv) *cobra.Command { func renderCatalogList(writer io.Writer, features terminal.Features, rows []catalogListRow, tips []string) error { colors := terminal.NewColorizer(features) - renderTitle(writer, features, "Integration Catalog", "Integrations your portal makes available to you.") + commandutil.RenderTitle(writer, features, "Integration Catalog", "Integrations your portal makes available to you.") if len(rows) == 0 { _, _ = fmt.Fprintln(writer, "(no integrations)") - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } table := output.Table{ @@ -293,12 +294,12 @@ func renderCatalogList(writer io.Writer, features terminal.Features, rows []cata return err } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } func renderCatalogDetail(writer io.Writer, features terminal.Features, item *consumerapi.Provider, tools []consumerapi.SpecificationTool, owned []consumerapi.MagicMcpServer, tips []string) error { colors := terminal.NewColorizer(features) - renderTitle(writer, features, item.Name, item.ID) + commandutil.RenderTitle(writer, features, item.Name, item.ID) items := []output.DataListItem{ {Label: "Type", Value: catalogTypeLabel(*item)}, @@ -311,7 +312,7 @@ func renderCatalogDetail(writer io.Writer, features terminal.Features, item *con } items = append(items, output.DataListItem{Label: "Provider", Value: provider}) } - if description := optionalString(item.Description); description != "" { + if description := commandutil.OptionalString(item.Description); description != "" { items = append(items, output.DataListItem{Label: "Description", Value: description}) } if err := output.RenderDataList(writer, items); err != nil { @@ -355,7 +356,7 @@ func renderCatalogDetail(writer io.Writer, features terminal.Features, item *con } } - if readme := optionalString(item.Readme); readme != "" { + if readme := commandutil.OptionalString(item.Readme); readme != "" { _, _ = fmt.Fprintln(writer) _, _ = fmt.Fprintln(writer, colors.Accent("Readme")) _, _ = fmt.Fprintln(writer, readme) @@ -370,21 +371,21 @@ func renderCatalogDetail(writer io.Writer, features terminal.Features, item *con MaxWidth: features.Width, } for _, tool := range tools { - table.Rows = append(table.Rows, []string{colors.Bold(tool.Key), tool.Name, optionalString(tool.Description)}) + table.Rows = append(table.Rows, []string{colors.Bold(tool.Key), tool.Name, commandutil.OptionalString(tool.Description)}) } if err := table.Render(writer); err != nil { return err } } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } func catalogSpecificationID(item *consumerapi.Provider) string { if item.Provider == nil || item.Provider.CurrentVersion == nil { return "" } - return optionalString(item.Provider.CurrentVersion.SpecificationID) + return commandutil.OptionalString(item.Provider.CurrentVersion.SpecificationID) } func catalogTypeLabel(item consumerapi.Provider) string { diff --git a/internal/commands/consumer/commands.go b/internal/commands/consumer/commands.go index 1aefcf7..9fff6eb 100644 --- a/internal/commands/consumer/commands.go +++ b/internal/commands/consumer/commands.go @@ -1,6 +1,7 @@ package consumer import ( + skillscmd "github.com/metorial/cli/internal/commands/skills" "github.com/metorial/cli/internal/commandutil" "github.com/spf13/cobra" ) @@ -20,5 +21,5 @@ func NewCommands(ctx commandutil.Context) []*cobra.Command { commandutil.SetCommandCategory(command, commandutil.CommandCategoryIntegrations) } - return commands + return append(commands, skillscmd.NewCommand(ctx)) } diff --git a/internal/commands/consumer/common.go b/internal/commands/consumer/common.go index 5d4449e..ce686fc 100644 --- a/internal/commands/consumer/common.go +++ b/internal/commands/consumer/common.go @@ -36,27 +36,6 @@ func (e commandEnv) client() (*consumerapi.Client, error) { return consumerapi.New(runtime) } -func (e commandEnv) structured() (bool, error) { - format, err := output.ParseFormat(e.format()) - if err != nil { - return false, err - } - - return format == output.FormatStructured, nil -} - -func (e commandEnv) write(command *cobra.Command, value any, render func(io.Writer, terminal.Features) error) error { - structured, err := e.structured() - if err != nil { - return err - } - if !structured { - return commandutil.WriteValue(command.OutOrStdout(), e.app().StdoutFeatures(), e.format(), value) - } - - return render(command.OutOrStdout(), e.app().StdoutFeatures()) -} - type paginationOptions struct { Limit float64 After string @@ -91,40 +70,14 @@ func paginationTips(base string, options paginationOptions, firstID, lastID stri tips := []string{} if pagination.HasMoreAfter { - tips = append(tips, fmt.Sprintf("%s --after %s", base, shellQuote(lastID))) + tips = append(tips, fmt.Sprintf("%s --after %s", base, commandutil.ShellQuote(lastID))) } if pagination.HasMoreBefore { - tips = append(tips, fmt.Sprintf("%s --before %s", base, shellQuote(firstID))) + tips = append(tips, fmt.Sprintf("%s --before %s", base, commandutil.ShellQuote(firstID))) } return tips } -func renderTips(writer io.Writer, features terminal.Features, tips []string) error { - if len(tips) == 0 { - return nil - } - - colors := terminal.NewColorizer(features) - _, _ = fmt.Fprintln(writer) - _, _ = fmt.Fprintln(writer, colors.Notice("Tips")) - for _, tip := range tips { - if strings.TrimSpace(tip) == "" { - continue - } - _, _ = fmt.Fprintf(writer, " %s\n", tip) - } - return nil -} - -func renderTitle(writer io.Writer, features terminal.Features, title, subtitle string) { - colors := terminal.NewColorizer(features) - _, _ = fmt.Fprintln(writer, colors.Bold(title)) - if strings.TrimSpace(subtitle) != "" { - _, _ = fmt.Fprintln(writer, colors.Muted(subtitle)) - } - _, _ = fmt.Fprintln(writer) -} - func renderServerEndpoints(writer io.Writer, features terminal.Features, server *consumerapi.MagicMcpServer) error { if len(server.Endpoints) == 0 { return nil @@ -163,7 +116,7 @@ func integrationIdentifier(server *consumerapi.MagicMcpServer) string { } func integrationTitle(server *consumerapi.MagicMcpServer) string { - return firstNonEmpty(optionalString(server.Name), integrationIdentifier(server)) + return commandutil.FirstNonEmpty(commandutil.OptionalString(server.Name), integrationIdentifier(server)) } func integrationTips(server *consumerapi.MagicMcpServer, includeInstall bool) []string { @@ -180,20 +133,6 @@ func integrationTips(server *consumerapi.MagicMcpServer, includeInstall bool) [] return tips } -func optionalArg(args []string, index int) string { - if index < 0 || index >= len(args) { - return "" - } - return args[index] -} - -func optionalString(value *string) string { - if value == nil { - return "" - } - return strings.TrimSpace(*value) -} - func stringPtrIfSet(value string) *string { value = strings.TrimSpace(value) if value == "" { @@ -201,17 +140,3 @@ func stringPtrIfSet(value string) *string { } return &value } - -func firstNonEmpty(values ...string) string { - return commandutil.FirstNonEmpty(values...) -} - -func shellQuote(value string) string { - if value == "" { - return "''" - } - if !strings.ContainsAny(value, " \t\n'\"\\") { - return value - } - return "'" + strings.ReplaceAll(value, "'", `'"'"'`) + "'" -} diff --git a/internal/commands/consumer/install.go b/internal/commands/consumer/install.go index 781354c..3f0e5ed 100644 --- a/internal/commands/consumer/install.go +++ b/internal/commands/consumer/install.go @@ -13,6 +13,7 @@ import ( "sort" "strings" + "github.com/metorial/cli/internal/commandutil" "github.com/metorial/cli/internal/consumerapi" "github.com/metorial/cli/internal/output" "github.com/metorial/cli/internal/terminal" @@ -182,10 +183,10 @@ func (a fileClientAdapter) detect() clientDetection { return clientDetection{Installed: false, Usable: false, Location: path} } func (a fileClientAdapter) buildInstallPlan(server remoteServerDefinition) (clientInstallPlan, error) { - containerKey := firstNonEmpty(a.topLevelKey, "mcpServers") - typeField := firstNonEmpty(a.fieldNames.Type, "type") - urlField := firstNonEmpty(a.fieldNames.URL, "url") - headersField := firstNonEmpty(a.fieldNames.Headers, "headers") + containerKey := commandutil.FirstNonEmpty(a.topLevelKey, "mcpServers") + typeField := commandutil.FirstNonEmpty(a.fieldNames.Type, "type") + urlField := commandutil.FirstNonEmpty(a.fieldNames.URL, "url") + headersField := commandutil.FirstNonEmpty(a.fieldNames.Headers, "headers") includeTypeField := true if !a.includeTypeField { includeTypeField = false @@ -249,7 +250,7 @@ func newInstallCommands(env commandEnv) []*cobra.Command { "metorial install custom ", } - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "items": rows, "tips": tips, }, func(writer io.Writer, features terminal.Features) error { @@ -304,7 +305,7 @@ func newInstallCommands(env commandEnv) []*cobra.Command { result.File = &typed } - return env.write(command, result, func(writer io.Writer, features terminal.Features) error { + return env.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { return renderIntegrationInstallResult(writer, features, result) }) }, @@ -329,7 +330,7 @@ func newInstallCommands(env commandEnv) []*cobra.Command { Token: token.Secret, } - return env.write(command, result, func(writer io.Writer, features terminal.Features) error { + return env.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { return renderCustomInstallResult(writer, features, result) }) }, @@ -409,7 +410,7 @@ func init() { addAdapter(newFileAdapter("amazon-bedrock", "Amazon Bedrock", "json", filepath.Join(paths.homeDir, "Amazon Bedrock Client", "mcp_config.json"))) addAdapter(newFileAdapter("amazonq", "Amazon Q", "json", filepath.Join(paths.homeDir, ".aws", "amazonq", "mcp.json"))) addAdapter(newFileAdapter("tome", "Tome", "json", filepath.Join(paths.homeDir, ".tome", "mcp_config.json"))) - addAdapter(newFileAdapter("librechat", "LibreChat", "yaml", filepath.Join(firstNonEmpty(os.Getenv("LIBRECHAT_CONFIG_DIR"), paths.homeDir), "LibreChat", "librechat.yaml"))) + addAdapter(newFileAdapter("librechat", "LibreChat", "yaml", filepath.Join(commandutil.FirstNonEmpty(os.Getenv("LIBRECHAT_CONFIG_DIR"), paths.homeDir), "LibreChat", "librechat.yaml"))) windsurf := newFileAdapter("windsurf", "Windsurf", "json", filepath.Join(paths.homeDir, ".codeium", "windsurf", "mcp_config.json")) windsurf.fieldNames.URL = "serverUrl" @@ -453,7 +454,7 @@ func currentPlatformPaths() platformClientPaths { case "windows": return platformClientPaths{ homeDir: homeDir, - baseDir: firstNonEmpty(os.Getenv("APPDATA"), filepath.Join(homeDir, "AppData", "Roaming")), + baseDir: commandutil.FirstNonEmpty(os.Getenv("APPDATA"), filepath.Join(homeDir, "AppData", "Roaming")), vscodePath: filepath.Join("Code", "User", "globalStorage"), } case "darwin": @@ -465,7 +466,7 @@ func currentPlatformPaths() platformClientPaths { default: return platformClientPaths{ homeDir: homeDir, - baseDir: firstNonEmpty(os.Getenv("XDG_CONFIG_HOME"), filepath.Join(homeDir, ".config")), + baseDir: commandutil.FirstNonEmpty(os.Getenv("XDG_CONFIG_HOME"), filepath.Join(homeDir, ".config")), vscodePath: filepath.Join("Code", "User", "globalStorage"), } } @@ -784,7 +785,7 @@ func renderIntegrationClientList(writer io.Writer, features terminal.Features, r if err := table.Render(writer); err != nil { return err } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } func renderIntegrationInstallResult(writer io.Writer, features terminal.Features, result integrationInstallResult) error { diff --git a/internal/commands/consumer/integrations.go b/internal/commands/consumer/integrations.go index cfdc281..8606a69 100644 --- a/internal/commands/consumer/integrations.go +++ b/internal/commands/consumer/integrations.go @@ -5,6 +5,7 @@ import ( "io" "strings" + "github.com/metorial/cli/internal/commandutil" "github.com/metorial/cli/internal/consumerapi" "github.com/metorial/cli/internal/output" "github.com/metorial/cli/internal/terminal" @@ -42,7 +43,7 @@ func newListCommand(env commandEnv) *cobra.Command { return err } - search := strings.TrimSpace(optionalArg(args, 0)) + search := strings.TrimSpace(commandutil.OptionalArg(args, 0)) result, err := client.ListMagicMcpServers(pagination.list(), search, "") if err != nil { return err @@ -55,8 +56,8 @@ func newListCommand(env commandEnv) *cobra.Command { ID: server.ID, Alias: integrationIdentifier(server), Status: server.Status, - Name: optionalString(server.Name), - Description: optionalString(server.Description), + Name: commandutil.OptionalString(server.Name), + Description: commandutil.OptionalString(server.Description), EndpointURL: primaryServerURL(server), }) } @@ -71,12 +72,12 @@ func newListCommand(env commandEnv) *cobra.Command { } base := "metorial list" if search != "" { - base += " " + shellQuote(search) + base += " " + commandutil.ShellQuote(search) } tips = append(tips, paginationTips(base, pagination, firstID, lastID, result.Pagination)...) tips = append(tips, "metorial catalog list") - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "items": result.Items, "rows": rows, "pagination": result.Pagination, @@ -108,7 +109,7 @@ func newGetCommand(env commandEnv) *cobra.Command { tips := integrationTips(server, true) - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "integration": server, "tips": tips, }, func(writer io.Writer, features terminal.Features) error { @@ -155,7 +156,7 @@ func newUpdateCommand(env commandEnv) *cobra.Command { tips := []string{fmt.Sprintf("metorial get %s", integrationIdentifier(updated))} - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "integration": updated, "tips": tips, }, func(writer io.Writer, features terminal.Features) error { @@ -204,7 +205,7 @@ func newRemoveCommand(env commandEnv) *cobra.Command { return err } - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "integration": deleted, }, func(writer io.Writer, features terminal.Features) error { colors := terminal.NewColorizer(features) @@ -221,11 +222,11 @@ func newRemoveCommand(env commandEnv) *cobra.Command { func renderIntegrationList(writer io.Writer, features terminal.Features, rows []integrationListRow, tips []string) error { colors := terminal.NewColorizer(features) - renderTitle(writer, features, "Your Integrations", "") + commandutil.RenderTitle(writer, features, "Your Integrations", "") if len(rows) == 0 { _, _ = fmt.Fprintln(writer, "(no integrations)") - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } table := output.Table{ @@ -249,17 +250,17 @@ func renderIntegrationList(writer io.Writer, features terminal.Features, rows [] return err } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } func renderIntegrationDetail(writer io.Writer, features terminal.Features, server *consumerapi.MagicMcpServer, tips []string) error { colors := terminal.NewColorizer(features) - renderTitle(writer, features, integrationTitle(server), server.ID) + commandutil.RenderTitle(writer, features, integrationTitle(server), server.ID) items := []output.DataListItem{ {Label: "Status", Value: server.Status}, } - if description := optionalString(server.Description); description != "" { + if description := commandutil.OptionalString(server.Description); description != "" { items = append(items, output.DataListItem{Label: "Description", Value: description}) } if err := output.RenderDataList(writer, items); err != nil { @@ -281,7 +282,7 @@ func renderIntegrationDetail(writer io.Writer, features terminal.Features, serve for _, provider := range server.Providers { name := provider.Name if provider.Provider != nil { - name = firstNonEmpty(provider.Provider.Name, provider.Name) + name = commandutil.FirstNonEmpty(provider.Provider.Name, provider.Name) } table.Rows = append(table.Rows, []string{name, provider.Status}) } @@ -290,5 +291,5 @@ func renderIntegrationDetail(writer io.Writer, features terminal.Features, serve } } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } diff --git a/internal/commands/consumer/setup.go b/internal/commands/consumer/setup.go index 334034f..087371f 100644 --- a/internal/commands/consumer/setup.go +++ b/internal/commands/consumer/setup.go @@ -7,6 +7,7 @@ import ( "time" "github.com/metorial/cli/internal/browser" + "github.com/metorial/cli/internal/commandutil" "github.com/metorial/cli/internal/consumerapi" "github.com/metorial/cli/internal/output" "github.com/metorial/cli/internal/terminal" @@ -43,7 +44,7 @@ func newSetupCommand(env commandEnv) *cobra.Command { return err } - item, err := resolveSetupItem(env, client, optionalArg(args, 0)) + item, err := resolveSetupItem(env, client, commandutil.OptionalArg(args, 0)) if err != nil { return err } @@ -96,7 +97,7 @@ func newReconnectCommand(env commandEnv) *cobra.Command { return err } - server, err := resolveReconnectServer(client, item, optionalArg(args, 1)) + server, err := resolveReconnectServer(client, item, commandutil.OptionalArg(args, 1)) if err != nil { return err } @@ -201,7 +202,7 @@ func runSetupSession(env commandEnv, command *cobra.Command, client *consumerapi return session, nil } - structured, err := env.structured() + structured, err := env.ctx.Structured() if err != nil { return nil, err } @@ -298,7 +299,7 @@ func waitForSetupSession(writer io.Writer, features terminal.Features, client *c func writeSetupResult(env commandEnv, command *cobra.Command, title string, item *consumerapi.Provider, session *consumerapi.SetupSession, server *consumerapi.MagicMcpServer) error { tips := append([]string{fmt.Sprintf("metorial get %s", integrationIdentifier(server))}, integrationTips(server, true)...) - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "item": item, "setup_session": session, "integration": server, @@ -307,7 +308,7 @@ func writeSetupResult(env commandEnv, command *cobra.Command, title string, item colors := terminal.NewColorizer(features) _, _ = fmt.Fprintln(writer, colors.Success(title)) _, _ = fmt.Fprintln(writer) - renderTitle(writer, features, integrationTitle(server), server.ID) + commandutil.RenderTitle(writer, features, integrationTitle(server), server.ID) if err := output.RenderDataList(writer, []output.DataListItem{ {Label: "Status", Value: server.Status}, @@ -318,6 +319,6 @@ func writeSetupResult(env commandEnv, command *cobra.Command, title string, item if err := renderServerEndpoints(writer, features, server); err != nil { return err } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) }) } diff --git a/internal/commands/consumer/tokens.go b/internal/commands/consumer/tokens.go index 0d7662b..5e9fb19 100644 --- a/internal/commands/consumer/tokens.go +++ b/internal/commands/consumer/tokens.go @@ -4,6 +4,7 @@ import ( "fmt" "io" + "github.com/metorial/cli/internal/commandutil" "github.com/metorial/cli/internal/consumerapi" "github.com/metorial/cli/internal/output" "github.com/metorial/cli/internal/terminal" @@ -48,7 +49,7 @@ func newTokensListCommand(env commandEnv) *cobra.Command { for _, token := range result.Items { rows = append(rows, tokenListRow{ ID: token.ID, - Name: optionalString(token.Name), + Name: commandutil.OptionalString(token.Name), Integration: tokenIntegrationLabel(token), CreatedAt: token.CreatedAt.Format("2006-01-02 15:04"), }) @@ -56,7 +57,7 @@ func newTokensListCommand(env commandEnv) *cobra.Command { tips := []string{} if len(rows) > 0 { - tips = append(tips, fmt.Sprintf("metorial tokens revoke %s", shellQuote(rows[0].ID))) + tips = append(tips, fmt.Sprintf("metorial tokens revoke %s", commandutil.ShellQuote(rows[0].ID))) } firstID, lastID := "", "" if len(rows) > 0 { @@ -64,7 +65,7 @@ func newTokensListCommand(env commandEnv) *cobra.Command { } tips = append(tips, paginationTips("metorial tokens list", options, firstID, lastID, result.Pagination)...) - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "items": result.Items, "pagination": result.Pagination, "tips": tips, @@ -94,7 +95,7 @@ func newTokensRevokeCommand(env commandEnv) *cobra.Command { return err } - return env.write(command, token, func(writer io.Writer, features terminal.Features) error { + return env.ctx.Write(command, token, func(writer io.Writer, features terminal.Features) error { colors := terminal.NewColorizer(features) _, _ = fmt.Fprintf(writer, "%s %s\n", colors.Success("Revoked token"), token.ID) return nil @@ -107,7 +108,7 @@ func tokenIntegrationLabel(token consumerapi.MagicMcpToken) string { if token.Server == nil { return "" } - return firstNonEmpty(optionalString(token.Server.Name), token.Server.ID) + return commandutil.FirstNonEmpty(commandutil.OptionalString(token.Server.Name), token.Server.ID) } func renderTokenList(writer io.Writer, features terminal.Features, rows []tokenListRow, tips []string) error { @@ -134,5 +135,5 @@ func renderTokenList(writer io.Writer, features terminal.Features, rows []tokenL return err } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } diff --git a/internal/commands/consumer/tools.go b/internal/commands/consumer/tools.go index 65b7e14..c31aaad 100644 --- a/internal/commands/consumer/tools.go +++ b/internal/commands/consumer/tools.go @@ -10,6 +10,7 @@ import ( "github.com/google/jsonschema-go/jsonschema" "github.com/metorial/cli/internal/app" + "github.com/metorial/cli/internal/commandutil" "github.com/metorial/cli/internal/consumerapi" "github.com/metorial/cli/internal/fetch" "github.com/metorial/cli/internal/output" @@ -55,7 +56,7 @@ func newToolsCommand(env commandEnv) *cobra.Command { tips = append(tips, integrationTips(server, true)[1:]...) } - return env.write(command, map[string]any{ + return env.ctx.Write(command, map[string]any{ "integration": server, "providers": server.Providers, "tools": liveTools, @@ -170,7 +171,7 @@ func connectIntegration(ctx context.Context, env commandEnv, integrationID strin func renderIntegrationTools(writer io.Writer, features terminal.Features, server *consumerapi.MagicMcpServer, tools []*mcp.Tool, tips []string) error { colors := terminal.NewColorizer(features) serverTitle := integrationTitle(server) - renderTitle(writer, features, serverTitle, server.ID) + commandutil.RenderTitle(writer, features, serverTitle, server.ID) for _, provider := range server.Providers { if provider.Provider == nil { @@ -180,7 +181,7 @@ func renderIntegrationTools(writer io.Writer, features terminal.Features, server _, _ = fmt.Fprintln(writer, colors.Accent(provider.Provider.Name)) } _, _ = fmt.Fprintln(writer, colors.Muted(provider.Provider.Slug)) - if description := optionalString(provider.Provider.Description); description != "" { + if description := commandutil.OptionalString(provider.Provider.Description); description != "" { _, _ = fmt.Fprintln(writer, description) } } @@ -189,7 +190,7 @@ func renderIntegrationTools(writer io.Writer, features terminal.Features, server _, _ = fmt.Fprintln(writer, colors.Accent("Tools")) if len(tools) == 0 { _, _ = fmt.Fprintln(writer, "(no tools)") - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } table := output.Table{ @@ -201,13 +202,13 @@ func renderIntegrationTools(writer io.Writer, features terminal.Features, server if tool == nil { continue } - table.Rows = append(table.Rows, []string{tool.Name, firstNonEmpty(tool.Title, tool.Name), tool.Description}) + table.Rows = append(table.Rows, []string{tool.Name, commandutil.FirstNonEmpty(tool.Title, tool.Name), tool.Description}) } if err := table.Render(writer); err != nil { return err } - return renderTips(writer, features, tips) + return commandutil.RenderTips(writer, features, tips) } func resolveIntegrationToolInput(application *app.App, data string) (map[string]any, string, error) { @@ -333,7 +334,7 @@ func formatIntegrationInputValidationError(features terminal.Features, tool *mcp colors := terminal.NewColorizer(features) lines := []string{ colors.Warning("Input validation failed"), - fmt.Sprintf("Tool: %s", colors.Bold(firstNonEmpty(tool.Title, tool.Name))), + fmt.Sprintf("Tool: %s", colors.Bold(commandutil.FirstNonEmpty(tool.Title, tool.Name))), fmt.Sprintf("Key: %s", tool.Name), "", colors.Accent("Why"), @@ -420,7 +421,7 @@ func formatMCPToolResultError(features terminal.Features, server *consumerapi.Ma lines := []string{ colors.Warning("Tool call failed"), fmt.Sprintf("Integration: %s", colors.Bold(integrationTitle(server))), - fmt.Sprintf("Tool: %s", colors.Bold(firstNonEmpty(tool.Title, tool.Name))), + fmt.Sprintf("Tool: %s", colors.Bold(commandutil.FirstNonEmpty(tool.Title, tool.Name))), } message := "" diff --git a/internal/commands/consumer/whoami.go b/internal/commands/consumer/whoami.go index 98d589e..9398f7b 100644 --- a/internal/commands/consumer/whoami.go +++ b/internal/commands/consumer/whoami.go @@ -4,6 +4,7 @@ import ( "fmt" "io" + "github.com/metorial/cli/internal/commandutil" "github.com/metorial/cli/internal/consumerapi" "github.com/metorial/cli/internal/terminal" "github.com/spf13/cobra" @@ -40,10 +41,10 @@ func newWhoamiCommand(env commandEnv) *cobra.Command { if runtime.Profile != nil { result.PortalID = runtime.Profile.PortalID result.PortalName = runtime.Profile.PortalName - result.InstanceID = firstNonEmpty(runtime.Profile.InstanceID, result.InstanceID) + result.InstanceID = commandutil.FirstNonEmpty(runtime.Profile.InstanceID, result.InstanceID) } - return env.write(command, result, func(writer io.Writer, features terminal.Features) error { + return env.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { return renderWhoami(writer, features, result) }) }, @@ -54,9 +55,9 @@ func renderWhoami(writer io.Writer, features terminal.Features, result whoamiRes colors := terminal.NewColorizer(features) profile := result.Profile - renderTitle(writer, features, firstNonEmpty(profile.Name, profile.Email, profile.ID), profile.Email) + commandutil.RenderTitle(writer, features, commandutil.FirstNonEmpty(profile.Name, profile.Email, profile.ID), profile.Email) _, _ = fmt.Fprintf(writer, "%s %s\n", colors.Muted("Consumer:"), profile.ID) - if portal := firstNonEmpty(result.PortalName, result.PortalID); portal != "" { + if portal := commandutil.FirstNonEmpty(result.PortalName, result.PortalID); portal != "" { _, _ = fmt.Fprintf(writer, "%s %s\n", colors.Muted("Portal: "), portal) } if result.InstanceID != "" { @@ -79,5 +80,5 @@ func renderWhoami(writer io.Writer, features terminal.Features, result whoamiRes _, _ = fmt.Fprintf(writer, " %s\n", line) } - return renderTips(writer, features, []string{"metorial catalog list", "metorial list"}) + return commandutil.RenderTips(writer, features, []string{"metorial catalog list", "metorial list"}) } diff --git a/internal/commands/files/command.go b/internal/commands/files/command.go index 3be0a76..fdc9f95 100644 --- a/internal/commands/files/command.go +++ b/internal/commands/files/command.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/metorial/cli/internal/commandutil" + "github.com/metorial/cli/internal/fileupload" "github.com/metorial/cli/internal/output" "github.com/metorial/cli/internal/resourcecmd" "github.com/spf13/cobra" @@ -30,7 +31,7 @@ func Attach(root *cobra.Command, ctx commandutil.Context) error { } func newUploadCommand(ctx commandutil.Context) *cobra.Command { - options := uploadOptions{} + options := fileupload.Options{} var name string command := &cobra.Command{ @@ -77,7 +78,7 @@ Examples: } defer cleanup() - response, uploadErr := newUploader(runtime).upload(command.Context(), source, options) + response, uploadErr := fileupload.New(runtime).Upload(command.Context(), source, options) if response != nil { writer := command.OutOrStdout() if uploadErr != nil { @@ -106,36 +107,31 @@ Examples: // prepareSource resolves the local path to a single file to upload, zipping // directories into a temporary archive that cleanup removes. -func prepareSource(local, name string, stderr io.Writer) (uploadSource, func(), error) { +func prepareSource(local, name string, stderr io.Writer) (fileupload.Source, func(), error) { noop := func() {} absolute, err := filepath.Abs(local) if err != nil { - return uploadSource{}, noop, fmt.Errorf("metorial: %w", err) + return fileupload.Source{}, noop, fmt.Errorf("metorial: %w", err) } info, err := os.Stat(absolute) if err != nil { - return uploadSource{}, noop, fmt.Errorf("metorial: %w", err) + return fileupload.Source{}, noop, fmt.Errorf("metorial: %w", err) } if !info.IsDir() { if !info.Mode().IsRegular() { - return uploadSource{}, noop, fmt.Errorf("metorial: %s is not a regular file", local) + return fileupload.Source{}, noop, fmt.Errorf("metorial: %s is not a regular file", local) } if name == "" { name = filepath.Base(absolute) } - return uploadSource{ - Path: absolute, - Name: name, - ContentType: contentTypeFor(name), - Size: info.Size(), - }, noop, nil + return fileupload.FileSource(absolute, name, fileupload.ContentTypeFor(name), info.Size()), noop, nil } archive, err := os.CreateTemp("", "metorial-upload-*.zip") if err != nil { - return uploadSource{}, noop, fmt.Errorf("metorial: %w", err) + return fileupload.Source{}, noop, fmt.Errorf("metorial: %w", err) } cleanup := func() { _ = os.Remove(archive.Name()) } @@ -145,20 +141,20 @@ func prepareSource(local, name string, stderr io.Writer) (uploadSource, func(), } if err != nil { cleanup() - return uploadSource{}, noop, err + return fileupload.Source{}, noop, err } for _, skipped := range result.Skipped { fmt.Fprintf(stderr, "metorial: skipping %s: not a regular file\n", skipped) } if result.Files == 0 { cleanup() - return uploadSource{}, noop, fmt.Errorf("metorial: %s has no files to upload", local) + return fileupload.Source{}, noop, fmt.Errorf("metorial: %s has no files to upload", local) } archiveInfo, err := os.Stat(archive.Name()) if err != nil { cleanup() - return uploadSource{}, noop, fmt.Errorf("metorial: %w", err) + return fileupload.Source{}, noop, fmt.Errorf("metorial: %w", err) } if name == "" { @@ -170,10 +166,5 @@ func prepareSource(local, name string, stderr io.Writer) (uploadSource, func(), } fmt.Fprintf(stderr, "metorial: zipped %d %s from %s (%s)\n", result.Files, noun, local, commandutil.HumanBytes(archiveInfo.Size())) - return uploadSource{ - Path: archive.Name(), - Name: name, - ContentType: "application/zip", - Size: archiveInfo.Size(), - }, cleanup, nil + return fileupload.FileSource(archive.Name(), name, "application/zip", archiveInfo.Size()), cleanup, nil } diff --git a/internal/commands/files/command_test.go b/internal/commands/files/command_test.go new file mode 100644 index 0000000..e3ec15f --- /dev/null +++ b/internal/commands/files/command_test.go @@ -0,0 +1,109 @@ +package files + +import ( + "archive/zip" + "bytes" + "io" + "os" + "path/filepath" + "slices" + "testing" +) + +func TestZipDirectoryStoresContentsRelativeToTheDirectory(t *testing.T) { + directory := t.TempDir() + writeFile(t, filepath.Join(directory, "Dockerfile"), "FROM alpine") + writeFile(t, filepath.Join(directory, "src", "main.go"), "package main") + if err := os.MkdirAll(filepath.Join(directory, "empty"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("Dockerfile", filepath.Join(directory, "link")); err != nil { + t.Fatal(err) + } + exclude := filepath.Join(directory, "archive.zip") + writeFile(t, exclude, "partial") + + buffer := &bytes.Buffer{} + result, err := zipDirectory(directory, buffer, exclude) + if err != nil { + t.Fatal(err) + } + + if result.Files != 2 { + t.Fatalf("expected 2 files, got %d", result.Files) + } + if !slices.Equal(result.Skipped, []string{"link"}) { + t.Fatalf("expected only the symlink to be skipped, got %v", result.Skipped) + } + + reader, err := zip.NewReader(bytes.NewReader(buffer.Bytes()), int64(buffer.Len())) + if err != nil { + t.Fatal(err) + } + names := []string{} + contents := map[string]string{} + for _, file := range reader.File { + names = append(names, file.Name) + if file.FileInfo().IsDir() { + continue + } + handle, err := file.Open() + if err != nil { + t.Fatal(err) + } + data, _ := io.ReadAll(handle) + _ = handle.Close() + contents[file.Name] = string(data) + } + + slices.Sort(names) + if expected := []string{"Dockerfile", "empty/", "src/", "src/main.go"}; !slices.Equal(names, expected) { + t.Fatalf("expected entries %v, got %v", expected, names) + } + if contents["src/main.go"] != "package main" { + t.Fatalf("unexpected nested file contents %q", contents["src/main.go"]) + } +} + +func TestPrepareSourceZipsDirectories(t *testing.T) { + directory := filepath.Join(t.TempDir(), "context") + writeFile(t, filepath.Join(directory, "Dockerfile"), "FROM alpine") + + stderr := &bytes.Buffer{} + source, cleanup, err := prepareSource(directory, "", stderr) + if err != nil { + t.Fatal(err) + } + + if source.Name != "context.zip" || source.ContentType != "application/zip" { + t.Fatalf("unexpected source %+v", source) + } + archive, err := source.Open() + if err != nil { + t.Fatalf("expected archive to exist: %v", err) + } + _ = archive.Close() + + cleanup() + if _, err := source.Open(); !os.IsNotExist(err) { + t.Fatalf("expected archive to be removed, got %v", err) + } +} + +func TestPrepareSourceRejectsDirectoriesWithoutFiles(t *testing.T) { + _, _, err := prepareSource(t.TempDir(), "", io.Discard) + if err == nil { + t.Fatal("expected an error for an empty directory") + } +} + +func writeFile(t *testing.T, path, contents string) { + t.Helper() + + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/internal/commands/files/upload.go b/internal/commands/files/upload.go deleted file mode 100644 index b9dd566..0000000 --- a/internal/commands/files/upload.go +++ /dev/null @@ -1,236 +0,0 @@ -package files - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "mime" - "mime/multipart" - "net/http" - "net/textproto" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/metorial/cli/internal/config" - "github.com/metorial/cli/internal/fetch" -) - -// Must stay in sync with maxDirectUploadSize in the files API. -const maxDirectUploadSize = 100 * 1024 * 1024 - -const filesEndpoint = "/files" - -type uploadOptions struct { - Purpose string - Title string - StoreID string - StorePath string - StoreReplace bool - InstanceID string -} - -type uploadSource struct { - Path string - Name string - ContentType string - Size int64 -} - -type uploader struct { - runtime config.Runtime - client *http.Client - directLimit int64 -} - -func newUploader(runtime config.Runtime) *uploader { - return &uploader{ - runtime: runtime, - client: http.DefaultClient, - directLimit: maxDirectUploadSize, - } -} - -// upload returns the API response alongside any error so callers can render -// API error bodies the same way as other commands. -func (u *uploader) upload(ctx context.Context, source uploadSource, options uploadOptions) (*fetch.Response, error) { - if source.Size <= u.directLimit { - return u.uploadDirect(source, options) - } - return u.uploadWithURL(ctx, source, options) -} - -func (u *uploader) uploadDirect(source uploadSource, options uploadOptions) (*fetch.Response, error) { - body, contentType, err := multipartBody(source, options) - if err != nil { - return nil, err - } - - return fetch.Do(u.runtime, fetch.Request{ - Method: http.MethodPost, - Target: filesEndpoint, - Headers: map[string]string{"Content-Type": contentType}, - Body: body, - }) -} - -func (u *uploader) uploadWithURL(ctx context.Context, source uploadSource, options uploadOptions) (*fetch.Response, error) { - request := map[string]any{ - "mode": "get_upload_url", - "purpose": options.Purpose, - "file_name": source.Name, - "file_size": source.Size, - "file_type": source.ContentType, - } - if options.Title != "" { - request["title"] = options.Title - } - if options.InstanceID != "" { - request["instance_id"] = options.InstanceID - } - if options.StoreID != "" { - request["store_id"] = options.StoreID - request["path"] = options.StorePath - request["store_replace"] = options.StoreReplace - } - - response, err := u.postJSON(request) - if err != nil { - return response, err - } - - var session struct { - ID string `json:"id"` - Upload struct { - URL string `json:"url"` - Method string `json:"method"` - } `json:"upload"` - } - if err := json.Unmarshal(response.Body, &session); err != nil { - return nil, fmt.Errorf("metorial: invalid upload session response: %w", err) - } - if session.ID == "" || session.Upload.URL == "" { - return nil, errors.New("metorial: the API did not return an upload URL") - } - - if err := u.put(ctx, session.Upload.Method, session.Upload.URL, source); err != nil { - return nil, err - } - - complete := map[string]any{ - "mode": "complete", - "file_upload_id": session.ID, - } - if options.InstanceID != "" { - complete["instance_id"] = options.InstanceID - } - return u.postJSON(complete) -} - -func (u *uploader) postJSON(body map[string]any) (*fetch.Response, error) { - payload, err := json.Marshal(body) - if err != nil { - return nil, err - } - - return fetch.Do(u.runtime, fetch.Request{ - Method: http.MethodPost, - Target: filesEndpoint, - Headers: map[string]string{"Content-Type": "application/json"}, - Body: payload, - }) -} - -// put sends the file to the presigned storage URL. The URL signature covers the -// content type, so it must match the file_type declared when requesting the URL, -// and Metorial credentials must not be forwarded to the storage backend. -func (u *uploader) put(ctx context.Context, method, url string, source uploadSource) error { - if method == "" { - method = http.MethodPut - } - - file, err := os.Open(source.Path) - if err != nil { - return fmt.Errorf("metorial: %w", err) - } - defer file.Close() - - request, err := http.NewRequestWithContext(ctx, method, url, file) - if err != nil { - return fmt.Errorf("metorial: invalid upload URL: %w", err) - } - request.ContentLength = source.Size - request.Header.Set("Content-Type", source.ContentType) - - response, err := u.client.Do(request) - if err != nil { - return fmt.Errorf("metorial: failed to upload %s: %w", source.Name, err) - } - defer response.Body.Close() - - if response.StatusCode < 200 || response.StatusCode > 299 { - detail, _ := io.ReadAll(io.LimitReader(response.Body, 2048)) - message := strings.TrimSpace(string(detail)) - if message == "" { - return fmt.Errorf("metorial: failed to upload %s: storage responded with %s", source.Name, response.Status) - } - return fmt.Errorf("metorial: failed to upload %s: storage responded with %s: %s", source.Name, response.Status, message) - } - - _, _ = io.Copy(io.Discard, response.Body) - return nil -} - -func multipartBody(source uploadSource, options uploadOptions) ([]byte, string, error) { - buffer := &bytes.Buffer{} - writer := multipart.NewWriter(buffer) - - fields := [][2]string{{"purpose", options.Purpose}} - if options.Title != "" { - fields = append(fields, [2]string{"title", options.Title}) - } - if options.InstanceID != "" { - fields = append(fields, [2]string{"instance_id", options.InstanceID}) - } - if options.StoreID != "" { - fields = append(fields, - [2]string{"store_id", options.StoreID}, - [2]string{"path", options.StorePath}, - [2]string{"store_replace", strconv.FormatBool(options.StoreReplace)}, - ) - } - for _, field := range fields { - if err := writer.WriteField(field[0], field[1]); err != nil { - return nil, "", err - } - } - - header := textproto.MIMEHeader{} - header.Set("Content-Disposition", fmt.Sprintf(`form-data; name="file"; filename="%s"`, quoteEscaper.Replace(source.Name))) - header.Set("Content-Type", source.ContentType) - part, err := writer.CreatePart(header) - if err != nil { - return nil, "", err - } - if err := copyFile(part, source.Path); err != nil { - return nil, "", fmt.Errorf("metorial: %w", err) - } - - if err := writer.Close(); err != nil { - return nil, "", err - } - return buffer.Bytes(), writer.FormDataContentType(), nil -} - -var quoteEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`) - -func contentTypeFor(name string) string { - if contentType := mime.TypeByExtension(filepath.Ext(name)); contentType != "" { - return contentType - } - return "application/octet-stream" -} diff --git a/internal/commands/files/upload_test.go b/internal/commands/files/upload_test.go deleted file mode 100644 index 8a36ef8..0000000 --- a/internal/commands/files/upload_test.go +++ /dev/null @@ -1,314 +0,0 @@ -package files - -import ( - "archive/zip" - "bytes" - "context" - "encoding/json" - "io" - "net/http" - "net/http/httptest" - "net/url" - "os" - "path/filepath" - "slices" - "testing" - - "github.com/metorial/cli/internal/config" -) - -func TestZipDirectoryStoresContentsRelativeToTheDirectory(t *testing.T) { - directory := t.TempDir() - writeFile(t, filepath.Join(directory, "Dockerfile"), "FROM alpine") - writeFile(t, filepath.Join(directory, "src", "main.go"), "package main") - if err := os.MkdirAll(filepath.Join(directory, "empty"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.Symlink("Dockerfile", filepath.Join(directory, "link")); err != nil { - t.Fatal(err) - } - exclude := filepath.Join(directory, "archive.zip") - writeFile(t, exclude, "partial") - - buffer := &bytes.Buffer{} - result, err := zipDirectory(directory, buffer, exclude) - if err != nil { - t.Fatal(err) - } - - if result.Files != 2 { - t.Fatalf("expected 2 files, got %d", result.Files) - } - if !slices.Equal(result.Skipped, []string{"link"}) { - t.Fatalf("expected only the symlink to be skipped, got %v", result.Skipped) - } - - reader, err := zip.NewReader(bytes.NewReader(buffer.Bytes()), int64(buffer.Len())) - if err != nil { - t.Fatal(err) - } - names := []string{} - contents := map[string]string{} - for _, file := range reader.File { - names = append(names, file.Name) - if file.FileInfo().IsDir() { - continue - } - handle, err := file.Open() - if err != nil { - t.Fatal(err) - } - data, _ := io.ReadAll(handle) - _ = handle.Close() - contents[file.Name] = string(data) - } - - slices.Sort(names) - if expected := []string{"Dockerfile", "empty/", "src/", "src/main.go"}; !slices.Equal(names, expected) { - t.Fatalf("expected entries %v, got %v", expected, names) - } - if contents["src/main.go"] != "package main" { - t.Fatalf("unexpected nested file contents %q", contents["src/main.go"]) - } -} - -func TestPrepareSourceZipsDirectories(t *testing.T) { - directory := filepath.Join(t.TempDir(), "context") - writeFile(t, filepath.Join(directory, "Dockerfile"), "FROM alpine") - - stderr := &bytes.Buffer{} - source, cleanup, err := prepareSource(directory, "", stderr) - if err != nil { - t.Fatal(err) - } - - if source.Name != "context.zip" || source.ContentType != "application/zip" { - t.Fatalf("unexpected source %+v", source) - } - if _, err := os.Stat(source.Path); err != nil { - t.Fatalf("expected archive to exist: %v", err) - } - - cleanup() - if _, err := os.Stat(source.Path); !os.IsNotExist(err) { - t.Fatalf("expected archive to be removed, got %v", err) - } -} - -func TestPrepareSourceRejectsDirectoriesWithoutFiles(t *testing.T) { - _, _, err := prepareSource(t.TempDir(), "", io.Discard) - if err == nil { - t.Fatal("expected an error for an empty directory") - } -} - -func TestUploadSendsSmallFilesAsMultipart(t *testing.T) { - local := filepath.Join(t.TempDir(), "notes.txt") - writeFile(t, local, "hello") - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.URL.Path != "/files" { - t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) - } - if err := r.ParseMultipartForm(1 << 20); err != nil { - t.Fatal(err) - } - - expected := map[string]string{ - "purpose": "generic", - "title": "Notes", - "instance_id": "inst_1", - "store_id": "store_1", - "path": "docs/notes.txt", - "store_replace": "true", - } - for key, value := range expected { - if got := r.FormValue(key); got != value { - t.Errorf("expected %s=%q, got %q", key, value, got) - } - } - - file, header, err := r.FormFile("file") - if err != nil { - t.Fatal(err) - } - data, _ := io.ReadAll(file) - if header.Filename != "notes.txt" || string(data) != "hello" { - t.Errorf("unexpected file %q with contents %q", header.Filename, data) - } - - writeJSON(w, http.StatusOK, map[string]any{"object": "file", "id": "file_1"}) - })) - defer server.Close() - - response, err := newTestUploader(t, server.URL, 1024).upload(context.Background(), localSource(t, local), uploadOptions{ - Purpose: "generic", - Title: "Notes", - InstanceID: "inst_1", - StoreID: "store_1", - StorePath: "docs/notes.txt", - StoreReplace: true, - }) - if err != nil { - t.Fatal(err) - } - assertFileID(t, response.Body, "file_1") -} - -func TestUploadSendsLargeFilesThroughAnUploadURL(t *testing.T) { - local := filepath.Join(t.TempDir(), "archive.bin") - writeFile(t, local, "large contents") - - steps := []string{} - var server *httptest.Server - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - switch { - case r.Method == http.MethodPut && r.URL.Path == "/storage": - steps = append(steps, "put") - if r.Header.Get("Authorization") != "" { - t.Error("expected no Metorial credentials on the storage request") - } - if got := r.Header.Get("Content-Type"); got != "application/octet-stream" { - t.Errorf("expected the declared content type, got %q", got) - } - data, _ := io.ReadAll(r.Body) - if string(data) != "large contents" { - t.Errorf("unexpected uploaded contents %q", data) - } - w.WriteHeader(http.StatusOK) - - case r.Method == http.MethodPost && r.URL.Path == "/files": - body := map[string]any{} - if err := json.NewDecoder(r.Body).Decode(&body); err != nil { - t.Fatal(err) - } - - switch body["mode"] { - case "get_upload_url": - steps = append(steps, "get_upload_url") - if body["file_name"] != "archive.bin" || body["file_size"] != float64(len("large contents")) || body["purpose"] != "skill_export" { - t.Errorf("unexpected upload URL request %v", body) - } - writeJSON(w, http.StatusOK, map[string]any{ - "object": "file_upload", - "id": "fup_1", - "upload": map[string]any{"url": server.URL + "/storage", "method": "PUT"}, - }) - case "complete": - steps = append(steps, "complete") - if body["file_upload_id"] != "fup_1" { - t.Errorf("unexpected complete request %v", body) - } - writeJSON(w, http.StatusOK, map[string]any{"object": "file", "id": "file_2"}) - default: - t.Errorf("unexpected mode %v", body["mode"]) - } - - default: - t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) - } - })) - defer server.Close() - - response, err := newTestUploader(t, server.URL, 4).upload(context.Background(), localSource(t, local), uploadOptions{Purpose: "skill_export"}) - if err != nil { - t.Fatal(err) - } - assertFileID(t, response.Body, "file_2") - - if expected := []string{"get_upload_url", "put", "complete"}; !slices.Equal(steps, expected) { - t.Fatalf("expected steps %v, got %v", expected, steps) - } -} - -func TestUploadReturnsAPIErrorResponses(t *testing.T) { - local := filepath.Join(t.TempDir(), "notes.txt") - writeFile(t, local, "hello") - - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusBadRequest, map[string]any{"status": "error", "message": "Unknown file purpose"}) - })) - defer server.Close() - - response, err := newTestUploader(t, server.URL, 1024).upload(context.Background(), localSource(t, local), uploadOptions{Purpose: "nope"}) - if err == nil { - t.Fatal("expected an error") - } - if response == nil || response.StatusCode != http.StatusBadRequest { - t.Fatalf("expected the error response to be returned, got %+v", response) - } -} - -func TestUploadReportsStorageFailures(t *testing.T) { - local := filepath.Join(t.TempDir(), "archive.bin") - writeFile(t, local, "large contents") - - var server *httptest.Server - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodPut { - http.Error(w, "SignatureDoesNotMatch", http.StatusForbidden) - return - } - writeJSON(w, http.StatusOK, map[string]any{ - "id": "fup_1", - "upload": map[string]any{"url": server.URL + "/storage", "method": "PUT"}, - }) - })) - defer server.Close() - - _, err := newTestUploader(t, server.URL, 4).upload(context.Background(), localSource(t, local), uploadOptions{Purpose: "generic"}) - if err == nil || !bytes.Contains([]byte(err.Error()), []byte("SignatureDoesNotMatch")) { - t.Fatalf("expected the storage error to be reported, got %v", err) - } -} - -func newTestUploader(t *testing.T, host string, directLimit int64) *uploader { - t.Helper() - - hostURL, err := url.Parse(host) - if err != nil { - t.Fatal(err) - } - uploader := newUploader(config.Runtime{APIKey: "metorial_sk_test", APIHost: host, APIHostURL: hostURL}) - uploader.directLimit = directLimit - return uploader -} - -func localSource(t *testing.T, path string) uploadSource { - t.Helper() - - source, _, err := prepareSource(path, "", io.Discard) - if err != nil { - t.Fatal(err) - } - return source -} - -func writeFile(t *testing.T, path, contents string) { - t.Helper() - - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { - t.Fatal(err) - } -} - -func writeJSON(w http.ResponseWriter, status int, value any) { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _ = json.NewEncoder(w).Encode(value) -} - -func assertFileID(t *testing.T, body []byte, expected string) { - t.Helper() - - file := map[string]any{} - if err := json.Unmarshal(body, &file); err != nil { - t.Fatal(err) - } - if file["id"] != expected { - t.Fatalf("expected file %s, got %v", expected, file["id"]) - } -} diff --git a/internal/commands/resources/command_test.go b/internal/commands/resources/command_test.go index 01703ff..54a530b 100644 --- a/internal/commands/resources/command_test.go +++ b/internal/commands/resources/command_test.go @@ -199,6 +199,55 @@ func TestBuildResourceBodyMergesExplicitJSONAndFlags(t *testing.T) { } } +func TestBuildResourceBodySendsNullForNullableStringFlags(t *testing.T) { + t.Parallel() + + resource := resourcecmd.ResourceSpec{Plural: "files", Singular: "files"} + operation := resourcecmd.OperationSpec{ + Name: resourcecmd.OperationName("put"), + Method: "PUT", + Path: "/skills/{skillId}/files", + Short: "Write a file", + Flags: []resourcecmd.FlagSpec{ + {Name: "if-match", Type: resourcecmd.FlagString, Target: "body.if_match", Usage: "if_match", Nullable: true}, + {Name: "content", Type: resourcecmd.FlagString, Target: "body.content", Usage: "content"}, + }, + } + + build := func(ifMatch string) map[string]any { + t.Helper() + + command, err := newPublicResourceAction(&app.App{}, &rootOptionsView{}, resource, operation) + if err != nil { + t.Fatalf("newPublicResourceAction() error = %v", err) + } + if err := command.Flags().Set("if-match", ifMatch); err != nil { + t.Fatalf("Set(if-match) error = %v", err) + } + if err := command.Flags().Set("content", "null"); err != nil { + t.Fatalf("Set(content) error = %v", err) + } + + body, err := buildResourceBody(command, resource, operation, nil, "") + if err != nil { + t.Fatalf("buildResourceBody() error = %v", err) + } + return body + } + + body := build("null") + if value, ok := body["if_match"]; !ok || value != nil { + t.Fatalf("if_match = %#v, present = %v, want JSON null", value, ok) + } + if body["content"] != "null" { + t.Fatalf("content = %#v, want the string \"null\" for a non-nullable flag", body["content"]) + } + + if body := build("etag_123"); body["if_match"] != "etag_123" { + t.Fatalf("if_match = %#v, want etag_123", body["if_match"]) + } +} + func sampleAPIKeyCreateOperation() resourcecmd.OperationSpec { return resourcecmd.OperationSpec{ Name: resourcecmd.OperationCreate, diff --git a/internal/commands/resources/public.go b/internal/commands/resources/public.go index 7e25c06..7681048 100644 --- a/internal/commands/resources/public.go +++ b/internal/commands/resources/public.go @@ -373,7 +373,11 @@ func buildResourceBody(command *cobra.Command, resource resourcecmd.ResourceSpec if err != nil { return nil, err } - body[key] = value + if flag.Nullable && value == "null" { + body[key] = nil + } else { + body[key] = value + } case resourcecmd.FlagBool: value, err := command.Flags().GetBool(flag.Name) if err != nil { diff --git a/internal/commands/skills/command.go b/internal/commands/skills/command.go new file mode 100644 index 0000000..65cbc98 --- /dev/null +++ b/internal/commands/skills/command.go @@ -0,0 +1,608 @@ +package skills + +import ( + "errors" + "fmt" + "io" + "os" + "path" + "strings" + + "github.com/metorial/cli/internal/commandutil" + "github.com/metorial/cli/internal/output" + "github.com/metorial/cli/internal/resourcecmd" + "github.com/metorial/cli/internal/skillsapi" + "github.com/metorial/cli/internal/skillsync" + "github.com/metorial/cli/internal/terminal" + "github.com/spf13/cobra" +) + +const defaultPageLimit = 15 + +type env struct { + ctx commandutil.Context +} + +func NewCommand(ctx commandutil.Context) *cobra.Command { + e := env{ctx: ctx} + + command := &cobra.Command{ + Use: "skills", + Aliases: []string{"skill"}, + Short: "Create, edit and sync your Metorial skills", + Long: strings.TrimSpace(` +Work with Metorial skills from your terminal. + +Pull a skill into a local folder, edit it with any tool, and push the changes back. + +Examples: + metorial skills list + metorial skills pull release-notes ./release-notes + metorial skills status ./release-notes + metorial skills push ./release-notes +`), + } + + command.AddCommand( + e.newListCommand(), + e.newGetCommand(), + e.newCreateCommand(), + e.newFilesCommand(), + ) + command.AddCommand(e.localCommands()...) + + return command +} + +func Attach(root *cobra.Command, ctx commandutil.Context) error { + skills := resourcecmd.CommandByName(root, "skills") + if skills == nil { + return errors.New("metorial: the generated skills command is missing") + } + + e := env{ctx: ctx} + for _, command := range e.localCommands() { + commandutil.ConfigureCommand(command) + commandutil.SetCommandCategory(command, commandutil.CommandCategoryResource) + skills.AddCommand(command) + } + + return nil +} + +func (e env) localCommands() []*cobra.Command { + return []*cobra.Command{ + e.newPullCommand(), + e.newPushCommand(), + e.newStatusCommand(), + } +} + +func (e env) client() (*skillsapi.Client, error) { + runtime, err := e.ctx.ResolveRuntime() + if err != nil { + return nil, err + } + + return skillsapi.New(runtime) +} + +func (e env) newListCommand() *cobra.Command { + var search string + limit := float64(defaultPageLimit) + + command := &cobra.Command{ + Use: "list", + Short: "List the skills you have access to", + Args: cobra.NoArgs, + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + result, err := client.ListSkills(skillsapi.ListOptions{Limit: limit}, search) + if err != nil { + return err + } + + return e.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { + if len(result.Items) == 0 { + _, _ = fmt.Fprintln(writer, "No skills found.") + return nil + } + + colors := terminal.NewColorizer(features) + table := output.Table{ + Columns: []string{colors.Accent("ID"), colors.Accent("Slug"), colors.Accent("Name")}, + Features: features, + MaxWidth: features.Width, + } + for _, skill := range result.Items { + table.Rows = append(table.Rows, []string{skill.ID, commandutil.OptionalString(skill.Slug), skill.Name}) + } + if err := table.Render(writer); err != nil { + return err + } + + return commandutil.RenderTips(writer, features, []string{ + fmt.Sprintf("%s skills pull %s", binaryName(command), commandutil.ShellQuote(skillReference(result.Items[0]))), + }) + }) + }, + } + + command.Flags().StringVar(&search, "search", "", "Filter skills by name or description") + command.Flags().Float64Var(&limit, "limit", limit, "Limit the number of skills") + return command +} + +func (e env) newGetCommand() *cobra.Command { + return &cobra.Command{ + Use: "get ", + Short: "Show a skill and its files", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + skill, err := client.ResolveSkill(args[0]) + if err != nil { + return err + } + + files, err := client.ListFiles(skill.ID) + if err != nil { + return err + } + + value := map[string]any{"skill": skill, "files": files.Items} + return e.ctx.Write(command, value, func(writer io.Writer, features terminal.Features) error { + commandutil.RenderTitle(writer, features, skill.Name, commandutil.FirstNonEmpty(commandutil.OptionalString(skill.Description), skill.ID)) + return renderFileTable(writer, features, files.Items) + }) + }, + } +} + +func (e env) newCreateCommand() *cobra.Command { + input := skillsapi.CreateSkillInput{} + + command := &cobra.Command{ + Use: "create ", + Short: "Create a new skill", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + input.Name = args[0] + skill, err := client.CreateSkill(input) + if err != nil { + return err + } + + return e.ctx.Write(command, skill, func(writer io.Writer, features terminal.Features) error { + colors := terminal.NewColorizer(features) + _, _ = fmt.Fprintf(writer, "%s %s (%s)\n", colors.Success("Created skill"), skill.Name, skill.ID) + return commandutil.RenderTips(writer, features, []string{ + fmt.Sprintf("%s skills pull %s", binaryName(command), commandutil.ShellQuote(skillReference(*skill))), + }) + }) + }, + } + + command.Flags().StringVar(&input.Description, "description", "", "A short description of the skill") + command.Flags().StringVar(&input.TemplateID, "template", "", "Create the skill from this skill template") + return command +} + +func (e env) newPullCommand() *cobra.Command { + options := skillsync.PullOptions{} + + command := &cobra.Command{ + Use: "pull [dir]", + Short: "Download a skill into a local folder", + Long: strings.TrimSpace(` +Download every file of a skill into a local folder. The folder defaults to the +skill's slug. Running pull again in the same folder updates it, as long as it has +no local changes (use --force to discard them). +`), + Args: cobra.RangeArgs(1, 2), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + skill, err := client.ResolveSkill(args[0]) + if err != nil { + return err + } + + dir := commandutil.FirstNonEmpty(commandutil.OptionalArg(args, 1), skillReference(*skill)) + result, err := skillsync.Pull(client, skill, dir, options) + if err != nil { + return err + } + + return e.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { + colors := terminal.NewColorizer(features) + _, _ = fmt.Fprintf(writer, "%s %d file(s) of %s into %s\n", colors.Success("Pulled"), len(result.Written), skill.Name, dir) + if len(result.Removed) > 0 { + _, _ = fmt.Fprintf(writer, "%s %s\n", colors.Muted("Removed:"), strings.Join(result.Removed, ", ")) + } + if len(result.Skipped) > 0 { + _, _ = fmt.Fprintf(writer, "%s %s\n", colors.Warning("Skipped ignored paths:"), strings.Join(result.Skipped, ", ")) + } + return commandutil.RenderTips(writer, features, []string{ + fmt.Sprintf("%s skills status %s", binaryName(command), commandutil.ShellQuote(dir)), + fmt.Sprintf("%s skills push %s", binaryName(command), commandutil.ShellQuote(dir)), + }) + }) + }, + } + + command.Flags().BoolVar(&options.Force, "force", false, "Overwrite local changes and non-empty folders") + return command +} + +func (e env) newPushCommand() *cobra.Command { + options := skillsync.PushOptions{} + + command := &cobra.Command{ + Use: "push [dir]", + Short: "Upload local changes of a pulled skill", + Long: strings.TrimSpace(` +Upload the files you added, changed or deleted since the last pull or push. + +Before sending each batch of up to 100 changes, push checks that none of the files +it touches changed remotely since your last pull, and stops if they did. Pull to +update your copy first, or pass --force to overwrite the remote changes. Batches are +not atomic: if the server rejects a change, the changes before it stay applied and +are recorded locally, so status shows what is left. + +Push never deletes /SKILL.md. + +These paths are never pushed or pulled: .git, .metorial, .DS_Store, node_modules, +.idea, .vscode, .env, .env.*, *.swp, *~ and .metorial-* temporary files. Add up to +30 more patterns (one per line, matched per path segment, or against the whole path +when they contain a /) to a .metorialignore file in the skill folder. The +.metorialignore file is itself synced. +`), + Args: cobra.MaximumNArgs(1), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + dir := commandutil.FirstNonEmpty(commandutil.OptionalArg(args, 0), ".") + result, err := skillsync.Push(client, dir, options) + if err != nil { + return err + } + + return e.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { + colors := terminal.NewColorizer(features) + if len(result.Changes) == 0 { + _, _ = fmt.Fprintln(writer, "Nothing to push.") + return nil + } + + verb := colors.Success("Pushed") + if result.DryRun { + verb = colors.Notice("Would push") + } + _, _ = fmt.Fprintf(writer, "%s %d change(s)\n", verb, len(result.Changes)) + renderChanges(writer, features, result.Changes) + return nil + }) + }, + } + + command.Flags().BoolVar(&options.Force, "force", false, "Overwrite changes made remotely since the last pull") + command.Flags().BoolVar(&options.DryRun, "dry-run", false, "Show what would be pushed without changing anything") + return command +} + +func (e env) newStatusCommand() *cobra.Command { + return &cobra.Command{ + Use: "status [dir]", + Short: "Show local and remote changes of a pulled skill", + Args: cobra.MaximumNArgs(1), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + dir := commandutil.FirstNonEmpty(commandutil.OptionalArg(args, 0), ".") + status, err := skillsync.GetStatus(client, dir) + if err != nil { + return err + } + + return e.ctx.Write(command, status, func(writer io.Writer, features terminal.Features) error { + colors := terminal.NewColorizer(features) + if status.Clean() { + _, _ = fmt.Fprintln(writer, "Up to date with the remote skill.") + return nil + } + + if len(status.Local) > 0 { + _, _ = fmt.Fprintln(writer, colors.Bold("Local changes")) + renderChanges(writer, features, status.Local) + } + if len(status.Remote) > 0 { + _, _ = fmt.Fprintln(writer, colors.Bold("Remote changes")) + renderChanges(writer, features, status.Remote) + } + + reference := commandutil.ShellQuote(commandutil.FirstNonEmpty(status.SkillSlug, status.SkillID)) + pull := fmt.Sprintf("%s skills pull %s %s", binaryName(command), reference, commandutil.ShellQuote(dir)) + push := fmt.Sprintf("%s skills push %s", binaryName(command), commandutil.ShellQuote(dir)) + + tips := []string{} + switch { + case status.Conflicted(): + _, _ = fmt.Fprintf(writer, "%s %s\n", colors.Warning("Changed on both sides:"), strings.Join(status.Conflicts(), ", ")) + pull += " --force" + push += " --force" + width := max(len(pull), len(push)) + tips = append(tips, + fmt.Sprintf("%-*s # discard local changes", width, pull), + fmt.Sprintf("%-*s # overwrite remote changes", width, push), + ) + case len(status.Local) > 0 && len(status.Remote) > 0: + tips = append(tips, push, pull) + case len(status.Remote) > 0: + tips = append(tips, pull) + default: + tips = append(tips, push) + } + return commandutil.RenderTips(writer, features, tips) + }) + }, + } +} + +func (e env) newFilesCommand() *cobra.Command { + command := &cobra.Command{ + Use: "files", + Short: "Read and edit individual files of a skill", + } + + command.AddCommand( + e.newFilesListCommand(), + e.newFilesCatCommand(), + e.newFilesPutCommand(), + e.newFilesRemoveCommand(), + e.newFilesMoveCommand(), + ) + + return command +} + +func (e env) newFilesListCommand() *cobra.Command { + return &cobra.Command{ + Use: "list ", + Aliases: []string{"ls"}, + Short: "List the files of a skill", + Args: cobra.ExactArgs(1), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + skill, err := client.ResolveSkill(args[0]) + if err != nil { + return err + } + + files, err := client.ListFiles(skill.ID) + if err != nil { + return err + } + + return e.ctx.Write(command, files, func(writer io.Writer, features terminal.Features) error { + return renderFileTable(writer, features, files.Items) + }) + }, + } +} + +func (e env) newFilesCatCommand() *cobra.Command { + return &cobra.Command{ + Use: "cat ", + Short: "Print the content of a skill file", + Args: cobra.ExactArgs(2), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + skill, err := client.ResolveSkill(args[0]) + if err != nil { + return err + } + + file, content, err := client.ReadFile(skill.ID, args[1]) + if err != nil { + return err + } + + _, err = command.OutOrStdout().Write(skillsapi.RestoreTrailingNewline(file.Kind, content)) + return err + }, + } +} + +func (e env) newFilesPutCommand() *cobra.Command { + var source string + var ifMatch string + + command := &cobra.Command{ + Use: "put ", + Short: "Create or replace a skill file from a local file or stdin", + Args: cobra.ExactArgs(2), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + skill, err := client.ResolveSkill(args[0]) + if err != nil { + return err + } + + content, err := readSource(command, source) + if err != nil { + return err + } + + isText, err := skillsapi.IsText(args[1], content) + if err != nil { + return err + } + + change := skillsapi.ChangeInput{Path: args[1], IfMatch: skillsapi.ParseIfMatch(ifMatch)} + if isText { + text := string(content) + change.Content = &text + } else { + uploaded, err := client.UploadFile(path.Base(args[1]), content) + if err != nil { + return err + } + change.FileID = uploaded.ID + } + + result, err := client.PutFile(skill.ID, change) + if err != nil { + return err + } + + return e.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { + colors := terminal.NewColorizer(features) + _, _ = fmt.Fprintf(writer, "%s %s\n", colors.Success("Wrote"), result.Path) + return nil + }) + }, + } + + command.Flags().StringVarP(&source, "file", "f", "", "Read the content from this local file instead of stdin") + command.Flags().StringVar(&ifMatch, "if-match", "", "Only write if the remote file still has this etag, or \"absent\" if it must not exist yet") + return command +} + +func (e env) newFilesRemoveCommand() *cobra.Command { + var ifMatch string + + command := &cobra.Command{ + Use: "remove ", + Aliases: []string{"rm"}, + Short: "Delete a skill file", + Args: cobra.ExactArgs(2), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + skill, err := client.ResolveSkill(args[0]) + if err != nil { + return err + } + + etag, err := skillsapi.ParseEtag(ifMatch) + if err != nil { + return err + } + + result, err := client.DeleteFile(skill.ID, args[1], etag) + if err != nil { + return err + } + + return e.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { + colors := terminal.NewColorizer(features) + _, _ = fmt.Fprintf(writer, "%s %s\n", colors.Success("Deleted"), result.Path) + return nil + }) + }, + } + + command.Flags().StringVar(&ifMatch, "if-match", "", "Only delete if the remote file still has this etag") + return command +} + +func (e env) newFilesMoveCommand() *cobra.Command { + var ifMatch string + + command := &cobra.Command{ + Use: "move ", + Aliases: []string{"mv"}, + Short: "Move or rename a skill file", + Args: cobra.ExactArgs(3), + RunE: func(command *cobra.Command, args []string) error { + client, err := e.client() + if err != nil { + return err + } + + skill, err := client.ResolveSkill(args[0]) + if err != nil { + return err + } + + etag, err := skillsapi.ParseEtag(ifMatch) + if err != nil { + return err + } + + change := skillsapi.ChangeInput{Path: args[1], ToPath: args[2]} + if etag != "" { + change.IfMatch = skillsapi.MatchEtag(etag) + } + + result, err := client.MoveFile(skill.ID, change) + if err != nil { + return err + } + + return e.ctx.Write(command, result, func(writer io.Writer, features terminal.Features) error { + colors := terminal.NewColorizer(features) + from := commandutil.FirstNonEmpty(commandutil.OptionalString(result.PreviousPath), args[1]) + _, _ = fmt.Fprintf(writer, "%s %s -> %s\n", colors.Success("Moved"), from, result.Path) + return nil + }) + }, + } + + command.Flags().StringVar(&ifMatch, "if-match", "", "Only move if the remote file still has this etag") + return command +} + +func readSource(command *cobra.Command, source string) ([]byte, error) { + if source != "" { + content, err := os.ReadFile(source) + if err != nil { + return nil, fmt.Errorf("metorial: failed to read %s: %w", source, err) + } + return content, nil + } + + content, err := io.ReadAll(command.InOrStdin()) + if err != nil { + return nil, fmt.Errorf("metorial: failed to read stdin: %w", err) + } + return content, nil +} diff --git a/internal/commands/skills/command_test.go b/internal/commands/skills/command_test.go new file mode 100644 index 0000000..ec33b12 --- /dev/null +++ b/internal/commands/skills/command_test.go @@ -0,0 +1,79 @@ +package skills + +import ( + "bytes" + "strings" + "testing" + + "github.com/metorial/cli/internal/app" + "github.com/metorial/cli/internal/commandutil" + "github.com/metorial/cli/internal/config" + "github.com/metorial/cli/internal/skillsapi/skillstest" + "github.com/metorial/cli/internal/terminal" +) + +func runSkills(t *testing.T, fake *skillstest.Server, stdin string, args ...string) error { + t.Helper() + + features := terminal.Features{Width: 120} + application := app.NewWithIO(strings.NewReader(stdin), &bytes.Buffer{}, &bytes.Buffer{}, features, features) + application.SetSubjectType(config.SubjectTypeConsumer) + + command := NewCommand(commandutil.NewContext(application, &commandutil.RootOptions{ + APIKey: "test-token", + APIHost: fake.URL, + Format: "json", + })) + command.SetIn(strings.NewReader(stdin)) + command.SetOut(&bytes.Buffer{}) + command.SetErr(&bytes.Buffer{}) + command.SetArgs(args) + return command.Execute() +} + +func TestFilesPutMoveRemoveHonorIfMatch(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/notes.md", "notes") + + if err := runSkills(t, fake, "new", "files", "put", skillstest.SkillSlug, "/notes.md", "--if-match", "absent"); err == nil { + t.Fatal("put --if-match absent should fail for an existing file") + } + if err := runSkills(t, fake, "", "files", "mv", skillstest.SkillSlug, "/notes.md", "/docs/notes.md", "--if-match", "stale"); err == nil { + t.Fatal("mv with a stale etag should fail") + } + if err := runSkills(t, fake, "", "files", "mv", skillstest.SkillSlug, "/notes.md", "/docs/notes.md"); err != nil { + t.Fatalf("mv error = %v", err) + } + if content, _ := fake.Content("/docs/notes.md"); content != "notes" { + t.Fatalf("moved content = %q", content) + } + if err := runSkills(t, fake, "", "files", "rm", skillstest.SkillSlug, "/docs/notes.md", "--if-match", " absent "); err == nil || !strings.Contains(err.Error(), "only applies to writes") { + t.Fatalf("rm --if-match absent error = %v", err) + } + if err := runSkills(t, fake, "", "files", "rm", skillstest.SkillSlug, "/docs/notes.md"); err != nil { + t.Fatalf("rm error = %v", err) + } + if _, exists := fake.Content("/docs/notes.md"); exists { + t.Fatal("notes.md should be deleted") + } +} + +func TestFilesPutUploadsBinaryWithConsumerToken(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + if err := runSkills(t, fake, "\x89PNG\x00binary", "files", "put", skillstest.SkillSlug, "/assets/logo.png"); err != nil { + t.Fatalf("put error = %v", err) + } + if fake.Uploads != 1 || len(fake.StorageErrors) != 0 { + t.Fatalf("uploads = %d, storage errors = %v; want one signed upload", fake.Uploads, fake.StorageErrors) + } + if content, _ := fake.Content("/assets/logo.png"); content != "\x89PNG\x00binary" { + t.Fatalf("remote logo.png = %q", content) + } +} diff --git a/internal/commands/skills/render.go b/internal/commands/skills/render.go new file mode 100644 index 0000000..6a8897d --- /dev/null +++ b/internal/commands/skills/render.go @@ -0,0 +1,56 @@ +package skills + +import ( + "fmt" + "io" + + "github.com/metorial/cli/internal/commandutil" + "github.com/metorial/cli/internal/output" + "github.com/metorial/cli/internal/skillsapi" + "github.com/metorial/cli/internal/skillsync" + "github.com/metorial/cli/internal/terminal" + "github.com/spf13/cobra" +) + +func renderFileTable(writer io.Writer, features terminal.Features, files []skillsapi.File) error { + if len(files) == 0 { + _, _ = fmt.Fprintln(writer, "This skill has no files.") + return nil + } + + colors := terminal.NewColorizer(features) + table := output.Table{ + Columns: []string{colors.Accent("Path"), colors.Accent("Kind"), colors.Accent("Size")}, + Features: features, + MaxWidth: features.Width, + } + for _, file := range files { + table.Rows = append(table.Rows, []string{file.Path, file.Kind, commandutil.HumanBytes(file.Size)}) + } + + return table.Render(writer) +} + +func renderChanges(writer io.Writer, features terminal.Features, changes []skillsync.Change) { + colors := terminal.NewColorizer(features) + + for _, change := range changes { + label := colors.Success("added ") + switch change.Kind { + case skillsync.ChangeModified: + label = colors.Notice("modified") + case skillsync.ChangeDeleted: + label = colors.Warning("deleted ") + } + + _, _ = fmt.Fprintf(writer, " %s %s\n", label, change.Path) + } +} + +func binaryName(command *cobra.Command) string { + return command.Root().Name() +} + +func skillReference(skill skillsapi.Skill) string { + return commandutil.FirstNonEmpty(commandutil.OptionalString(skill.Slug), skill.ID) +} diff --git a/internal/commandutil/render.go b/internal/commandutil/render.go new file mode 100644 index 0000000..8a88c8e --- /dev/null +++ b/internal/commandutil/render.go @@ -0,0 +1,84 @@ +package commandutil + +import ( + "fmt" + "io" + "regexp" + "strings" + + "github.com/metorial/cli/internal/output" + "github.com/metorial/cli/internal/terminal" + "github.com/spf13/cobra" +) + +func (c Context) Write(command *cobra.Command, value any, render func(io.Writer, terminal.Features) error) error { + structured, err := c.Structured() + if err != nil { + return err + } + + features := c.App.StdoutFeatures() + if !structured { + return WriteValue(command.OutOrStdout(), features, c.Options.Format, value) + } + + return render(command.OutOrStdout(), features) +} + +func (c Context) Structured() (bool, error) { + format, err := output.ParseFormat(c.Options.Format) + if err != nil { + return false, err + } + + return format == output.FormatStructured, nil +} + +func RenderTitle(writer io.Writer, features terminal.Features, title, subtitle string) { + colors := terminal.NewColorizer(features) + _, _ = fmt.Fprintln(writer, colors.Bold(title)) + if strings.TrimSpace(subtitle) != "" { + _, _ = fmt.Fprintln(writer, colors.Muted(subtitle)) + } + _, _ = fmt.Fprintln(writer) +} + +func RenderTips(writer io.Writer, features terminal.Features, tips []string) error { + if len(tips) == 0 { + return nil + } + + colors := terminal.NewColorizer(features) + _, _ = fmt.Fprintln(writer) + _, _ = fmt.Fprintln(writer, colors.Notice("Tips")) + for _, tip := range tips { + if strings.TrimSpace(tip) == "" { + continue + } + _, _ = fmt.Fprintf(writer, " %s\n", tip) + } + return nil +} + +func OptionalArg(args []string, index int) string { + if index < 0 || index >= len(args) { + return "" + } + return args[index] +} + +func OptionalString(value *string) string { + if value == nil { + return "" + } + return strings.TrimSpace(*value) +} + +var shellSafe = regexp.MustCompile(`^[A-Za-z0-9_@%+=:,./-]+$`) + +func ShellQuote(value string) string { + if shellSafe.MatchString(value) { + return value + } + return "'" + strings.ReplaceAll(value, "'", `'"'"'`) + "'" +} diff --git a/internal/commandutil/render_test.go b/internal/commandutil/render_test.go new file mode 100644 index 0000000..e61332b --- /dev/null +++ b/internal/commandutil/render_test.go @@ -0,0 +1,24 @@ +package commandutil + +import "testing" + +func TestShellQuote(t *testing.T) { + for value, want := range map[string]string{ + "": "''", + "release-notes": "release-notes", + "./skills/a_b@1.0": "./skills/a_b@1.0", + "key=value,x:y+z%": "key=value,x:y+z%", + "my skill": "'my skill'", + "it's": `'it'"'"'s'`, + "$HOME": "'$HOME'", + "a;rm -rf /": "'a;rm -rf /'", + "`id`": "'`id`'", + "*.md": "'*.md'", + "~/skills": "'~/skills'", + "a&b|c>dd= 300 { - return describeError(target, responseError(response)) - } - - if output == nil || len(response.Body) == 0 { - return nil - } - if err := json.Unmarshal(response.Body, output); err != nil { - return fmt.Errorf("metorial: failed to decode response from %s: %w", target, err) - } - - return nil -} - -func responseError(response *fetch.Response) *metorial.Error { - apiErr := &metorial.Error{StatusCode: response.StatusCode} - - body := struct { - Code string `json:"code"` - Message string `json:"message"` - }{} - if json.Unmarshal(response.Body, &body) == nil { - apiErr.Code = body.Code - apiErr.Message = strings.TrimSpace(body.Message) - } - if apiErr.Message == "" { - apiErr.Message = http.StatusText(response.StatusCode) - } - - return apiErr + return apiclient.Client{Runtime: c.runtime, Describe: describeError}.Do(method, target, body, output) } func describeError(target string, err error) error { @@ -226,34 +172,3 @@ func describeError(target string, err error) error { return err } - -func listQuery(options ListOptions, extra url.Values) string { - values := url.Values{} - for key, items := range extra { - for _, item := range items { - values.Add(key, item) - } - } - - if options.Limit > 0 { - values.Set("limit", strconv.FormatFloat(options.Limit, 'f', -1, 64)) - } - if after := strings.TrimSpace(options.After); after != "" { - values.Set("after", after) - } - if before := strings.TrimSpace(options.Before); before != "" { - values.Set("before", before) - } - if cursor := strings.TrimSpace(options.Cursor); cursor != "" { - values.Set("cursor", cursor) - } - - if len(values) == 0 { - return "" - } - return "?" + values.Encode() -} - -func pathSegment(value string) string { - return url.PathEscape(strings.TrimSpace(value)) -} diff --git a/internal/consumerapi/types.go b/internal/consumerapi/types.go index c01e690..959af95 100644 --- a/internal/consumerapi/types.go +++ b/internal/consumerapi/types.go @@ -1,6 +1,10 @@ package consumerapi -import "time" +import ( + "time" + + "github.com/metorial/cli/internal/apiclient" +) const ( ProviderTypeProviderTemplate = "provider_template" @@ -16,17 +20,10 @@ const ( SetupStatusDeleted = "deleted" ) -type Pagination struct { - HasMoreBefore bool `json:"has_more_before"` - HasMoreAfter bool `json:"has_more_after"` -} - -type ListOptions struct { - Limit float64 - After string - Before string - Cursor string -} +type ( + Pagination = apiclient.Pagination + ListOptions = apiclient.ListOptions +) type Profile struct { Object string `json:"object"` diff --git a/internal/fileupload/upload.go b/internal/fileupload/upload.go new file mode 100644 index 0000000..377e69d --- /dev/null +++ b/internal/fileupload/upload.go @@ -0,0 +1,199 @@ +package fileupload + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "strings" + "time" + + "github.com/metorial/cli/internal/config" + "github.com/metorial/cli/internal/fetch" +) + +const ( + filesEndpoint = "/files" + signedUploadsUnavailable = "Signed upload URLs are not available" +) + +type Options struct { + Purpose string + Title string + StoreID string + StorePath string + StoreReplace bool + InstanceID string +} + +type Source struct { + Name string + ContentType string + Size int64 + Open func() (io.ReadCloser, error) +} + +func FileSource(filePath, name, contentType string, size int64) Source { + return Source{ + Name: name, + ContentType: contentType, + Size: size, + Open: func() (io.ReadCloser, error) { return os.Open(filePath) }, + } +} + +func BytesSource(name string, content []byte) Source { + return Source{ + Name: name, + ContentType: ContentTypeFor(name), + Size: int64(len(content)), + Open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(content)), nil }, + } +} + +func ContentTypeFor(name string) string { + if contentType := mime.TypeByExtension(path.Ext(name)); contentType != "" { + return contentType + } + return "application/octet-stream" +} + +type Uploader struct { + Runtime config.Runtime + // Never sent to the storage upload. + Headers map[string]string + Client *http.Client +} + +func New(runtime config.Runtime) *Uploader { + return &Uploader{Runtime: runtime, Client: NewTransferClient()} +} + +func NewTransferClient() *http.Client { + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = time.Minute + return &http.Client{Transport: transport, Timeout: 30 * time.Minute} +} + +// Returns the API response even on error so callers can render the error body. +func (u *Uploader) Upload(ctx context.Context, source Source, options Options) (*fetch.Response, error) { + request := map[string]any{ + "mode": "get_upload_url", + "purpose": options.Purpose, + "file_name": source.Name, + "file_size": source.Size, + "file_type": source.ContentType, + } + if options.Title != "" { + request["title"] = options.Title + } + if options.InstanceID != "" { + request["instance_id"] = options.InstanceID + } + if options.StoreID != "" { + request["store_id"] = options.StoreID + request["path"] = options.StorePath + request["store_replace"] = options.StoreReplace + } + + response, err := u.postJSON(request) + if err != nil { + if response != nil && response.StatusCode == http.StatusBadRequest && strings.Contains(err.Error(), signedUploadsUnavailable) { + err = fmt.Errorf("%w\nThe server's file storage can't issue signed upload URLs; configure UPLOAD_HOST or presign-capable object storage", err) + } + return response, err + } + + var session struct { + ID string `json:"id"` + Upload struct { + URL string `json:"url"` + Method string `json:"method"` + } `json:"upload"` + } + if err := json.Unmarshal(response.Body, &session); err != nil { + return nil, fmt.Errorf("metorial: invalid upload session response: %w", err) + } + if session.ID == "" || session.Upload.URL == "" { + return nil, errors.New("metorial: the API did not return an upload URL") + } + + if err := u.put(ctx, session.Upload.Method, session.Upload.URL, source); err != nil { + return nil, err + } + + complete := map[string]any{ + "mode": "complete", + "file_upload_id": session.ID, + } + if options.InstanceID != "" { + complete["instance_id"] = options.InstanceID + } + return u.postJSON(complete) +} + +func (u *Uploader) postJSON(body map[string]any) (*fetch.Response, error) { + payload, err := json.Marshal(body) + if err != nil { + return nil, err + } + + headers := map[string]string{} + for key, value := range u.Headers { + headers[key] = value + } + headers["Content-Type"] = "application/json" + + return fetch.Do(u.Runtime, fetch.Request{ + Method: http.MethodPost, + Target: filesEndpoint, + Headers: headers, + Body: payload, + }) +} + +// put sends the file to the presigned storage URL. The URL signature covers the +// content type, so it must match the file_type declared when requesting the URL, +// and Metorial credentials must not be forwarded to the storage backend. +func (u *Uploader) put(ctx context.Context, method, url string, source Source) error { + if method == "" { + method = http.MethodPut + } + + content, err := source.Open() + if err != nil { + return fmt.Errorf("metorial: %w", err) + } + defer content.Close() + + request, err := http.NewRequestWithContext(ctx, method, url, content) + if err != nil { + return fmt.Errorf("metorial: invalid upload URL: %w", err) + } + request.ContentLength = source.Size + request.Header.Set("Content-Type", source.ContentType) + + response, err := u.Client.Do(request) + if err != nil { + return fmt.Errorf("metorial: failed to upload %s: %w", source.Name, err) + } + defer response.Body.Close() + + if response.StatusCode < 200 || response.StatusCode > 299 { + detail, _ := io.ReadAll(io.LimitReader(response.Body, 2048)) + message := strings.TrimSpace(string(detail)) + if message == "" { + return fmt.Errorf("metorial: failed to upload %s: storage responded with %s", source.Name, response.Status) + } + return fmt.Errorf("metorial: failed to upload %s: storage responded with %s: %s", source.Name, response.Status, message) + } + + _, _ = io.Copy(io.Discard, response.Body) + return nil +} diff --git a/internal/fileupload/upload_test.go b/internal/fileupload/upload_test.go new file mode 100644 index 0000000..186d995 --- /dev/null +++ b/internal/fileupload/upload_test.go @@ -0,0 +1,193 @@ +package fileupload + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/metorial/cli/internal/config" +) + +const unavailableMessage = "Signed upload URLs are not available for the configured object storage" + +func TestUploadSendsFilesThroughAnUploadURL(t *testing.T) { + local := filepath.Join(t.TempDir(), "archive.bin") + if err := os.WriteFile(local, []byte("file contents"), 0o644); err != nil { + t.Fatal(err) + } + + steps := []string{} + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPut && r.URL.Path == "/storage": + steps = append(steps, "put") + if r.Header.Get("Authorization") != "" || r.Header.Get("Metorial-Version") != "" { + t.Error("expected no Metorial headers on the storage request") + } + if got := r.Header.Get("Content-Type"); got != "application/octet-stream" { + t.Errorf("expected the declared content type, got %q", got) + } + data, _ := io.ReadAll(r.Body) + if string(data) != "file contents" { + t.Errorf("unexpected uploaded contents %q", data) + } + w.WriteHeader(http.StatusOK) + + case r.Method == http.MethodPost && r.URL.Path == "/files": + if r.Header.Get("Metorial-Version") != "test_version" { + t.Errorf("expected the client headers on the files request, got %v", r.Header) + } + body := map[string]any{} + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Fatal(err) + } + + switch body["mode"] { + case "get_upload_url": + steps = append(steps, "get_upload_url") + expected := map[string]any{ + "purpose": "skill_export", + "file_name": "archive.bin", + "file_size": float64(len("file contents")), + "file_type": "application/octet-stream", + "title": "Archive", + "instance_id": "inst_1", + "store_id": "store_1", + "path": "docs/archive.bin", + "store_replace": true, + } + for key, value := range expected { + if body[key] != value { + t.Errorf("expected %s=%v, got %v", key, value, body[key]) + } + } + writeJSON(w, http.StatusOK, map[string]any{ + "object": "file_upload", + "id": "fup_1", + "upload": map[string]any{"url": server.URL + "/storage", "method": "PUT"}, + }) + case "complete": + steps = append(steps, "complete") + if body["file_upload_id"] != "fup_1" || body["instance_id"] != "inst_1" { + t.Errorf("unexpected complete request %v", body) + } + writeJSON(w, http.StatusOK, map[string]any{"object": "file", "id": "file_2"}) + default: + t.Errorf("unexpected mode %v", body["mode"]) + } + + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + response, err := newTestUploader(t, server.URL).Upload(context.Background(), FileSource(local, "archive.bin", "application/octet-stream", int64(len("file contents"))), Options{ + Purpose: "skill_export", + Title: "Archive", + InstanceID: "inst_1", + StoreID: "store_1", + StorePath: "docs/archive.bin", + StoreReplace: true, + }) + if err != nil { + t.Fatal(err) + } + assertFileID(t, response.Body, "file_2") + + if expected := []string{"get_upload_url", "put", "complete"}; !slices.Equal(steps, expected) { + t.Fatalf("expected steps %v, got %v", expected, steps) + } +} + +func TestUploadReturnsAPIErrorResponses(t *testing.T) { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + writeJSON(w, http.StatusBadRequest, map[string]any{"code": "bad_request", "message": unavailableMessage}) + })) + defer server.Close() + + response, err := newTestUploader(t, server.URL).Upload(context.Background(), BytesSource("notes.txt", []byte("hello")), Options{Purpose: "generic"}) + if err == nil || !strings.Contains(err.Error(), unavailableMessage) { + t.Fatalf("expected the API error, got %v", err) + } + if !strings.Contains(err.Error(), "configure UPLOAD_HOST or presign-capable object storage") { + t.Fatalf("expected a storage configuration hint, got %v", err) + } + if response == nil || response.StatusCode != http.StatusBadRequest { + t.Fatalf("expected the error response to be returned, got %+v", response) + } + if requests != 1 { + t.Fatalf("expected a single request, got %d", requests) + } +} + +func TestUploadReportsStorageFailures(t *testing.T) { + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPut { + http.Error(w, "SignatureDoesNotMatch", http.StatusForbidden) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "id": "fup_1", + "upload": map[string]any{"url": server.URL + "/storage", "method": "PUT"}, + }) + })) + defer server.Close() + + _, err := newTestUploader(t, server.URL).Upload(context.Background(), BytesSource("archive.bin", []byte("large contents")), Options{Purpose: "generic"}) + if err == nil || !bytes.Contains([]byte(err.Error()), []byte("SignatureDoesNotMatch")) { + t.Fatalf("expected the storage error to be reported, got %v", err) + } +} + +func newTestUploader(t *testing.T, host string) *Uploader { + t.Helper() + + hostURL, err := url.Parse(host) + if err != nil { + t.Fatal(err) + } + uploader := New(config.Runtime{APIKey: "metorial_sk_test", APIHost: host, APIHostURL: hostURL}) + uploader.Headers = map[string]string{"Metorial-Version": "test_version"} + return uploader +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} + +func assertFileID(t *testing.T, body []byte, expected string) { + t.Helper() + + file := map[string]any{} + if err := json.Unmarshal(body, &file); err != nil { + t.Fatal(err) + } + if file["id"] != expected { + t.Fatalf("expected file %s, got %v", expected, file["id"]) + } +} + +func TestNewUploaderTimesOutStorageRequests(t *testing.T) { + uploader := New(config.Runtime{}) + + transport, ok := uploader.Client.Transport.(*http.Transport) + if !ok || transport.ResponseHeaderTimeout <= 0 || uploader.Client.Timeout <= 0 { + t.Fatalf("expected the storage client to time out, got client timeout %v and transport %#v", uploader.Client.Timeout, uploader.Client.Transport) + } +} diff --git a/internal/generated/admin/catalog.go b/internal/generated/admin/catalog.go index 183dc51..fd87593 100644 --- a/internal/generated/admin/catalog.go +++ b/internal/generated/admin/catalog.go @@ -584,7 +584,7 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "Custom key-value pairs for storing additional information", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, {Name: "tool-filters", Type: resourcecmd.FlagStringSlice, Target: "body.tool_filters", Usage: "tool_filters", Required: false, Repeated: true}, - {Name: "locked-provider-version-id", Type: resourcecmd.FlagString, Target: "body.locked_provider_version_id", Usage: "Pin this deployment to a specific provider version, or null to follow latest", Required: false, Repeated: false}, + {Name: "locked-provider-version-id", Type: resourcecmd.FlagString, Target: "body.locked_provider_version_id", Usage: "Pin this deployment to a specific provider version, or null to follow latest", Required: false, Repeated: false, Nullable: true}, }, }, }, @@ -1003,7 +1003,7 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "Optional identity to link this setup session to", Required: false, Repeated: false}, {Name: "consumer-id", Type: resourcecmd.FlagString, Target: "body.consumer_id", Usage: "Optional consumer to link this setup session to", Required: false, Repeated: false}, {Name: "redirect-url", Type: resourcecmd.FlagString, Target: "body.redirect_url", Usage: "redirect_url", Required: false, Repeated: false}, - {Name: "type", Type: resourcecmd.FlagString, Target: "body.type", Usage: "The type of setup session, determining the flow and outcome of the session", Required: false, Repeated: false}, + {Name: "type", Type: resourcecmd.FlagString, Target: "body.type", Usage: "The type of setup session, determining the flow and outcome of the session. `admin` creates an admin consent link for auth methods that support it, never creates credentials or configs, and cannot be bound to an identity, consumer, or config.", Required: false, Repeated: false}, {Name: "configuration", Type: resourcecmd.FlagJSON, Target: "body.configuration", Usage: "configuration", Required: false, Repeated: false}, {Name: "configuration-file", Type: resourcecmd.FlagJSONFile, Target: "body.configuration", Usage: "Read configuration JSON from a file", Required: false, Repeated: false}, }, @@ -2794,8 +2794,8 @@ func Catalog() []resourcecmd.ResourceSpec { Flags: []resourcecmd.FlagSpec{ {Name: "provider-id", Type: resourcecmd.FlagString, Target: "body.provider_id", Usage: "provider_id", Required: true, Repeated: false}, {Name: "provider-deployment-id", Type: resourcecmd.FlagString, Target: "body.provider_deployment_id", Usage: "provider_deployment_id", Required: false, Repeated: false}, - {Name: "provider-config-id", Type: resourcecmd.FlagString, Target: "body.provider_config_id", Usage: "provider_config_id", Required: false, Repeated: false}, - {Name: "provider-auth-config-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_config_id", Usage: "provider_auth_config_id", Required: false, Repeated: false}, + {Name: "provider-config-id", Type: resourcecmd.FlagString, Target: "body.provider_config_id", Usage: "provider_config_id", Required: false, Repeated: false, Nullable: true}, + {Name: "provider-auth-config-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_config_id", Usage: "provider_auth_config_id", Required: false, Repeated: false, Nullable: true}, {Name: "tool-filters", Type: resourcecmd.FlagStringSlice, Target: "body.tool_filters", Usage: "tool_filters", Required: false, Repeated: true}, }, }, @@ -2864,8 +2864,8 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "provider-deployment-id", Type: resourcecmd.FlagString, Target: "body.provider_deployment_id", Usage: "provider_deployment_id", Required: false, Repeated: false}, - {Name: "provider-config-id", Type: resourcecmd.FlagString, Target: "body.provider_config_id", Usage: "provider_config_id", Required: false, Repeated: false}, - {Name: "provider-auth-config-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_config_id", Usage: "provider_auth_config_id", Required: false, Repeated: false}, + {Name: "provider-config-id", Type: resourcecmd.FlagString, Target: "body.provider_config_id", Usage: "provider_config_id", Required: false, Repeated: false, Nullable: true}, + {Name: "provider-auth-config-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_config_id", Usage: "provider_auth_config_id", Required: false, Repeated: false, Nullable: true}, {Name: "tool-filters", Type: resourcecmd.FlagStringSlice, Target: "body.tool_filters", Usage: "tool_filters", Required: false, Repeated: true}, }, }, @@ -3014,6 +3014,7 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, {Name: "consumer-profile-id", Type: resourcecmd.FlagString, Target: "body.consumer_profile_id", Usage: "consumer_profile_id", Required: false, Repeated: false}, {Name: "skill-plugin-id", Type: resourcecmd.FlagString, Target: "body.skill_plugin_id", Usage: "skill_plugin_id", Required: false, Repeated: false}, + {Name: "enable-skill-editing", Type: resourcecmd.FlagBool, Target: "body.enable_skill_editing", Usage: "Expose skill editing tools on this endpoint so the connecting consumer can read and edit the skills they have access to. Only connections authenticated with a magic MCP token bound to a single consumer get the tools; connections using a publishable key do not. Requires skills to be enabled for the instance and consumer skill editing to be allowed for the project. The tools disappear if either is turned off later.", Required: false, Repeated: false}, {Name: "magic-mcp-servers", Type: resourcecmd.FlagJSON, Target: "body.magic_mcp_servers", Usage: "magic_mcp_servers", Required: false, Repeated: false}, {Name: "magic-mcp-servers-file", Type: resourcecmd.FlagJSONFile, Target: "body.magic_mcp_servers", Usage: "Read magic_mcp_servers JSON from a file", Required: false, Repeated: false}, }, @@ -3087,6 +3088,7 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, + {Name: "enable-skill-editing", Type: resourcecmd.FlagBool, Target: "body.enable_skill_editing", Usage: "Expose skill editing tools on this endpoint so the connecting consumer can read and edit the skills they have access to. Only connections authenticated with a magic MCP token bound to a single consumer get the tools; connections using a publishable key do not. Requires skills to be enabled for the instance and consumer skill editing to be allowed for the project. The tools disappear if either is turned off later.", Required: false, Repeated: false}, }, }, }, @@ -3263,7 +3265,7 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "compatibility", Type: resourcecmd.FlagString, Target: "body.compatibility", Usage: "compatibility", Required: false, Repeated: false}, {Name: "client-metadata", Type: resourcecmd.FlagJSON, Target: "body.client_metadata", Usage: "client_metadata", Required: false, Repeated: false}, {Name: "client-metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.client_metadata", Usage: "Read client_metadata JSON from a file", Required: false, Repeated: false}, - {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false}, + {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false, Nullable: true}, {Name: "template-id", Type: resourcecmd.FlagString, Target: "body.template_id", Usage: "template_id", Required: false, Repeated: false}, {Name: "skill-group-id", Type: resourcecmd.FlagString, Target: "body.skill_group_id", Usage: "skill_group_id", Required: false, Repeated: false}, }, @@ -3325,16 +3327,16 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, {Name: "client-name", Type: resourcecmd.FlagString, Target: "body.client_name", Usage: "client_name", Required: false, Repeated: false}, {Name: "client-description", Type: resourcecmd.FlagString, Target: "body.client_description", Usage: "client_description", Required: false, Repeated: false}, - {Name: "license", Type: resourcecmd.FlagString, Target: "body.license", Usage: "license", Required: false, Repeated: false}, - {Name: "compatibility", Type: resourcecmd.FlagString, Target: "body.compatibility", Usage: "compatibility", Required: false, Repeated: false}, + {Name: "license", Type: resourcecmd.FlagString, Target: "body.license", Usage: "license", Required: false, Repeated: false, Nullable: true}, + {Name: "compatibility", Type: resourcecmd.FlagString, Target: "body.compatibility", Usage: "compatibility", Required: false, Repeated: false, Nullable: true}, {Name: "client-metadata", Type: resourcecmd.FlagJSON, Target: "body.client_metadata", Usage: "client_metadata", Required: false, Repeated: false}, {Name: "client-metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.client_metadata", Usage: "Read client_metadata JSON from a file", Required: false, Repeated: false}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, - {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false}, + {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false, Nullable: true}, }, }, }, @@ -3467,7 +3469,7 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: true, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, {Name: "content", Type: resourcecmd.FlagString, Target: "body.content", Usage: "content", Required: false, Repeated: false}, }, }, @@ -3527,7 +3529,103 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, + }, + }, + }, + }, + { + Plural: "files", + Singular: "files", + Short: "Skill Files", + Long: "Read and edit the files of a skill by path.", + Operations: []resourcecmd.OperationSpec{ + { + Name: "batch", + Method: "POST", + Path: "/skills/:skillId/files/batch", + Short: "Apply skill file changes", + Long: "Applies up to 100 put, delete and move changes one after another, in order. Every change is validated before any is applied, but changes are not applied atomically: if a change fails while being applied, the changes before it stay applied. Every change can carry its own if_match precondition.", + SDKMapping: "skills.files.batch", + Args: []resourcecmd.ArgumentSpec{ + {Name: "skill-id", Target: "path.skillId", Required: true, Description: "skill id"}, + }, + Flags: []resourcecmd.FlagSpec{ + {Name: "changes", Type: resourcecmd.FlagJSON, Target: "body.changes", Usage: "The changes to apply, in order.", Required: true, Repeated: false}, + {Name: "changes-file", Type: resourcecmd.FlagJSONFile, Target: "body.changes", Usage: "Read changes JSON from a file", Required: false, Repeated: false}, + }, + }, + { + Name: "delete", + Method: "DELETE", + Path: "/skills/:skillId/files", + Short: "Delete skill file", + Long: "Deletes the file at a path. Directories that were created automatically and become empty are removed. The root SKILL.md cannot be deleted.", + SDKMapping: "skills.files.delete", + Args: []resourcecmd.ArgumentSpec{ + {Name: "skill-id", Target: "path.skillId", Required: true, Description: "skill id"}, + }, + Flags: []resourcecmd.FlagSpec{ + {Name: "path", Type: resourcecmd.FlagString, Target: "query.path", Usage: "Path of the file inside the skill. A leading \"/\" is optional. Paths may contain letters, numbers, spaces and the characters . _ - ( ) $.", Required: false, Repeated: false}, + {Name: "if-match", Type: resourcecmd.FlagString, Target: "query.if_match", Usage: "Only apply the change if the file still has this etag. Omit to skip the check.", Required: false, Repeated: false}, + }, + }, + { + Name: "get", + Method: "GET", + Path: "/skills/:skillId/files/content", + Short: "Get skill file", + Long: "Retrieves a single skill file by path. Documents include their text content, other files include a download URL.", + SDKMapping: "skills.files.get", + Args: []resourcecmd.ArgumentSpec{ + {Name: "skill-id", Target: "path.skillId", Required: true, Description: "skill id"}, + }, + Flags: []resourcecmd.FlagSpec{ + {Name: "path", Type: resourcecmd.FlagString, Target: "query.path", Usage: "Path of the file inside the skill. A leading \"/\" is optional. Paths may contain letters, numbers, spaces and the characters . _ - ( ) $.", Required: false, Repeated: false}, + }, + }, + { + Name: "list", + Method: "GET", + Path: "/skills/:skillId/files", + Short: "List skill files", + Long: "Lists every file of a skill with its path, size and etag. Directories are implied by file paths.", + SDKMapping: "skills.files.list", + Args: []resourcecmd.ArgumentSpec{ + {Name: "skill-id", Target: "path.skillId", Required: true, Description: "skill id"}, + }, + }, + { + Name: "move", + Method: "POST", + Path: "/skills/:skillId/files/move", + Short: "Move skill file", + Long: "Moves or renames a file. The root SKILL.md cannot be moved.", + SDKMapping: "skills.files.move", + Args: []resourcecmd.ArgumentSpec{ + {Name: "skill-id", Target: "path.skillId", Required: true, Description: "skill id"}, + }, + Flags: []resourcecmd.FlagSpec{ + {Name: "path", Type: resourcecmd.FlagString, Target: "body.path", Usage: "Path of the file inside the skill. A leading \"/\" is optional. Paths may contain letters, numbers, spaces and the characters . _ - ( ) $.", Required: true, Repeated: false}, + {Name: "to-path", Type: resourcecmd.FlagString, Target: "body.to_path", Usage: "The new path of the file.", Required: true, Repeated: false}, + {Name: "if-match", Type: resourcecmd.FlagString, Target: "body.if_match", Usage: "Only apply the change if the file still has this etag. Omit to skip the check.", Required: false, Repeated: false}, + }, + }, + { + Name: "put", + Method: "PUT", + Path: "/skills/:skillId/files", + Short: "Write skill file", + Long: "Creates or replaces the file at a path. Parent directories are created automatically.", + SDKMapping: "skills.files.put", + Args: []resourcecmd.ArgumentSpec{ + {Name: "skill-id", Target: "path.skillId", Required: true, Description: "skill id"}, + }, + Flags: []resourcecmd.FlagSpec{ + {Name: "path", Type: resourcecmd.FlagString, Target: "body.path", Usage: "Path of the file inside the skill. A leading \"/\" is optional. Paths may contain letters, numbers, spaces and the characters . _ - ( ) $.", Required: true, Repeated: false}, + {Name: "content", Type: resourcecmd.FlagString, Target: "body.content", Usage: "The new text content of the file. Markdown files are stored as documents, other files as plain text files.", Required: false, Repeated: false}, + {Name: "file-id", Type: resourcecmd.FlagString, Target: "body.file_id", Usage: "The ID of a previously uploaded file to place at the path. Use this for binary files. Files that belong to a document cannot be attached; use content to write markdown documents.", Required: false, Repeated: false}, + {Name: "if-match", Type: resourcecmd.FlagString, Target: "body.if_match", Usage: "Only apply the change if the file still has this etag. Pass null to require that no file exists at the path yet. Omit to skip the check.", Required: false, Repeated: false, Nullable: true}, }, }, }, @@ -3647,7 +3745,7 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, {Name: "skill-ids", Type: resourcecmd.FlagStringSlice, Target: "body.skill_ids", Usage: "skill_ids", Required: false, Repeated: true}, @@ -3808,7 +3906,7 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, }, @@ -3938,7 +4036,7 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "source-skill-id", Type: resourcecmd.FlagString, Target: "body.source_skill_id", Usage: "source_skill_id", Required: true, Repeated: false}, {Name: "target-skill-id", Type: resourcecmd.FlagString, Target: "body.target_skill_id", Usage: "target_skill_id", Required: false, Repeated: false}, {Name: "title", Type: resourcecmd.FlagString, Target: "body.title", Usage: "title", Required: true, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, }, }, { @@ -4023,11 +4121,11 @@ func Catalog() []resourcecmd.ResourceSpec { SDKMapping: "skills.plugins.create", Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: true, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, - {Name: "long-description", Type: resourcecmd.FlagString, Target: "body.long_description", Usage: "long_description", Required: false, Repeated: false}, - {Name: "category", Type: resourcecmd.FlagString, Target: "body.category", Usage: "category", Required: false, Repeated: false}, - {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false}, - {Name: "skill-configuration-id", Type: resourcecmd.FlagString, Target: "body.skill_configuration_id", Usage: "skill_configuration_id", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, + {Name: "long-description", Type: resourcecmd.FlagString, Target: "body.long_description", Usage: "long_description", Required: false, Repeated: false, Nullable: true}, + {Name: "category", Type: resourcecmd.FlagString, Target: "body.category", Usage: "category", Required: false, Repeated: false, Nullable: true}, + {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false, Nullable: true}, + {Name: "skill-configuration-id", Type: resourcecmd.FlagString, Target: "body.skill_configuration_id", Usage: "skill_configuration_id", Required: false, Repeated: false, Nullable: true}, {Name: "skill-marketplace-id", Type: resourcecmd.FlagString, Target: "body.skill_marketplace_id", Usage: "skill_marketplace_id", Required: false, Repeated: false}, }, }, @@ -4077,11 +4175,11 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, - {Name: "long-description", Type: resourcecmd.FlagString, Target: "body.long_description", Usage: "long_description", Required: false, Repeated: false}, - {Name: "category", Type: resourcecmd.FlagString, Target: "body.category", Usage: "category", Required: false, Repeated: false}, - {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false}, - {Name: "skill-configuration-id", Type: resourcecmd.FlagString, Target: "body.skill_configuration_id", Usage: "skill_configuration_id", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, + {Name: "long-description", Type: resourcecmd.FlagString, Target: "body.long_description", Usage: "long_description", Required: false, Repeated: false, Nullable: true}, + {Name: "category", Type: resourcecmd.FlagString, Target: "body.category", Usage: "category", Required: false, Repeated: false, Nullable: true}, + {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false, Nullable: true}, + {Name: "skill-configuration-id", Type: resourcecmd.FlagString, Target: "body.skill_configuration_id", Usage: "skill_configuration_id", Required: false, Repeated: false, Nullable: true}, }, }, }, @@ -4112,9 +4210,9 @@ func Catalog() []resourcecmd.ResourceSpec { SDKMapping: "skills.marketplaces.create", Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: true, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, - {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false}, - {Name: "skill-configuration-id", Type: resourcecmd.FlagString, Target: "body.skill_configuration_id", Usage: "skill_configuration_id", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, + {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false, Nullable: true}, + {Name: "skill-configuration-id", Type: resourcecmd.FlagString, Target: "body.skill_configuration_id", Usage: "skill_configuration_id", Required: false, Repeated: false, Nullable: true}, {Name: "repository-access-mode", Type: resourcecmd.FlagString, Target: "body.repository_access_mode", Usage: "repository_access_mode", Required: false, Repeated: false}, {Name: "force-merge-or-push", Type: resourcecmd.FlagBool, Target: "body.force_merge_or_push", Usage: "force_merge_or_push", Required: false, Repeated: false}, {Name: "merge-before-checks-pass", Type: resourcecmd.FlagBool, Target: "body.merge_before_checks_pass", Usage: "merge_before_checks_pass", Required: false, Repeated: false}, @@ -4175,9 +4273,9 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, - {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false}, - {Name: "skill-configuration-id", Type: resourcecmd.FlagString, Target: "body.skill_configuration_id", Usage: "skill_configuration_id", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, + {Name: "image-file-id", Type: resourcecmd.FlagString, Target: "body.image_file_id", Usage: "image_file_id", Required: false, Repeated: false, Nullable: true}, + {Name: "skill-configuration-id", Type: resourcecmd.FlagString, Target: "body.skill_configuration_id", Usage: "skill_configuration_id", Required: false, Repeated: false, Nullable: true}, {Name: "repository-access-mode", Type: resourcecmd.FlagString, Target: "body.repository_access_mode", Usage: "repository_access_mode", Required: false, Repeated: false}, {Name: "force-merge-or-push", Type: resourcecmd.FlagBool, Target: "body.force_merge_or_push", Usage: "force_merge_or_push", Required: false, Repeated: false}, {Name: "merge-before-checks-pass", Type: resourcecmd.FlagBool, Target: "body.merge_before_checks_pass", Usage: "merge_before_checks_pass", Required: false, Repeated: false}, @@ -4615,6 +4713,24 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "machine-id", Target: "path.machineId", Required: true, Description: "machine id"}, }, }, + { + Name: "exec", + Method: "POST", + Path: "/sandbox/machines/:machineId/exec", + Short: "Run a command", + Long: "Runs a shell command in the machine and returns its exit code and output. Non-interactive; starts the machine if needed.", + SDKMapping: "sandbox.machines.exec", + Args: []resourcecmd.ArgumentSpec{ + {Name: "machine-id", Target: "path.machineId", Required: true, Description: "machine id"}, + }, + Flags: []resourcecmd.FlagSpec{ + {Name: "command", Type: resourcecmd.FlagString, Target: "body.command", Usage: "The command to run with /bin/sh -c", Required: true, Repeated: false}, + {Name: "cwd", Type: resourcecmd.FlagString, Target: "body.cwd", Usage: "The working directory to run the command in", Required: false, Repeated: false}, + {Name: "env", Type: resourcecmd.FlagJSON, Target: "body.env", Usage: "Environment variables to set for the command", Required: false, Repeated: false}, + {Name: "env-file", Type: resourcecmd.FlagJSONFile, Target: "body.env", Usage: "Read env JSON from a file", Required: false, Repeated: false}, + {Name: "timeout-seconds", Type: resourcecmd.FlagFloat, Target: "body.timeout_seconds", Usage: "How long the command may run before it is stopped, in seconds. Defaults to 60, at most 300.", Required: false, Repeated: false}, + }, + }, { Name: "get", Method: "GET", @@ -5105,7 +5221,7 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "image-id", Target: "path.imageId", Required: true, Description: "image id"}, }, Flags: []resourcecmd.FlagSpec{ - {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "A name for the image", Required: false, Repeated: false}, + {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "A name for the image", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "User-defined metadata stored in Metorial. Replaces existing metadata when updated.", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, }, @@ -5372,7 +5488,7 @@ func Catalog() []resourcecmd.ResourceSpec { Flags: []resourcecmd.FlagSpec{ {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "User-defined metadata stored in Metorial. Replaces existing metadata when updated.", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, - {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "A name for the disk. Pass null to remove it.", Required: false, Repeated: false}, + {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "A name for the disk. Pass null to remove it.", Required: false, Repeated: false, Nullable: true}, }, }, }, @@ -5611,7 +5727,7 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "Display name of the chat connection", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Description of the connection", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Description of the connection", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "Metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, {Name: "private-metadata", Type: resourcecmd.FlagJSON, Target: "body.private_metadata", Usage: "Private metadata", Required: false, Repeated: false}, @@ -5641,8 +5757,8 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, {Name: "private-metadata", Type: resourcecmd.FlagJSON, Target: "body.private_metadata", Usage: "Private metadata for the instance", Required: false, Repeated: false}, {Name: "private-metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.private_metadata", Usage: "Read private_metadata JSON from a file", Required: false, Repeated: false}, - {Name: "identity-actor-id", Type: resourcecmd.FlagString, Target: "body.identity_actor_id", Usage: "Identity actor to run this instance as", Required: false, Repeated: false}, - {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "Identity to run this instance as", Required: false, Repeated: false}, + {Name: "identity-actor-id", Type: resourcecmd.FlagString, Target: "body.identity_actor_id", Usage: "Identity actor to run this instance as", Required: false, Repeated: false, Nullable: true}, + {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "Identity to run this instance as", Required: false, Repeated: false, Nullable: true}, }, }, { @@ -5700,7 +5816,7 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "Display name of the chat instance", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Description of the instance", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Description of the instance", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "Metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, {Name: "private-metadata", Type: resourcecmd.FlagJSON, Target: "body.private_metadata", Usage: "Private metadata", Required: false, Repeated: false}, @@ -5807,7 +5923,7 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "integration-id", Type: resourcecmd.FlagString, Target: "body.integration_id", Usage: "Integration the integration provider belongs to", Required: true, Repeated: false}, {Name: "integration-provider-id", Type: resourcecmd.FlagString, Target: "body.integration_provider_id", Usage: "Integration provider to enable callbacks for", Required: true, Repeated: false}, {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "Display name for the callback. Defaults to the name of the integration provider.", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Description for the callback. Defaults to the description of the integration provider.", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Description for the callback. Defaults to the description of the integration provider.", Required: false, Repeated: false, Nullable: true}, }, }, { @@ -5867,7 +5983,7 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "Updated display name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Updated description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Updated description", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "Updated custom metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, }, @@ -6102,7 +6218,7 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "Updated display name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Updated description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "Updated description", Required: false, Repeated: false, Nullable: true}, {Name: "webhook", Type: resourcecmd.FlagJSON, Target: "body.webhook", Usage: "webhook", Required: false, Repeated: false}, {Name: "webhook-file", Type: resourcecmd.FlagJSONFile, Target: "body.webhook", Usage: "Read webhook JSON from a file", Required: false, Repeated: false}, {Name: "retry", Type: resourcecmd.FlagJSON, Target: "body.retry", Usage: "Retry policy for deliveries to this destination. Deliveries capture the policy when they are scheduled, so changes only affect new deliveries.", Required: false, Repeated: false}, @@ -6912,8 +7028,8 @@ func Catalog() []resourcecmd.ResourceSpec { Flags: []resourcecmd.FlagSpec{ {Name: "consumer-group-id", Type: resourcecmd.FlagString, Target: "body.consumer_group_id", Usage: "consumer_group_id", Required: true, Repeated: false}, {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, - {Name: "readme", Type: resourcecmd.FlagString, Target: "body.readme", Usage: "readme", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, + {Name: "readme", Type: resourcecmd.FlagString, Target: "body.readme", Usage: "readme", Required: false, Repeated: false, Nullable: true}, }, }, { @@ -6982,8 +7098,8 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, - {Name: "readme", Type: resourcecmd.FlagString, Target: "body.readme", Usage: "readme", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, + {Name: "readme", Type: resourcecmd.FlagString, Target: "body.readme", Usage: "readme", Required: false, Repeated: false, Nullable: true}, }, }, }, @@ -7160,7 +7276,7 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, {Name: "use-integration-name-in-tool-names", Type: resourcecmd.FlagBool, Target: "body.use_integration_name_in_tool_names", Usage: "use_integration_name_in_tool_names", Required: false, Repeated: false}, @@ -7189,9 +7305,9 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "integration-id", Type: resourcecmd.FlagString, Target: "body.integration_id", Usage: "integration_id", Required: true, Repeated: false}, {Name: "provider-id", Type: resourcecmd.FlagString, Target: "body.provider_id", Usage: "provider_id", Required: true, Repeated: false}, {Name: "provider-deployment-id", Type: resourcecmd.FlagString, Target: "body.provider_deployment_id", Usage: "provider_deployment_id", Required: true, Repeated: false}, - {Name: "provider-auth-method-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_method_id", Usage: "provider_auth_method_id", Required: false, Repeated: false}, - {Name: "provider-auth-credentials-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_credentials_id", Usage: "provider_auth_credentials_id", Required: false, Repeated: false}, - {Name: "provider-config-id", Type: resourcecmd.FlagString, Target: "body.provider_config_id", Usage: "provider_config_id", Required: false, Repeated: false}, + {Name: "provider-auth-method-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_method_id", Usage: "provider_auth_method_id", Required: false, Repeated: false, Nullable: true}, + {Name: "provider-auth-credentials-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_credentials_id", Usage: "provider_auth_credentials_id", Required: false, Repeated: false, Nullable: true}, + {Name: "provider-config-id", Type: resourcecmd.FlagString, Target: "body.provider_config_id", Usage: "provider_config_id", Required: false, Repeated: false, Nullable: true}, {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, @@ -7261,11 +7377,11 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "provider-deployment-id", Type: resourcecmd.FlagString, Target: "body.provider_deployment_id", Usage: "provider_deployment_id", Required: false, Repeated: false}, - {Name: "provider-auth-method-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_method_id", Usage: "provider_auth_method_id", Required: false, Repeated: false}, - {Name: "provider-auth-credentials-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_credentials_id", Usage: "provider_auth_credentials_id", Required: false, Repeated: false}, - {Name: "provider-config-id", Type: resourcecmd.FlagString, Target: "body.provider_config_id", Usage: "provider_config_id", Required: false, Repeated: false}, + {Name: "provider-auth-method-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_method_id", Usage: "provider_auth_method_id", Required: false, Repeated: false, Nullable: true}, + {Name: "provider-auth-credentials-id", Type: resourcecmd.FlagString, Target: "body.provider_auth_credentials_id", Usage: "provider_auth_credentials_id", Required: false, Repeated: false, Nullable: true}, + {Name: "provider-config-id", Type: resourcecmd.FlagString, Target: "body.provider_config_id", Usage: "provider_config_id", Required: false, Repeated: false, Nullable: true}, {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, {Name: "tool-filters", Type: resourcecmd.FlagStringSlice, Target: "body.tool_filters", Usage: "tool_filters", Required: false, Repeated: true}, @@ -7294,8 +7410,8 @@ func Catalog() []resourcecmd.ResourceSpec { {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, - {Name: "identity-actor-id", Type: resourcecmd.FlagString, Target: "body.identity_actor_id", Usage: "identity_actor_id", Required: false, Repeated: false}, - {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "identity_id", Required: false, Repeated: false}, + {Name: "identity-actor-id", Type: resourcecmd.FlagString, Target: "body.identity_actor_id", Usage: "identity_actor_id", Required: false, Repeated: false, Nullable: true}, + {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "identity_id", Required: false, Repeated: false, Nullable: true}, {Name: "providers", Type: resourcecmd.FlagJSON, Target: "body.providers", Usage: "providers", Required: false, Repeated: false}, {Name: "providers-file", Type: resourcecmd.FlagJSONFile, Target: "body.providers", Usage: "Read providers JSON from a file", Required: false, Repeated: false}, }, @@ -7398,11 +7514,11 @@ func Catalog() []resourcecmd.ResourceSpec { }, Flags: []resourcecmd.FlagSpec{ {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: false, Repeated: false}, - {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, + {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false, Nullable: true}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, - {Name: "identity-actor-id", Type: resourcecmd.FlagString, Target: "body.identity_actor_id", Usage: "identity_actor_id", Required: false, Repeated: false}, - {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "identity_id", Required: false, Repeated: false}, + {Name: "identity-actor-id", Type: resourcecmd.FlagString, Target: "body.identity_actor_id", Usage: "identity_actor_id", Required: false, Repeated: false, Nullable: true}, + {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "identity_id", Required: false, Repeated: false, Nullable: true}, {Name: "providers", Type: resourcecmd.FlagJSON, Target: "body.providers", Usage: "providers", Required: false, Repeated: false}, {Name: "providers-file", Type: resourcecmd.FlagJSONFile, Target: "body.providers", Usage: "Read providers JSON from a file", Required: false, Repeated: false}, }, @@ -7420,16 +7536,17 @@ func Catalog() []resourcecmd.ResourceSpec { Method: "POST", Path: "/integration-setup-sessions", Short: "Create integration setup session", - Long: "Creates a new integration setup session and draft integration instance.", + Long: "Creates a new integration setup session. Creates a draft integration instance unless integration_instance_id is given, in which case that instance is reused and its config and auth config are overwritten on completion.", SDKMapping: "integrations.setupSessions.create", Flags: []resourcecmd.FlagSpec{ {Name: "integration-id", Type: resourcecmd.FlagString, Target: "body.integration_id", Usage: "integration_id", Required: true, Repeated: false}, + {Name: "integration-instance-id", Type: resourcecmd.FlagString, Target: "body.integration_instance_id", Usage: "integration_instance_id", Required: false, Repeated: false}, {Name: "name", Type: resourcecmd.FlagString, Target: "body.name", Usage: "name", Required: true, Repeated: false}, {Name: "description", Type: resourcecmd.FlagString, Target: "body.description", Usage: "description", Required: false, Repeated: false}, {Name: "metadata", Type: resourcecmd.FlagJSON, Target: "body.metadata", Usage: "metadata", Required: false, Repeated: false}, {Name: "metadata-file", Type: resourcecmd.FlagJSONFile, Target: "body.metadata", Usage: "Read metadata JSON from a file", Required: false, Repeated: false}, - {Name: "identity-actor-id", Type: resourcecmd.FlagString, Target: "body.identity_actor_id", Usage: "identity_actor_id", Required: false, Repeated: false}, - {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "identity_id", Required: false, Repeated: false}, + {Name: "identity-actor-id", Type: resourcecmd.FlagString, Target: "body.identity_actor_id", Usage: "identity_actor_id", Required: false, Repeated: false, Nullable: true}, + {Name: "identity-id", Type: resourcecmd.FlagString, Target: "body.identity_id", Usage: "identity_id", Required: false, Repeated: false, Nullable: true}, {Name: "expires-at", Type: resourcecmd.FlagString, Target: "body.expires_at", Usage: "expires_at", Required: false, Repeated: false}, {Name: "redirect-url", Type: resourcecmd.FlagString, Target: "body.redirect_url", Usage: "redirect_url", Required: false, Repeated: false}, {Name: "configuration", Type: resourcecmd.FlagJSON, Target: "body.configuration", Usage: "configuration", Required: false, Repeated: false}, diff --git a/internal/resourcecmd/types.go b/internal/resourcecmd/types.go index 6059a23..cb2fb46 100644 --- a/internal/resourcecmd/types.go +++ b/internal/resourcecmd/types.go @@ -44,6 +44,8 @@ type FlagSpec struct { Required bool Repeated bool Shorthand string + // A nullable string flag sends JSON null when given the literal "null". + Nullable bool } type OperationSpec struct { diff --git a/internal/skillsapi/client.go b/internal/skillsapi/client.go new file mode 100644 index 0000000..734170a --- /dev/null +++ b/internal/skillsapi/client.go @@ -0,0 +1,244 @@ +package skillsapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/metorial/cli/internal/apiclient" + "github.com/metorial/cli/internal/config" + "github.com/metorial/cli/internal/fileupload" + metorial "github.com/metorial/metorial-go/v1" +) + +const ( + maxUploadBytes = 100 * 1024 * 1024 + maxDownloadBytes = 200 * 1024 * 1024 + uploadPurpose = "generic" + + // Pinned: the consumer API version has no skill endpoints. + APIVersion = "mt_2026_01_01_magnetar" +) + +var downloadClient = fileupload.NewTransferClient() + +type Client struct { + api apiclient.Client +} + +func New(runtime config.Runtime) (*Client, error) { + if err := runtime.RequireAPIKey(); err != nil { + return nil, err + } + + return &Client{ + api: apiclient.Client{ + Runtime: runtime, + Headers: map[string]string{"Metorial-Version": APIVersion}, + Describe: describeError, + }, + }, nil +} + +func (c *Client) APIHost() string { + return c.api.Runtime.APIHost +} + +func (c *Client) ListSkills(options ListOptions, search string) (*SkillList, error) { + extra := url.Values{} + if search = strings.TrimSpace(search); search != "" { + extra.Set("search", search) + } + + result := &SkillList{} + return result, c.api.Do(http.MethodGet, "/skills"+apiclient.ListQuery(options, extra), nil, result) +} + +func (c *Client) GetSkill(skillID string) (*Skill, error) { + result := &Skill{} + return result, c.api.Do(http.MethodGet, "/skills/"+apiclient.PathSegment(skillID), nil, result) +} + +// Older backends resolve only IDs and can't search slugs, hence the paging fallback. +func (c *Client) ResolveSkill(reference string) (*Skill, error) { + reference = strings.TrimSpace(reference) + if reference == "" { + return nil, errors.New("metorial: a skill ID or slug is required") + } + + skill, err := c.GetSkill(reference) + if err == nil { + return skill, nil + } + if !apiclient.IsStatus(err, http.StatusNotFound) { + return nil, err + } + + options := ListOptions{Limit: 100} + for { + page, err := c.ListSkills(options, "") + if err != nil { + return nil, err + } + for _, candidate := range page.Items { + if candidate.Slug != nil && *candidate.Slug == reference { + return &candidate, nil + } + } + + if !page.Pagination.HasMoreAfter || len(page.Items) == 0 { + break + } + after := page.Pagination.After + if after == "" { + after = page.Items[len(page.Items)-1].ID + } + if after == options.After { + break + } + options.After = after + } + + return nil, fmt.Errorf("metorial: no skill found with ID or slug %q", reference) +} + +func (c *Client) CreateSkill(input CreateSkillInput) (*Skill, error) { + result := &Skill{} + return result, c.api.Do(http.MethodPost, "/skills", input, result) +} + +func (c *Client) ListFiles(skillID string) (*FileList, error) { + result := &FileList{} + return result, c.api.Do(http.MethodGet, filesTarget(skillID, ""), nil, result) +} + +func (c *Client) GetFile(skillID string, filePath string) (*FileContent, error) { + query := url.Values{} + query.Set("path", filePath) + + result := &FileContent{} + return result, c.api.Do(http.MethodGet, filesTarget(skillID, "/content")+"?"+query.Encode(), nil, result) +} + +func (c *Client) PutFile(skillID string, change ChangeInput) (*FileChange, error) { + result := &FileChange{} + return result, c.api.Do(http.MethodPut, filesTarget(skillID, ""), change, result) +} + +func (c *Client) DeleteFile(skillID string, filePath string, ifMatch string) (*FileChange, error) { + query := url.Values{} + query.Set("path", filePath) + if ifMatch != "" { + query.Set("if_match", ifMatch) + } + + result := &FileChange{} + return result, c.api.Do(http.MethodDelete, filesTarget(skillID, "")+"?"+query.Encode(), nil, result) +} + +func (c *Client) MoveFile(skillID string, change ChangeInput) (*FileChange, error) { + result := &FileChange{} + return result, c.api.Do(http.MethodPost, filesTarget(skillID, "/move"), change, result) +} + +func (c *Client) ApplyChanges(skillID string, changes []ChangeInput) (*FileChangeList, error) { + body := map[string]any{"changes": changes} + + result := &FileChangeList{} + return result, c.api.Do(http.MethodPost, filesTarget(skillID, "/batch"), body, result) +} + +func (c *Client) ReadFile(skillID string, filePath string) (*FileContent, []byte, error) { + file, err := c.GetFile(skillID, filePath) + if err != nil { + return nil, nil, err + } + + if file.Content != nil { + return file, []byte(*file.Content), nil + } + if file.DownloadURL == nil || strings.TrimSpace(*file.DownloadURL) == "" { + return nil, nil, fmt.Errorf("metorial: the content of %s cannot be downloaded from this deployment", file.Path) + } + + content, err := download(*file.DownloadURL) + if err != nil { + return nil, nil, fmt.Errorf("metorial: failed to download %s: %w", file.Path, err) + } + + return file, content, nil +} + +func (c *Client) UploadFile(name string, content []byte) (*UploadedFile, error) { + if len(content) > maxUploadBytes { + return nil, fmt.Errorf("metorial: %s is larger than %d MB", name, maxUploadBytes/1024/1024) + } + + uploader := fileupload.New(c.api.Runtime) + uploader.Headers = c.api.Headers + + response, err := uploader.Upload(context.Background(), fileupload.BytesSource(name, content), fileupload.Options{ + Purpose: uploadPurpose, + InstanceID: c.api.Runtime.InstanceID, + }) + if err != nil { + return nil, describeError("/files", err) + } + + result := &UploadedFile{} + if err := json.Unmarshal(response.Body, result); err != nil { + return nil, fmt.Errorf("metorial: failed to decode response from /files: %w", err) + } + return result, nil +} + +func download(target string) ([]byte, error) { + response, err := downloadClient.Get(target) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode < 200 || response.StatusCode >= 300 { + return nil, fmt.Errorf("download returned %s", response.Status) + } + + content, err := io.ReadAll(io.LimitReader(response.Body, maxDownloadBytes+1)) + if err != nil { + return nil, err + } + if len(content) > maxDownloadBytes { + return nil, fmt.Errorf("file is larger than %d MB", maxDownloadBytes/1024/1024) + } + + return content, nil +} + +func describeError(_ string, err error) error { + apiErr := &metorial.Error{} + if !errors.As(err, &apiErr) { + return err + } + + switch apiErr.StatusCode { + case http.StatusUnauthorized: + return fmt.Errorf("metorial: your session is no longer valid: %s\nSign in again with the login command", apiErr.Message) + case http.StatusForbidden: + return fmt.Errorf("metorial: you do not have access to this skill: %s", apiErr.Message) + } + + return err +} + +func IsConflict(err error) bool { + return apiclient.IsStatus(err, http.StatusPreconditionFailed) +} + +func filesTarget(skillID string, suffix string) string { + return "/skills/" + apiclient.PathSegment(skillID) + "/files" + suffix +} diff --git a/internal/skillsapi/client_test.go b/internal/skillsapi/client_test.go new file mode 100644 index 0000000..ce4f1e8 --- /dev/null +++ b/internal/skillsapi/client_test.go @@ -0,0 +1,55 @@ +package skillsapi_test + +import ( + "testing" + + "github.com/metorial/cli/internal/skillsapi/skillstest" +) + +func TestResolveSkillResolvesSlugDirectly(t *testing.T) { + fake := skillstest.NewServer(t) + fake.AddSkills(250) + + skill, err := fake.Client(t).ResolveSkill(skillstest.SkillSlug) + if err != nil { + t.Fatalf("ResolveSkill() error = %v", err) + } + if skill.ID != skillstest.SkillID { + t.Fatalf("resolved %s, want %s", skill.ID, skillstest.SkillID) + } + if fake.SkillLists != 0 { + t.Fatalf("listed skills %d times, want 0", fake.SkillLists) + } +} + +func TestResolveSkillFallsBackToPagingOnLegacyBackends(t *testing.T) { + fake := skillstest.NewServer(t) + fake.AddSkills(250) + fake.LegacySkillLookup = true + + skill, err := fake.Client(t).ResolveSkill(skillstest.SkillSlug) + if err != nil { + t.Fatalf("ResolveSkill() error = %v", err) + } + if skill.ID != skillstest.SkillID { + t.Fatalf("resolved %s, want %s", skill.ID, skillstest.SkillID) + } + + if _, err := fake.Client(t).ResolveSkill("missing-skill"); err == nil { + t.Fatal("ResolveSkill() should fail for an unknown slug") + } +} + +func TestResolveSkillStopsWhenTheCursorDoesNotAdvance(t *testing.T) { + fake := skillstest.NewServer(t) + fake.AddSkills(250) + fake.LegacySkillLookup = true + fake.IgnoreAfterCursor = true + + if _, err := fake.Client(t).ResolveSkill("missing-skill"); err == nil { + t.Fatal("ResolveSkill() should fail for an unknown slug") + } + if fake.SkillLists != 2 { + t.Fatalf("listed skills %d times, want 2", fake.SkillLists) + } +} diff --git a/internal/skillsapi/content.go b/internal/skillsapi/content.go new file mode 100644 index 0000000..69d764c --- /dev/null +++ b/internal/skillsapi/content.go @@ -0,0 +1,67 @@ +package skillsapi + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + "unicode/utf8" +) + +const IfMatchAbsent = "absent" + +const MaxTextBytes = 5 * 1024 * 1024 + +var documentExtensions = map[string]bool{".md": true, ".markdown": true} + +func IsText(filePath string, content []byte) (bool, error) { + if !documentExtensions[strings.ToLower(path.Ext(filePath))] { + return len(content) <= MaxTextBytes && utf8.Valid(content) && !bytes.Contains(content, []byte{0}), nil + } + + if !utf8.Valid(content) { + return false, fmt.Errorf("metorial: %s is not valid UTF-8; Markdown files become documents and must be UTF-8 text", filePath) + } + if len(content) > MaxTextBytes { + return false, fmt.Errorf("metorial: %s is larger than %d MB, the limit for Markdown documents", filePath, MaxTextBytes/1024/1024) + } + return true, nil +} + +// The document store strips trailing newlines. +func RestoreTrailingNewline(kind string, content []byte) []byte { + if kind == FileKindDocument && len(content) > 0 && content[len(content)-1] != '\n' { + return append(content, '\n') + } + return content +} + +func MatchEtag(etag string) json.RawMessage { + encoded, _ := json.Marshal(etag) + return encoded +} + +func MatchAbsent() json.RawMessage { + return json.RawMessage("null") +} + +func ParseIfMatch(value string) json.RawMessage { + value = strings.TrimSpace(value) + switch value { + case "": + return nil + case IfMatchAbsent: + return MatchAbsent() + } + return MatchEtag(value) +} + +func ParseEtag(value string) (string, error) { + value = strings.TrimSpace(value) + if value == IfMatchAbsent { + return "", errors.New(`metorial: if-match "absent" only applies to writes; pass the etag of the file instead`) + } + return value, nil +} diff --git a/internal/skillsapi/content_test.go b/internal/skillsapi/content_test.go new file mode 100644 index 0000000..d8af552 --- /dev/null +++ b/internal/skillsapi/content_test.go @@ -0,0 +1,25 @@ +package skillsapi_test + +import ( + "bytes" + "strings" + "testing" + + "github.com/metorial/cli/internal/skillsapi" +) + +func TestIsTextValidatesMarkdownDocuments(t *testing.T) { + if _, err := skillsapi.IsText("/notes.md", []byte("caf\xe9")); err == nil || !strings.Contains(err.Error(), "UTF-8") { + t.Fatalf("invalid UTF-8 error = %v", err) + } + if _, err := skillsapi.IsText("/big.md", bytes.Repeat([]byte("a"), skillsapi.MaxTextBytes+1)); err == nil { + t.Fatal("IsText() should reject documents larger than MaxTextBytes") + } + + if isText, err := skillsapi.IsText("/notes.md", []byte("# Notes")); err != nil || !isText { + t.Fatalf("IsText(notes.md) = %v, %v", isText, err) + } + if isText, err := skillsapi.IsText("/logo.png", []byte("\x89PNG\x00")); err != nil || isText { + t.Fatalf("IsText(logo.png) = %v, %v", isText, err) + } +} diff --git a/internal/skillsapi/skillstest/fake.go b/internal/skillsapi/skillstest/fake.go new file mode 100644 index 0000000..e3992af --- /dev/null +++ b/internal/skillsapi/skillstest/fake.go @@ -0,0 +1,456 @@ +package skillstest + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "path" + "sort" + "strconv" + "strings" + "sync" + "testing" + + "github.com/metorial/cli/internal/config" + "github.com/metorial/cli/internal/skillsapi" +) + +const SkillID = "skill_test" +const SkillSlug = "release-notes" + +type file struct { + kind string + content []byte + etag string +} + +type pendingUpload struct { + contentType string + content []byte + uploaded bool +} + +type Server struct { + URL string + + mu sync.Mutex + files map[string]*file + uploads map[string][]byte + pending map[string]*pendingUpload + skills []map[string]any + nextID int + Batches int + Uploads int + Downloads int + SkillLists int + + // Uploads a real signed URL would reject. + StorageErrors []string + + // GET /skills/:id ignores slugs, like older backends. + LegacySkillLookup bool + + FailPath string + + FailReadPath string + + FailUploads bool + + IgnoreAfterCursor bool +} + +func NewServer(t *testing.T) *Server { + t.Helper() + + fake := &Server{files: map[string]*file{}, uploads: map[string][]byte{}, pending: map[string]*pendingUpload{}} + fake.skills = []map[string]any{fake.skill()} + server := httptest.NewServer(http.HandlerFunc(fake.handle)) + t.Cleanup(server.Close) + + fake.URL = server.URL + return fake +} + +func (s *Server) Client(t *testing.T) *skillsapi.Client { + t.Helper() + + hostURL, err := url.Parse(s.URL) + if err != nil { + t.Fatalf("url.Parse() error = %v", err) + } + + client, err := skillsapi.New(config.Runtime{APIKey: "test-token", APIHost: s.URL, APIHostURL: hostURL}) + if err != nil { + t.Fatalf("skillsapi.New() error = %v", err) + } + + return client +} + +// Prepends skills so finding the test skill takes several pages. +func (s *Server) AddSkills(count int) { + s.mu.Lock() + defer s.mu.Unlock() + + others := []map[string]any{} + for index := range count { + id := fmt.Sprintf("skill_other_%d", index) + others = append(others, map[string]any{"object": "skill", "id": id, "slug": "other-" + id, "name": "Other", "status": "active"}) + } + s.skills = append(others, s.skills...) +} + +func (s *Server) SetFile(filePath string, content string) { + s.mu.Lock() + defer s.mu.Unlock() + + s.setFileLocked(filePath, []byte(content)) +} + +func (s *Server) DeleteFile(filePath string) { + s.mu.Lock() + defer s.mu.Unlock() + + delete(s.files, filePath) +} + +func (s *Server) Content(filePath string) (string, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + stored, ok := s.files[filePath] + if !ok { + return "", false + } + return string(stored.content), true +} + +func (s *Server) setFileLocked(filePath string, content []byte) *file { + kind := skillsapi.FileKindFile + etag := "" + if ext := strings.ToLower(path.Ext(filePath)); ext == ".md" || ext == ".markdown" { + kind = skillsapi.FileKindDocument + // The real document store strips trailing newlines. + content = []byte(strings.TrimRight(string(content), "\n")) + sum := sha256.Sum256(content) + etag = hex.EncodeToString(sum[:]) + } else { + s.nextID++ + etag = fmt.Sprintf("fil_%d", s.nextID) + } + + stored := &file{kind: kind, content: content, etag: etag} + s.files[filePath] = stored + return stored +} + +func (s *Server) presentLocked(filePath string) map[string]any { + stored := s.files[filePath] + return map[string]any{ + "object": "skill.file", + "path": filePath, + "kind": stored.kind, + "size": len(stored.content), + "etag": stored.etag, + "skill_id": SkillID, + "updated_at": "2026-01-01T00:00:00Z", + } +} + +func (s *Server) handle(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + defer s.mu.Unlock() + + // The real API exposes these endpoints only on the magnetar version. + if (strings.HasPrefix(r.URL.Path, "/skills") || r.URL.Path == "/files") && r.Header.Get("Metorial-Version") != skillsapi.APIVersion { + writeJSON(w, http.StatusNotFound, map[string]any{"code": "not_found", "message": "The requested endpoint could not be found."}) + return + } + + switch { + case r.Method == http.MethodGet && r.URL.Path == "/skills": + s.SkillLists++ + s.listSkillsLocked(w, r.URL.Query()) + + case r.Method == http.MethodGet && r.URL.Path == "/skills/"+SkillID: + writeJSON(w, http.StatusOK, s.skill()) + + case r.Method == http.MethodGet && r.URL.Path == "/skills/"+SkillSlug && !s.LegacySkillLookup: + writeJSON(w, http.StatusOK, s.skill()) + + case r.Method == http.MethodGet && r.URL.Path == "/skills/"+SkillID+"/files": + paths := make([]string, 0, len(s.files)) + for filePath := range s.files { + paths = append(paths, filePath) + } + sort.Strings(paths) + + items := []any{} + for _, filePath := range paths { + items = append(items, s.presentLocked(filePath)) + } + writeJSON(w, http.StatusOK, map[string]any{"items": items}) + + case r.Method == http.MethodGet && r.URL.Path == "/skills/"+SkillID+"/files/content": + filePath := normalize(r.URL.Query().Get("path")) + if filePath == s.FailReadPath { + writeJSON(w, http.StatusInternalServerError, map[string]any{"code": "internal_server_error", "message": filePath + " could not be read"}) + return + } + stored, ok := s.files[filePath] + if !ok { + writeJSON(w, http.StatusNotFound, map[string]any{"code": "not_found", "message": "File not found"}) + return + } + + presented := s.presentLocked(filePath) + if stored.kind == skillsapi.FileKindDocument { + presented["content"] = string(stored.content) + } else { + presented["download_url"] = s.URL + "/download" + filePath + } + writeJSON(w, http.StatusOK, presented) + + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/download/"): + stored, ok := s.files[strings.TrimPrefix(r.URL.Path, "/download")] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + s.Downloads++ + _, _ = w.Write(stored.content) + + case r.Method == http.MethodPut && r.URL.Path == "/skills/"+SkillID+"/files": + s.applySingleLocked(w, r, skillsapi.ChangeTypePut) + + case r.Method == http.MethodPost && r.URL.Path == "/skills/"+SkillID+"/files/move": + s.applySingleLocked(w, r, skillsapi.ChangeTypeMove) + + case r.Method == http.MethodDelete && r.URL.Path == "/skills/"+SkillID+"/files": + change := map[string]any{"type": skillsapi.ChangeTypeDelete, "path": r.URL.Query().Get("path")} + if r.URL.Query().Has("if_match") { + change["if_match"] = r.URL.Query().Get("if_match") + } + s.writeChangesLocked(w, []map[string]any{change}, true) + + case r.Method == http.MethodPost && r.URL.Path == "/skills/"+SkillID+"/files/batch": + body := struct { + Changes []map[string]any `json:"changes"` + }{} + _ = json.NewDecoder(r.Body).Decode(&body) + s.Batches++ + s.writeChangesLocked(w, body.Changes, false) + + case r.Method == http.MethodPost && r.URL.Path == "/files": + s.handleSignedUploadLocked(w, r) + + case r.Method == http.MethodPut && strings.HasPrefix(r.URL.Path, "/storage/"): + s.storeUploadLocked(w, r) + + default: + writeJSON(w, http.StatusNotFound, map[string]any{"code": "not_found", "message": r.Method + " " + r.URL.Path}) + } +} + +func (s *Server) handleSignedUploadLocked(w http.ResponseWriter, r *http.Request) { + body := map[string]any{} + _ = json.NewDecoder(r.Body).Decode(&body) + + if s.FailUploads { + writeJSON(w, http.StatusBadRequest, map[string]any{"code": "bad_request", "message": "Signed upload URLs are not available for the configured object storage"}) + return + } + + switch body["mode"] { + case "get_upload_url": + s.nextID++ + id := fmt.Sprintf("fup_%d", s.nextID) + contentType, _ := body["file_type"].(string) + s.pending[id] = &pendingUpload{contentType: contentType} + writeJSON(w, http.StatusOK, map[string]any{ + "object": "file_upload", + "id": id, + "upload": map[string]any{"url": s.URL + "/storage/" + id, "method": http.MethodPut}, + }) + + case "complete": + id, _ := body["file_upload_id"].(string) + pending, ok := s.pending[id] + if !ok || !pending.uploaded { + writeJSON(w, http.StatusBadRequest, map[string]any{"code": "bad_request", "message": "No content has been uploaded for file upload " + id}) + return + } + delete(s.pending, id) + s.nextID++ + s.Uploads++ + fileID := fmt.Sprintf("upl_%d", s.nextID) + s.uploads[fileID] = pending.content + writeJSON(w, http.StatusOK, map[string]any{"object": "file", "id": fileID}) + + default: + writeJSON(w, http.StatusBadRequest, map[string]any{"code": "bad_request", "message": "mode must be one of get_upload_url, complete"}) + } +} + +func (s *Server) storeUploadLocked(w http.ResponseWriter, r *http.Request) { + id := strings.TrimPrefix(r.URL.Path, "/storage/") + pending, ok := s.pending[id] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + if r.Header.Get("Authorization") != "" { + s.StorageErrors = append(s.StorageErrors, id+": sent Metorial credentials") + } + if got := r.Header.Get("Content-Type"); got != pending.contentType { + s.StorageErrors = append(s.StorageErrors, fmt.Sprintf("%s: content type %q does not match %q", id, got, pending.contentType)) + } + + pending.content, _ = io.ReadAll(r.Body) + pending.uploaded = true + w.WriteHeader(http.StatusOK) +} + +func (s *Server) applySingleLocked(w http.ResponseWriter, r *http.Request, changeType string) { + change := map[string]any{} + _ = json.NewDecoder(r.Body).Decode(&change) + change["type"] = changeType + s.writeChangesLocked(w, []map[string]any{change}, true) +} + +func (s *Server) writeChangesLocked(w http.ResponseWriter, changes []map[string]any, single bool) { + results, status, code, message := s.applyLocked(changes) + switch { + case status != http.StatusOK: + writeJSON(w, status, map[string]any{"code": code, "message": message}) + case single: + writeJSON(w, http.StatusOK, results[0]) + default: + writeJSON(w, http.StatusOK, map[string]any{"items": results}) + } +} + +// Like the real API: stops at the first failing change and keeps earlier ones. +func (s *Server) applyLocked(changes []map[string]any) ([]any, int, string, string) { + results := []any{} + for _, change := range changes { + filePath := normalize(stringField(change, "path")) + changeType := stringField(change, "type") + + if status, code, message := s.checkLocked(change, filePath, changeType); status != http.StatusOK { + return nil, status, code, message + } + + switch changeType { + case skillsapi.ChangeTypeDelete: + delete(s.files, filePath) + results = append(results, map[string]any{"object": "skill.file_change", "type": changeType, "path": filePath, "previous_path": nil, "file": nil}) + + case skillsapi.ChangeTypeMove: + toPath := normalize(stringField(change, "to_path")) + s.files[toPath] = s.files[filePath] + delete(s.files, filePath) + results = append(results, map[string]any{"object": "skill.file_change", "type": changeType, "path": toPath, "previous_path": filePath, "file": s.presentLocked(toPath)}) + + default: + content := []byte(stringField(change, "content")) + if fileID := stringField(change, "file_id"); fileID != "" { + content = s.uploads[fileID] + } + s.setFileLocked(filePath, content) + results = append(results, map[string]any{"object": "skill.file_change", "type": changeType, "path": filePath, "previous_path": nil, "file": s.presentLocked(filePath)}) + } + } + + return results, http.StatusOK, "", "" +} + +func (s *Server) checkLocked(change map[string]any, filePath string, changeType string) (int, string, string) { + if filePath == s.FailPath { + return http.StatusBadRequest, "error", filePath + " was rejected" + } + if filePath == "/SKILL.md" && changeType != skillsapi.ChangeTypePut { + return http.StatusBadRequest, "error", "The root /SKILL.md cannot be deleted or moved" + } + + ifMatch, hasIfMatch := change["if_match"] + if !hasIfMatch { + return http.StatusOK, "", "" + } + + stored, exists := s.files[filePath] + if ifMatch == nil && exists { + return http.StatusPreconditionFailed, "skill_file_exists", filePath + " already exists" + } + if _, ok := ifMatch.(string); ok && !exists { + return http.StatusPreconditionFailed, "skill_file_missing", filePath + " no longer exists" + } + if etag, ok := ifMatch.(string); ok && stored.etag != etag { + return http.StatusPreconditionFailed, "skill_file_changed", filePath + " changed" + } + return http.StatusOK, "", "" +} + +// Like the real search index, search matches names, never slugs. +func (s *Server) listSkillsLocked(w http.ResponseWriter, query url.Values) { + search := strings.ToLower(query.Get("search")) + limit, _ := strconv.Atoi(query.Get("limit")) + if limit <= 0 { + limit = 10 + } + + matches := []map[string]any{} + for _, skill := range s.skills { + if search == "" || strings.Contains(strings.ToLower(skill["name"].(string)), search) { + matches = append(matches, skill) + } + } + + if after := query.Get("after"); after != "" && !s.IgnoreAfterCursor { + for index, skill := range matches { + if skill["id"] == after { + matches = matches[index+1:] + break + } + } + } + + hasMore := len(matches) > limit + if hasMore { + matches = matches[:limit] + } + writeJSON(w, http.StatusOK, map[string]any{"items": matches, "pagination": map[string]any{"has_more_after": hasMore, "has_more_before": false}}) +} + +func (s *Server) skill() map[string]any { + return map[string]any{ + "object": "skill", + "id": SkillID, + "slug": SkillSlug, + "name": "Release Notes", + "status": "active", + } +} + +func normalize(filePath string) string { + if strings.HasPrefix(filePath, "/") { + return filePath + } + return "/" + filePath +} + +func stringField(values map[string]any, key string) string { + value, _ := values[key].(string) + return value +} + +func writeJSON(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} diff --git a/internal/skillsapi/types.go b/internal/skillsapi/types.go new file mode 100644 index 0000000..14dff94 --- /dev/null +++ b/internal/skillsapi/types.go @@ -0,0 +1,92 @@ +package skillsapi + +import ( + "encoding/json" + "time" + + "github.com/metorial/cli/internal/apiclient" +) + +const ( + FileKindDocument = "document" + FileKindFile = "file" + + ChangeTypePut = "put" + ChangeTypeDelete = "delete" + ChangeTypeMove = "move" +) + +type ListOptions = apiclient.ListOptions + +type Skill struct { + Object string `json:"object"` + ID string `json:"id"` + Status string `json:"status"` + Slug *string `json:"slug"` + Name string `json:"name"` + Description *string `json:"description"` + StoreID string `json:"store_id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type SkillList struct { + Items []Skill `json:"items"` + Pagination apiclient.Pagination `json:"pagination"` +} + +type CreateSkillInput struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + TemplateID string `json:"template_id,omitempty"` +} + +type File struct { + Object string `json:"object"` + Path string `json:"path"` + Kind string `json:"kind"` + Size int64 `json:"size"` + Etag string `json:"etag"` + SkillID string `json:"skill_id"` + StoreItemID string `json:"store_item_id"` + DocumentID *string `json:"document_id"` + FileID *string `json:"file_id"` + UpdatedAt time.Time `json:"updated_at"` +} + +type FileList struct { + Items []File `json:"items"` +} + +type FileContent struct { + File + Content *string `json:"content"` + DownloadURL *string `json:"download_url"` +} + +type FileChange struct { + Object string `json:"object"` + Type string `json:"type"` + Path string `json:"path"` + PreviousPath *string `json:"previous_path"` + File *File `json:"file"` +} + +type FileChangeList struct { + Items []FileChange `json:"items"` +} + +// IfMatch: omitted skips the check, null requires an empty path, a string is the required etag. +type ChangeInput struct { + Type string `json:"type,omitempty"` + Path string `json:"path"` + ToPath string `json:"to_path,omitempty"` + Content *string `json:"content,omitempty"` + FileID string `json:"file_id,omitempty"` + IfMatch json.RawMessage `json:"if_match,omitempty"` +} + +type UploadedFile struct { + Object string `json:"object"` + ID string `json:"id"` +} diff --git a/internal/skillsync/ignore.go b/internal/skillsync/ignore.go new file mode 100644 index 0000000..946c8fe --- /dev/null +++ b/internal/skillsync/ignore.go @@ -0,0 +1,107 @@ +package skillsync + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" +) + +const ( + ignoreFile = ".metorialignore" + maxIgnorePatterns = 30 +) + +var ignoredNames = map[string]bool{ + manifestDir: true, + ".git": true, + ".DS_Store": true, + "node_modules": true, + ".idea": true, + ".vscode": true, + ".env": true, +} + +var ignoredNamePatterns = []string{".metorial-*", ".env.*", "*.swp", "*~"} + +// Patterns with a slash are anchored to the skill root and also match folder prefixes. +type ignoreRules struct { + names []string + paths []string +} + +// Over the limit is an error: dropping rules could push or delete ignored files. +func loadIgnoreRules(dir string) (ignoreRules, error) { + content, err := os.ReadFile(filepath.Join(dir, ignoreFile)) + if errors.Is(err, os.ErrNotExist) { + return ignoreRules{}, nil + } + if err != nil { + return ignoreRules{}, fmt.Errorf("metorial: failed to read %s: %w", ignoreFile, err) + } + + rules := ignoreRules{} + scanner := bufio.NewScanner(bytes.NewReader(content)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + pattern := strings.TrimRight(line, "/") + switch { + case pattern == "": + continue + case strings.Contains(pattern, "/"): + rules.paths = append(rules.paths, strings.TrimPrefix(pattern, "/")) + default: + rules.names = append(rules.names, pattern) + } + } + if count := len(rules.names) + len(rules.paths); count > maxIgnorePatterns { + return ignoreRules{}, fmt.Errorf("metorial: %s has %d patterns; the limit is %d\nCombine some of them with wildcards, or remove the ones you no longer need", ignoreFile, count, maxIgnorePatterns) + } + + return rules, nil +} + +func ignoredSegment(name string) bool { + if ignoredNames[name] { + return true + } + for _, pattern := range ignoredNamePatterns { + if matched, _ := path.Match(pattern, name); matched { + return true + } + } + return false +} + +func (r ignoreRules) matches(remotePath string) bool { + relative := strings.TrimPrefix(remotePath, "/") + segments := strings.Split(relative, "/") + + for index, segment := range segments { + if ignoredSegment(segment) { + return true + } + for _, pattern := range r.names { + if matched, _ := path.Match(pattern, segment); matched { + return true + } + } + + prefix := strings.Join(segments[:index+1], "/") + for _, pattern := range r.paths { + if matched, _ := path.Match(pattern, prefix); matched { + return true + } + } + } + + return false +} diff --git a/internal/skillsync/manifest.go b/internal/skillsync/manifest.go new file mode 100644 index 0000000..3c05460 --- /dev/null +++ b/internal/skillsync/manifest.go @@ -0,0 +1,148 @@ +package skillsync + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/metorial/cli/internal/skillsapi" +) + +const ( + manifestDir = ".metorial" + manifestFile = "skill.json" +) + +type Manifest struct { + SkillID string `json:"skill_id"` + SkillSlug string `json:"skill_slug,omitempty"` + APIHost string `json:"api_host,omitempty"` + Files map[string]ManifestFile `json:"files"` +} + +type ManifestFile struct { + Etag string `json:"etag"` + SHA256 string `json:"sha256"` +} + +var ErrNoManifest = errors.New("metorial: this directory is not a pulled skill (no .metorial/skill.json)\nRun the skills pull command first") + +func manifestPath(dir string) string { + return filepath.Join(dir, manifestDir, manifestFile) +} + +func LoadManifest(dir string) (*Manifest, error) { + content, err := os.ReadFile(manifestPath(dir)) + if errors.Is(err, os.ErrNotExist) { + return nil, ErrNoManifest + } + if err != nil { + return nil, fmt.Errorf("metorial: failed to read skill manifest: %w", err) + } + + manifest := &Manifest{} + if err := json.Unmarshal(content, manifest); err != nil { + return nil, fmt.Errorf("metorial: the skill manifest at %s is invalid: %w", manifestPath(dir), err) + } + if manifest.Files == nil { + manifest.Files = map[string]ManifestFile{} + } + + return manifest, nil +} + +func loadManifestFor(client *skillsapi.Client, dir string) (*Manifest, error) { + manifest, err := LoadManifest(dir) + if err == nil && manifest.APIHost != "" && manifest.APIHost != client.APIHost() { + return nil, fmt.Errorf("metorial: %s was pulled from %s, but the current profile uses %s", dir, manifest.APIHost, client.APIHost()) + } + return manifest, err +} + +func SaveManifest(dir string, manifest *Manifest) error { + content, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return fmt.Errorf("metorial: failed to write skill manifest: %w", err) + } + + if err := os.MkdirAll(filepath.Join(dir, manifestDir), 0o755); err != nil { + return fmt.Errorf("metorial: failed to create %s: %w", manifestDir, err) + } + + if err := writeFileAtomic(manifestPath(dir), append(content, '\n')); err != nil { + return fmt.Errorf("metorial: failed to write skill manifest: %w", err) + } + return nil +} + +func hashBytes(content []byte) string { + sum := sha256.Sum256(content) + return hex.EncodeToString(sum[:]) +} + +func localPath(dir string, remotePath string) (string, error) { + if strings.ContainsAny(remotePath, `\:`) { + return "", fmt.Errorf("metorial: refusing to sync %q: skill paths cannot contain \\ or :", remotePath) + } + + target := filepath.Join(dir, filepath.FromSlash(strings.TrimPrefix(remotePath, "/"))) + relative, err := filepath.Rel(dir, target) + if err != nil || relative == "." || relative == ".." || filepath.IsAbs(relative) || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("metorial: refusing to sync %q: it points outside of %s", remotePath, dir) + } + + return target, nil +} + +func remotePath(dir string, local string) (string, error) { + relative, err := filepath.Rel(dir, local) + if err != nil { + return "", err + } + + return "/" + filepath.ToSlash(relative), nil +} + +// Refuses symlinks: status and push follow links, so replacing one detaches the local copy. +func writeFileAtomic(target string, content []byte) error { + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return err + } + + mode := fs.FileMode(0o644) + if info, err := os.Lstat(target); err == nil { + if info.Mode()&fs.ModeSymlink != 0 { + return errors.New("it is a symbolic link, and replacing it would break the link; replace the link with a regular file or remove it, then try again") + } + mode = info.Mode().Perm() + } + + temp, err := os.CreateTemp(filepath.Dir(target), ".metorial-*") + if err != nil { + return err + } + tempName := temp.Name() + + if err := temp.Chmod(mode); err != nil { + _ = temp.Close() + _ = os.Remove(tempName) + return err + } + if _, err := temp.Write(content); err != nil { + _ = temp.Close() + _ = os.Remove(tempName) + return err + } + if err := temp.Close(); err != nil { + _ = os.Remove(tempName) + return err + } + + return os.Rename(tempName, target) +} diff --git a/internal/skillsync/sync.go b/internal/skillsync/sync.go new file mode 100644 index 0000000..8d02278 --- /dev/null +++ b/internal/skillsync/sync.go @@ -0,0 +1,716 @@ +package skillsync + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/metorial/cli/internal/skillsapi" +) + +const ( + maxBatchChanges = 100 + skillFilePath = "/SKILL.md" +) + +var validPathSegment = regexp.MustCompile(`^[A-Za-z0-9._() $-]+$`) + +type ChangeKind string + +const ( + ChangeAdded ChangeKind = "added" + ChangeModified ChangeKind = "modified" + ChangeDeleted ChangeKind = "deleted" +) + +type Change struct { + Path string `json:"path"` + Kind ChangeKind `json:"kind"` +} + +type Status struct { + SkillID string `json:"skill_id"` + SkillSlug string `json:"skill_slug,omitempty"` + Local []Change `json:"local"` + Remote []Change `json:"remote"` +} + +func (s Status) Conflicts() []string { + remote := map[string]bool{} + for _, change := range s.Remote { + remote[change.Path] = true + } + + conflicts := []string{} + for _, change := range s.Local { + if remote[change.Path] { + conflicts = append(conflicts, change.Path) + } + } + return conflicts +} + +func (s Status) Conflicted() bool { + return len(s.Conflicts()) > 0 +} + +func (s Status) Clean() bool { + return len(s.Local) == 0 && len(s.Remote) == 0 +} + +type PullOptions struct { + Force bool +} + +type PullResult struct { + Skill skillsapi.Skill `json:"skill"` + Dir string `json:"dir"` + Written []string `json:"written"` + Removed []string `json:"removed"` + Skipped []string `json:"skipped"` +} + +type PushOptions struct { + Force bool + DryRun bool +} + +type PushResult struct { + SkillID string `json:"skill_id"` + DryRun bool `json:"dry_run"` + Changes []Change `json:"changes"` +} + +var ErrRemoteChanged = errors.New("metorial: the skill was changed remotely since your last pull\nRun the skills pull command to update your copy, or push with --force to overwrite the remote changes") + +// Stale files are removed first so case-only renames or file/folder swaps can't clobber new writes. +// A partial failure still updates the manifest so a retry doesn't see local changes. +func Pull(client *skillsapi.Client, skill *skillsapi.Skill, dir string, options PullOptions) (*PullResult, error) { + root, err := resolveDir(dir) + if err != nil { + return nil, err + } + + previous, err := LoadManifest(root) + if err != nil && !errors.Is(err, ErrNoManifest) { + return nil, err + } + + if previous != nil { + if previous.SkillID != skill.ID { + return nil, fmt.Errorf("metorial: %s already contains skill %s; pull %s into a different directory", dir, previous.SkillID, skill.ID) + } + + local, err := localChanges(root, previous) + if err != nil { + return nil, err + } + if len(local) > 0 && !options.Force { + return nil, fmt.Errorf("metorial: %s has %d local change(s) that a pull would overwrite\nPush them first, or pull with --force to discard them", dir, len(local)) + } + } else if !options.Force { + if err := assertEmptyDir(root); err != nil { + return nil, err + } + } + + rules, err := loadIgnoreRules(root) + if err != nil { + return nil, err + } + + files, err := client.ListFiles(skill.ID) + if err != nil { + return nil, err + } + + manifest := &Manifest{ + SkillID: skill.ID, + APIHost: client.APIHost(), + Files: map[string]ManifestFile{}, + } + if skill.Slug != nil { + manifest.SkillSlug = *skill.Slug + } + + result := &PullResult{Skill: *skill, Dir: dir, Written: []string{}, Removed: []string{}, Skipped: []string{}} + + targets := map[string]string{} + remotePaths := []string{} + for _, file := range files.Items { + if rules.matches(file.Path) { + result.Skipped = append(result.Skipped, file.Path) + continue + } + + target, err := localPath(root, file.Path) + if err != nil { + return nil, err + } + targets[file.Path] = target + remotePaths = append(remotePaths, file.Path) + } + if err := assertNoCaseCollisions(remotePaths); err != nil { + return nil, err + } + if _, remoteIgnore := targets["/"+ignoreFile]; remoteIgnore && previous == nil && !options.Force { + if _, err := os.Stat(filepath.Join(root, ignoreFile)); err == nil { + return nil, fmt.Errorf("metorial: the skill has its own %s, which would replace the one in %s\nMove yours aside and pull again, or pull with --force to replace it", ignoreFile, dir) + } + } + + removals := []string{} + if previous != nil { + for trackedPath, tracked := range previous.Files { + if rules.matches(trackedPath) { + continue + } + manifest.Files[trackedPath] = tracked + if _, stillRemote := targets[trackedPath]; !stillRemote { + removals = append(removals, trackedPath) + } + } + } + sort.Strings(removals) + + changed := false + fail := func(cause error) (*PullResult, error) { + if !changed { + return nil, cause + } + if err := SaveManifest(root, manifest); err != nil { + return nil, errors.Join(cause, err) + } + return nil, cause + } + + for _, trackedPath := range removals { + target, err := localPath(root, trackedPath) + if err != nil { + return fail(err) + } + if err := removeFile(root, target); err != nil { + return fail(fmt.Errorf("metorial: failed to remove %s: %w", trackedPath, err)) + } + + delete(manifest.Files, trackedPath) + result.Removed = append(result.Removed, trackedPath) + changed = true + } + + for _, file := range files.Items { + target, ok := targets[file.Path] + if !ok { + continue + } + + remote, content, err := client.ReadFile(skill.ID, file.Path) + if err != nil { + return fail(err) + } + + content = skillsapi.RestoreTrailingNewline(remote.Kind, content) + + if !unchangedSymlink(target, content) { + if err := writeFileAtomic(target, content); err != nil { + return fail(fmt.Errorf("metorial: failed to write %s: %w", file.Path, err)) + } + } + + manifest.Files[file.Path] = ManifestFile{Etag: remote.Etag, SHA256: hashBytes(content)} + result.Written = append(result.Written, file.Path) + changed = true + } + + return result, SaveManifest(root, manifest) +} + +func assertNoCaseCollisions(remotePaths []string) error { + seen := map[string]string{} + for _, remotePath := range remotePaths { + segments := strings.Split(strings.TrimPrefix(remotePath, "/"), "/") + for index := range segments { + prefix := "/" + strings.Join(segments[:index+1], "/") + lower := strings.ToLower(prefix) + if other, ok := seen[lower]; ok && other != prefix { + return fmt.Errorf("metorial: %s and %s differ only in letter case and would overwrite each other on case-insensitive file systems\nRename one of them in the skill before pulling", other, prefix) + } + seen[lower] = prefix + } + } + return nil +} + +// Prunes emptied folders so a remote folder-to-file change can be written. +func removeFile(root string, target string) error { + if err := os.Remove(target); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + for dir := filepath.Dir(target); strings.HasPrefix(dir, root+string(filepath.Separator)); dir = filepath.Dir(dir) { + if os.Remove(dir) != nil { + break + } + } + return nil +} + +func unchangedSymlink(target string, content []byte) bool { + info, err := os.Lstat(target) + if err != nil || info.Mode()&fs.ModeSymlink == 0 { + return false + } + + existing, err := os.ReadFile(target) + return err == nil && bytes.Equal(existing, content) +} + +func localChanges(dir string, manifest *Manifest) ([]Change, error) { + rules, err := loadIgnoreRules(dir) + if err != nil { + return nil, err + } + + localFiles, err := scanLocalFiles(dir, rules) + if err != nil { + return nil, err + } + + changes := []Change{} + for filePath, hash := range localFiles { + tracked, ok := manifest.Files[filePath] + if !ok { + changes = append(changes, Change{Path: filePath, Kind: ChangeAdded}) + continue + } + if tracked.SHA256 != hash { + changes = append(changes, Change{Path: filePath, Kind: ChangeModified}) + } + } + for filePath := range manifest.Files { + if _, ok := localFiles[filePath]; !ok && !rules.matches(filePath) { + changes = append(changes, Change{Path: filePath, Kind: ChangeDeleted}) + } + } + + sortChanges(changes) + return changes, nil +} + +func remoteChanges(client *skillsapi.Client, dir string, manifest *Manifest) ([]Change, error) { + rules, err := loadIgnoreRules(dir) + if err != nil { + return nil, err + } + + remote, err := remoteEtags(client, manifest.SkillID) + if err != nil { + return nil, err + } + + for filePath := range remote { + if rules.matches(filePath) { + delete(remote, filePath) + } + } + + changes := []Change{} + for filePath, etag := range remote { + tracked, ok := manifest.Files[filePath] + if !ok { + changes = append(changes, Change{Path: filePath, Kind: ChangeAdded}) + continue + } + if tracked.Etag != etag { + changes = append(changes, Change{Path: filePath, Kind: ChangeModified}) + } + } + for filePath := range manifest.Files { + if _, ok := remote[filePath]; !ok && !rules.matches(filePath) { + changes = append(changes, Change{Path: filePath, Kind: ChangeDeleted}) + } + } + + sortChanges(changes) + return changes, nil +} + +func GetStatus(client *skillsapi.Client, dir string) (*Status, error) { + root, err := resolveDir(dir) + if err != nil { + return nil, err + } + + manifest, err := loadManifestFor(client, root) + if err != nil { + return nil, err + } + + local, err := localChanges(root, manifest) + if err != nil { + return nil, err + } + + remote, err := remoteChanges(client, root, manifest) + if err != nil { + return nil, err + } + + return &Status{SkillID: manifest.SkillID, SkillSlug: manifest.SkillSlug, Local: local, Remote: remote}, nil +} + +// The server stops a batch at the first failure but keeps earlier changes, so each batch is +// pre-checked for remote changes and etag-guarded, and partial results are recorded. +func Push(client *skillsapi.Client, dir string, options PushOptions) (*PushResult, error) { + root, err := resolveDir(dir) + if err != nil { + return nil, err + } + + manifest, err := loadManifestFor(client, root) + if err != nil { + return nil, err + } + + changes, err := localChanges(root, manifest) + if err != nil { + return nil, err + } + + if err := assertValidPaths(changes); err != nil { + return nil, err + } + if err := assertKeepsSkillFile(changes); err != nil { + return nil, err + } + if err := assertValidContent(root, changes); err != nil { + return nil, err + } + + result := &PushResult{SkillID: manifest.SkillID, DryRun: options.DryRun, Changes: changes} + if options.DryRun || len(changes) == 0 { + return result, nil + } + + pushed := 0 + for start := 0; start < len(changes); start += maxBatchChanges { + batch := changes[start:min(start+maxBatchChanges, len(changes))] + + before, err := remoteEtags(client, manifest.SkillID) + if err != nil { + return nil, partialPushError(err, pushed, len(changes)) + } + if !options.Force { + if err := assertRemoteUnchanged(manifest, batch, before); err != nil { + return nil, partialPushError(err, pushed, len(changes)) + } + } + + inputs, hashes, err := buildChangeInputs(client, root, manifest, batch, options.Force) + if err != nil { + return nil, partialPushError(err, pushed, len(changes)) + } + + applied, err := client.ApplyChanges(manifest.SkillID, inputs) + if err != nil { + return nil, recordPartialPush(client, root, manifest, batch, hashes, before, pushed, len(changes), err) + } + + for index, change := range batch { + if change.Kind == ChangeDeleted { + delete(manifest.Files, change.Path) + continue + } + + etag := "" + if index < len(applied.Items) && applied.Items[index].File != nil { + etag = applied.Items[index].File.Etag + } + manifest.Files[change.Path] = ManifestFile{Etag: etag, SHA256: hashes[change.Path]} + } + + pushed += len(batch) + if err := SaveManifest(root, manifest); err != nil { + return nil, partialPushError(err, pushed, len(changes)) + } + } + + return result, nil +} + +func remoteEtags(client *skillsapi.Client, skillID string) (map[string]string, error) { + files, err := client.ListFiles(skillID) + if err != nil { + return nil, err + } + + etags := map[string]string{} + for _, file := range files.Items { + etags[file.Path] = file.Etag + } + return etags, nil +} + +func assertRemoteUnchanged(manifest *Manifest, batch []Change, remote map[string]string) error { + for _, change := range batch { + etag, exists := remote[change.Path] + tracked, isTracked := manifest.Files[change.Path] + if isTracked && (!exists || etag != tracked.Etag) || !isTracked && exists { + return ErrRemoteChanged + } + } + return nil +} + +func assertKeepsSkillFile(changes []Change) error { + for _, change := range changes { + if change.Kind == ChangeDeleted && change.Path == skillFilePath { + return fmt.Errorf("metorial: this push would delete %s, which every skill needs\nRestore the file, or pull again to get the remote copy back", skillFilePath) + } + } + return nil +} + +func partialPushError(cause error, applied int, total int) error { + if applied == 0 { + return cause + } + return fmt.Errorf("%w\nThe push stopped partway: %d of %d change(s) were applied; run the skills status command to see what is left", cause, applied, total) +} + +// Records what a failed batch applied anyway; documents may have lost their trailing newline. +func recordPartialPush(client *skillsapi.Client, dir string, manifest *Manifest, batch []Change, hashes map[string]string, before map[string]string, pushed int, total int, cause error) error { + if skillsapi.IsConflict(cause) { + cause = ErrRemoteChanged + } + + after, err := remoteEtags(client, manifest.SkillID) + if err != nil { + return fmt.Errorf("%w\nPart of the push may have been applied; run the skills status command before pushing again", cause) + } + + applied := 0 + for _, change := range batch { + etag, exists := after[change.Path] + if change.Kind == ChangeDeleted { + if !exists { + delete(manifest.Files, change.Path) + applied++ + } + continue + } + + if previous, existed := before[change.Path]; !exists || existed && previous == etag { + continue + } + if remoteMatches(client, manifest.SkillID, change.Path, hashes[change.Path]) { + manifest.Files[change.Path] = ManifestFile{Etag: etag, SHA256: hashes[change.Path]} + applied++ + } + } + + if err := SaveManifest(dir, manifest); err != nil { + return err + } + + return partialPushError(cause, pushed+applied, total) +} + +func remoteMatches(client *skillsapi.Client, skillID string, filePath string, hash string) bool { + remote, content, err := client.ReadFile(skillID, filePath) + if err != nil { + return false + } + return hashBytes(content) == hash || hashBytes(skillsapi.RestoreTrailingNewline(remote.Kind, content)) == hash +} + +func buildChangeInputs(client *skillsapi.Client, dir string, manifest *Manifest, changes []Change, force bool) ([]skillsapi.ChangeInput, map[string]string, error) { + inputs := make([]skillsapi.ChangeInput, 0, len(changes)) + hashes := map[string]string{} + + for _, change := range changes { + input := skillsapi.ChangeInput{Path: change.Path} + + if !force { + if tracked, ok := manifest.Files[change.Path]; ok { + input.IfMatch = skillsapi.MatchEtag(tracked.Etag) + } else { + input.IfMatch = skillsapi.MatchAbsent() + } + } + + if change.Kind == ChangeDeleted { + input.Type = skillsapi.ChangeTypeDelete + inputs = append(inputs, input) + continue + } + + target, err := localPath(dir, change.Path) + if err != nil { + return nil, nil, err + } + content, err := os.ReadFile(target) + if err != nil { + return nil, nil, fmt.Errorf("metorial: failed to read %s: %w", change.Path, err) + } + + input.Type = skillsapi.ChangeTypePut + hashes[change.Path] = hashBytes(content) + + isText, err := skillsapi.IsText(change.Path, content) + if err != nil { + return nil, nil, err + } + if isText { + text := string(content) + input.Content = &text + } else { + uploaded, err := client.UploadFile(path.Base(change.Path), content) + if err != nil { + return nil, nil, fmt.Errorf("metorial: failed to upload %s: %w", change.Path, err) + } + input.FileID = uploaded.ID + } + + inputs = append(inputs, input) + } + + return inputs, hashes, nil +} + +func resolveDir(dir string) (string, error) { + absolute, err := filepath.Abs(dir) + if err != nil { + return "", fmt.Errorf("metorial: failed to resolve %s: %w", dir, err) + } + + resolved, err := filepath.EvalSymlinks(absolute) + if errors.Is(err, os.ErrNotExist) { + return absolute, nil + } + if err != nil { + return "", fmt.Errorf("metorial: failed to resolve %s: %w", dir, err) + } + return resolved, nil +} + +// Non-file symlinks are an error: skipping them would make them look deleted. +func scanLocalFiles(dir string, rules ignoreRules) (map[string]string, error) { + files := map[string]string{} + + err := filepath.WalkDir(dir, func(current string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if current == dir { + return nil + } + + remote, err := remotePath(dir, current) + if err != nil { + return err + } + + if rules.matches(remote) { + if entry.IsDir() { + return filepath.SkipDir + } + return nil + } + if entry.IsDir() { + return nil + } + if entry.Type()&fs.ModeSymlink != 0 { + if info, err := os.Stat(current); err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("%s is a symbolic link that does not point to a regular file; replace it with a file or remove it", remote) + } + } else if !entry.Type().IsRegular() { + return nil + } + + content, err := os.ReadFile(current) + if err != nil { + return err + } + + files[remote] = hashBytes(content) + return nil + }) + if err != nil { + return nil, fmt.Errorf("metorial: failed to scan %s: %w", dir, err) + } + + return files, nil +} + +func assertValidPaths(changes []Change) error { + invalid := []string{} + for _, change := range changes { + for _, segment := range strings.Split(strings.TrimPrefix(change.Path, "/"), "/") { + if !validPathSegment.MatchString(segment) || strings.Trim(segment, ".") == "" { + invalid = append(invalid, change.Path) + break + } + } + } + + if len(invalid) == 0 { + return nil + } + + return fmt.Errorf("metorial: these paths contain characters skills do not support (allowed: letters, numbers, spaces and . _ - ( ) $):\n %s", strings.Join(invalid, "\n ")) +} + +// Runs before the first batch so a rejected file can't stop a half-applied push. +func assertValidContent(dir string, changes []Change) error { + problems := []error{} + for _, change := range changes { + if change.Kind == ChangeDeleted { + continue + } + + target, err := localPath(dir, change.Path) + if err != nil { + return err + } + content, err := os.ReadFile(target) + if err != nil { + return fmt.Errorf("metorial: failed to read %s: %w", change.Path, err) + } + if _, err := skillsapi.IsText(change.Path, content); err != nil { + problems = append(problems, err) + } + } + return errors.Join(problems...) +} + +func assertEmptyDir(dir string) error { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + + for _, entry := range entries { + if entry.Name() != ignoreFile && !ignoredSegment(entry.Name()) { + return fmt.Errorf("metorial: %s is not empty; pull into an empty directory or pass --force", dir) + } + } + + return nil +} + +func sortChanges(changes []Change) { + sort.Slice(changes, func(i, j int) bool { + return changes[i].Path < changes[j].Path + }) +} diff --git a/internal/skillsync/sync_test.go b/internal/skillsync/sync_test.go new file mode 100644 index 0000000..34f1a25 --- /dev/null +++ b/internal/skillsync/sync_test.go @@ -0,0 +1,915 @@ +package skillsync + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/metorial/cli/internal/skillsapi/skillstest" +) + +func pullTestSkill(t *testing.T, fake *skillstest.Server) string { + t.Helper() + + client := fake.Client(t) + skill, err := client.ResolveSkill(skillstest.SkillSlug) + if err != nil { + t.Fatalf("ResolveSkill() error = %v", err) + } + + dir := filepath.Join(t.TempDir(), "skill") + if _, err := Pull(client, skill, dir, PullOptions{}); err != nil { + t.Fatalf("Pull() error = %v", err) + } + + return dir +} + +func writeLocal(t *testing.T, dir string, relative string, content string) { + t.Helper() + + target := filepath.Join(dir, filepath.FromSlash(relative)) + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(target, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestPullWritesFilesAndManifest(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/assets/logo.png", "\x89PNG\x00binary") + + dir := pullTestSkill(t, fake) + + skillContent, err := os.ReadFile(filepath.Join(dir, "SKILL.md")) + if err != nil || string(skillContent) != "# Release Notes\n" { + t.Fatalf("SKILL.md = %q, %v", skillContent, err) + } + + logo, err := os.ReadFile(filepath.Join(dir, "assets", "logo.png")) + if err != nil || string(logo) != "\x89PNG\x00binary" { + t.Fatalf("logo.png = %q, %v", logo, err) + } + + manifest, err := LoadManifest(dir) + if err != nil { + t.Fatalf("LoadManifest() error = %v", err) + } + if manifest.SkillID != skillstest.SkillID || len(manifest.Files) != 2 { + t.Fatalf("manifest = %+v", manifest) + } +} + +func TestPushRoundTrip(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/references/old.md", "old") + + dir := pullTestSkill(t, fake) + client := fake.Client(t) + + writeLocal(t, dir, "SKILL.md", "# Release Notes\n\nNew section.\n") + writeLocal(t, dir, "scripts/build.sh", "echo build\n") + writeLocal(t, dir, "assets/icon.png", "\x00\x01binary") + if err := os.Remove(filepath.Join(dir, "references", "old.md")); err != nil { + t.Fatal(err) + } + + result, err := Push(client, dir, PushOptions{}) + if err != nil { + t.Fatalf("Push() error = %v", err) + } + if len(result.Changes) != 4 { + t.Fatalf("changes = %+v", result.Changes) + } + + if content, _ := fake.Content("/SKILL.md"); content != "# Release Notes\n\nNew section." { + t.Fatalf("remote SKILL.md = %q", content) + } + if content, _ := fake.Content("/assets/icon.png"); content != "\x00\x01binary" { + t.Fatalf("remote icon.png = %q", content) + } + if fake.Uploads != 1 || len(fake.StorageErrors) != 0 { + t.Fatalf("uploads = %d, storage errors = %v; want icon.png uploaded through a signed URL", fake.Uploads, fake.StorageErrors) + } + if _, exists := fake.Content("/references/old.md"); exists { + t.Fatal("old.md should be deleted remotely") + } + + status, err := GetStatus(client, dir) + if err != nil { + t.Fatalf("GetStatus() error = %v", err) + } + if !status.Clean() { + t.Fatalf("status after push = %+v", status) + } +} + +func TestPushRejectsRemoteChanges(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + client := fake.Client(t) + + fake.SetFile("/SKILL.md", "# Edited in the dashboard\n") + writeLocal(t, dir, "SKILL.md", "# Edited locally\n") + + if _, err := Push(client, dir, PushOptions{}); !errors.Is(err, ErrRemoteChanged) { + t.Fatalf("Push() error = %v, want ErrRemoteChanged", err) + } + if content, _ := fake.Content("/SKILL.md"); content != "# Edited in the dashboard" { + t.Fatalf("remote SKILL.md was overwritten: %q", content) + } + if fake.Batches != 0 { + t.Fatalf("the remote change should be caught before sending, got %d batch(es)", fake.Batches) + } + + if _, err := Push(client, dir, PushOptions{Force: true}); err != nil { + t.Fatalf("Push(force) error = %v", err) + } + if content, _ := fake.Content("/SKILL.md"); content != "# Edited locally" { + t.Fatalf("remote SKILL.md after force = %q", content) + } +} + +func TestStatusReportsLocalAndRemoteChanges(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + + writeLocal(t, dir, "notes.txt", "local") + fake.SetFile("/references/new.md", "remote") + + status, err := GetStatus(fake.Client(t), dir) + if err != nil { + t.Fatalf("GetStatus() error = %v", err) + } + + if len(status.Local) != 1 || status.Local[0] != (Change{Path: "/notes.txt", Kind: ChangeAdded}) { + t.Fatalf("local = %+v", status.Local) + } + if len(status.Remote) != 1 || status.Remote[0] != (Change{Path: "/references/new.md", Kind: ChangeAdded}) { + t.Fatalf("remote = %+v", status.Remote) + } +} + +func TestPullRefusesToOverwriteLocalChanges(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + client := fake.Client(t) + writeLocal(t, dir, "SKILL.md", "unsaved work") + + skill, err := client.GetSkill(skillstest.SkillID) + if err != nil { + t.Fatal(err) + } + + if _, err := Pull(client, skill, dir, PullOptions{}); err == nil { + t.Fatal("Pull() should refuse to overwrite local changes") + } + + content, _ := os.ReadFile(filepath.Join(dir, "SKILL.md")) + if string(content) != "unsaved work" { + t.Fatalf("local SKILL.md was overwritten: %q", content) + } +} + +func TestPushRejectsUnsupportedPaths(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + writeLocal(t, dir, "bad*name.md", "x") + + if _, err := Push(fake.Client(t), dir, PushOptions{}); err == nil { + t.Fatal("Push() should reject unsupported path characters") + } + if fake.Batches != 0 { + t.Fatalf("no batch should be sent, got %d", fake.Batches) + } +} + +func TestPullRestoresDocumentTrailingNewline(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n\nLast line.\n") + + dir := pullTestSkill(t, fake) + + content, err := os.ReadFile(filepath.Join(dir, "SKILL.md")) + if err != nil || string(content) != "# Release Notes\n\nLast line.\n" { + t.Fatalf("SKILL.md = %q, %v", content, err) + } + + status, err := GetStatus(fake.Client(t), dir) + if err != nil || !status.Clean() { + t.Fatalf("status after pull = %+v, %v", status, err) + } +} + +func TestStatusReportsConflicts(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + fake.SetFile("/SKILL.md", "# Remote edit\n") + writeLocal(t, dir, "SKILL.md", "# Local edit\n") + + status, err := GetStatus(fake.Client(t), dir) + if err != nil { + t.Fatalf("GetStatus() error = %v", err) + } + if !status.Conflicted() || status.SkillSlug != skillstest.SkillSlug { + t.Fatalf("status = %+v", status) + } +} + +func TestStatusChangesOnDifferentPathsDoNotConflict(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + fake.SetFile("/references/new.md", "remote") + writeLocal(t, dir, "SKILL.md", "# Local edit\n") + + status, err := GetStatus(fake.Client(t), dir) + if err != nil { + t.Fatalf("GetStatus() error = %v", err) + } + if status.Conflicted() || len(status.Local) != 1 || len(status.Remote) != 1 { + t.Fatalf("status = %+v", status) + } + + if _, err := Push(fake.Client(t), dir, PushOptions{}); err != nil { + t.Fatalf("Push() error = %v", err) + } +} + +func TestPullPreservesFileModes(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/scripts/build.sh", "echo build\n") + + dir := pullTestSkill(t, fake) + script := filepath.Join(dir, "scripts", "build.sh") + + info, err := os.Stat(filepath.Join(dir, "SKILL.md")) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o644 { + t.Fatalf("SKILL.md mode = %v, want 0644", info.Mode().Perm()) + } + + if err := os.Chmod(script, 0o755); err != nil { + t.Fatal(err) + } + fake.SetFile("/scripts/build.sh", "echo rebuilt\n") + + client := fake.Client(t) + skill, err := client.GetSkill(skillstest.SkillID) + if err != nil { + t.Fatal(err) + } + if _, err := Pull(client, skill, dir, PullOptions{}); err != nil { + t.Fatalf("Pull() error = %v", err) + } + + info, err = os.Stat(script) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Fatalf("build.sh mode = %v, want 0755", info.Mode().Perm()) + } +} + +func TestStatusAndPushThroughSymlinkedDir(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/references/api.md", "api") + + dir := pullTestSkill(t, fake) + client := fake.Client(t) + + link := filepath.Join(t.TempDir(), "link") + if err := os.Symlink(dir, link); err != nil { + t.Fatal(err) + } + + status, err := GetStatus(client, link) + if err != nil || !status.Clean() { + t.Fatalf("status through symlink = %+v, %v", status, err) + } + + writeLocal(t, dir, "SKILL.md", "# Release Notes\n\nEdited.\n") + result, err := Push(client, link, PushOptions{}) + if err != nil { + t.Fatalf("Push() error = %v", err) + } + if len(result.Changes) != 1 || result.Changes[0] != (Change{Path: "/SKILL.md", Kind: ChangeModified}) { + t.Fatalf("changes = %+v", result.Changes) + } + if _, exists := fake.Content("/references/api.md"); !exists { + t.Fatal("api.md must not be deleted remotely") + } +} + +func TestStatusFollowsSymlinkedFiles(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/references/api.md", "api") + + dir := pullTestSkill(t, fake) + + target := filepath.Join(t.TempDir(), "api.md") + if err := os.WriteFile(target, []byte("api\n"), 0o644); err != nil { + t.Fatal(err) + } + apiPath := filepath.Join(dir, "references", "api.md") + if err := os.Remove(apiPath); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, apiPath); err != nil { + t.Fatal(err) + } + + status, err := GetStatus(fake.Client(t), dir) + if err != nil || !status.Clean() { + t.Fatalf("status with symlinked file = %+v, %v", status, err) + } + + if err := os.Remove(target); err != nil { + t.Fatal(err) + } + if _, err := GetStatus(fake.Client(t), dir); err == nil || !strings.Contains(err.Error(), "symbolic link") { + t.Fatalf("broken symlink error = %v", err) + } +} + +func TestPushRefusesToDeleteSkillFile(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/references/api.md", "api") + + dir := pullTestSkill(t, fake) + for _, name := range []string{"SKILL.md", "references/api.md"} { + if err := os.Remove(filepath.Join(dir, filepath.FromSlash(name))); err != nil { + t.Fatal(err) + } + } + if _, err := Push(fake.Client(t), dir, PushOptions{Force: true}); err == nil || !strings.Contains(err.Error(), "/SKILL.md") { + t.Fatalf("Push() error = %v, want a /SKILL.md error", err) + } + if fake.Batches != 0 { + t.Fatalf("no batch should be sent, got %d", fake.Batches) + } +} + +func TestPushChecksRemoteBeforeSendingBatch(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + writeLocal(t, dir, "notes.md", "local") + fake.SetFile("/notes.md", "remote") + + if _, err := Push(fake.Client(t), dir, PushOptions{}); !errors.Is(err, ErrRemoteChanged) { + t.Fatalf("Push() error = %v, want ErrRemoteChanged", err) + } + if fake.Batches != 0 { + t.Fatalf("no batch should be sent, got %d", fake.Batches) + } +} + +func TestPushRecordsPartiallyAppliedBatch(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + client := fake.Client(t) + + writeLocal(t, dir, "a.md", "a\n") + writeLocal(t, dir, "b.md", "b\n") + writeLocal(t, dir, "c.txt", "c\n") + fake.FailPath = "/b.md" + + _, err := Push(client, dir, PushOptions{}) + if err == nil || !strings.Contains(err.Error(), "1 of 3") { + t.Fatalf("Push() error = %v, want a partial push error", err) + } + + status, err := GetStatus(client, dir) + if err != nil { + t.Fatalf("GetStatus() error = %v", err) + } + want := []Change{{Path: "/b.md", Kind: ChangeAdded}, {Path: "/c.txt", Kind: ChangeAdded}} + if len(status.Local) != 2 || status.Local[0] != want[0] || status.Local[1] != want[1] || len(status.Remote) != 0 { + t.Fatalf("status after partial push = %+v", status) + } + + fake.FailPath = "" + if _, err := Push(client, dir, PushOptions{}); err != nil { + t.Fatalf("Push() retry error = %v", err) + } + if status, err := GetStatus(client, dir); err != nil || !status.Clean() { + t.Fatalf("status after retry = %+v, %v", status, err) + } +} + +func TestPullSkipsIgnoredRemotePaths(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/.git/config", "[core]") + fake.SetFile("/.env", "TOKEN=secret") + fake.SetFile("/docs/.DS_Store", "junk") + + client := fake.Client(t) + skill, err := client.GetSkill(skillstest.SkillID) + if err != nil { + t.Fatal(err) + } + + dir := filepath.Join(t.TempDir(), "skill") + result, err := Pull(client, skill, dir, PullOptions{}) + if err != nil { + t.Fatalf("Pull() error = %v", err) + } + if len(result.Written) != 1 || len(result.Skipped) != 3 { + t.Fatalf("pull result = %+v", result) + } + if _, err := os.Stat(filepath.Join(dir, ".env")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf(".env should not be written, stat error = %v", err) + } + + status, err := GetStatus(client, dir) + if err != nil || !status.Clean() { + t.Fatalf("status after pull = %+v, %v", status, err) + } + if pushed, err := Push(client, dir, PushOptions{}); err != nil || len(pushed.Changes) != 0 { + t.Fatalf("Push() = %+v, %v", pushed, err) + } + if _, exists := fake.Content("/.env"); !exists { + t.Fatal(".env must not be deleted remotely") + } +} + +func TestLocalChangesSkipIgnoredFiles(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + for _, name := range []string{ + ".env", ".env.local", "SKILL.md.swp", "SKILL.md~", ".metorial-123", ".vscode/settings.json", + "node_modules/pkg/index.js", "drafts/idea.md", "references/scratch.tmp", + } { + writeLocal(t, dir, name, "ignored") + } + writeLocal(t, dir, ".metorialignore", "# local only\ndrafts/\nreferences/*.tmp\n") + writeLocal(t, dir, "references/api.md", "api") + + manifest, err := LoadManifest(dir) + if err != nil { + t.Fatal(err) + } + changes, err := localChanges(dir, manifest) + if err != nil { + t.Fatalf("localChanges() error = %v", err) + } + + want := []Change{{Path: "/.metorialignore", Kind: ChangeAdded}, {Path: "/references/api.md", Kind: ChangeAdded}} + if len(changes) != 2 || changes[0] != want[0] || changes[1] != want[1] { + t.Fatalf("changes = %+v", changes) + } +} + +func TestPullRejectsUnsafeAndCollidingPaths(t *testing.T) { + for name, paths := range map[string][]string{ + "traversal": {"/../evil.md"}, + "backslash": {`/a\b.md`}, + "colon": {"/a:b.md"}, + "case": {"/Notes.md", "/notes.md"}, + "dir case": {"/Docs/a.md", "/docs/b.md"}, + } { + t.Run(name, func(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + for _, filePath := range paths { + fake.SetFile(filePath, "x") + } + + client := fake.Client(t) + skill, err := client.GetSkill(skillstest.SkillID) + if err != nil { + t.Fatal(err) + } + + parent := t.TempDir() + dir := filepath.Join(parent, "skill") + if _, err := Pull(client, skill, dir, PullOptions{}); err == nil { + t.Fatal("Pull() should fail") + } + if _, err := os.Stat(filepath.Join(dir, "SKILL.md")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("nothing should be written, stat error = %v", err) + } + if _, err := os.Stat(filepath.Join(parent, "evil.md")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("evil.md escaped the skill folder, stat error = %v", err) + } + }) + } +} + +func repull(t *testing.T, fake *skillstest.Server, dir string, options PullOptions) (*PullResult, error) { + t.Helper() + + client := fake.Client(t) + skill, err := client.GetSkill(skillstest.SkillID) + if err != nil { + t.Fatal(err) + } + + return Pull(client, skill, dir, options) +} + +func assertClean(t *testing.T, fake *skillstest.Server, dir string) { + t.Helper() + + status, err := GetStatus(fake.Client(t), dir) + if err != nil || !status.Clean() { + t.Fatalf("status = %+v, %v", status, err) + } +} + +func TestPullAppliesRemoteCaseOnlyRename(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/Notes.md", "notes\n") + + dir := pullTestSkill(t, fake) + fake.DeleteFile("/Notes.md") + fake.SetFile("/notes.md", "notes\n") + + result, err := repull(t, fake, dir, PullOptions{}) + if err != nil { + t.Fatalf("Pull() error = %v", err) + } + if len(result.Removed) != 1 || result.Removed[0] != "/Notes.md" { + t.Fatalf("removed = %v", result.Removed) + } + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + names := map[string]bool{} + for _, entry := range entries { + names[entry.Name()] = true + } + if !names["notes.md"] || names["Notes.md"] { + t.Fatalf("directory entries = %v, want notes.md only", names) + } + if content, err := os.ReadFile(filepath.Join(dir, "notes.md")); err != nil || string(content) != "notes\n" { + t.Fatalf("notes.md = %q, %v", content, err) + } + + assertClean(t, fake, dir) + if pushed, err := Push(fake.Client(t), dir, PushOptions{}); err != nil || len(pushed.Changes) != 0 { + t.Fatalf("Push() = %+v, %v", pushed, err) + } + if _, exists := fake.Content("/notes.md"); !exists { + t.Fatal("notes.md must not be deleted remotely") + } +} + +func TestPullReplacesFileWithFolderAndBack(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/docs", "plain file") + + dir := pullTestSkill(t, fake) + fake.DeleteFile("/docs") + fake.SetFile("/docs/a.md", "a\n") + + if _, err := repull(t, fake, dir, PullOptions{}); err != nil { + t.Fatalf("Pull() file to folder error = %v", err) + } + if content, err := os.ReadFile(filepath.Join(dir, "docs", "a.md")); err != nil || string(content) != "a\n" { + t.Fatalf("docs/a.md = %q, %v", content, err) + } + assertClean(t, fake, dir) + + fake.DeleteFile("/docs/a.md") + fake.SetFile("/docs", "file again") + + if _, err := repull(t, fake, dir, PullOptions{}); err != nil { + t.Fatalf("Pull() folder to file error = %v", err) + } + if content, err := os.ReadFile(filepath.Join(dir, "docs")); err != nil || string(content) != "file again" { + t.Fatalf("docs = %q, %v", content, err) + } + assertClean(t, fake, dir) +} + +func TestPullFailingPartwayRecordsWrittenFiles(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/a.md", "a\n") + fake.SetFile("/b.md", "b\n") + fake.FailReadPath = "/b.md" + + dir := filepath.Join(t.TempDir(), "skill") + if _, err := repull(t, fake, dir, PullOptions{}); err == nil { + t.Fatal("Pull() should fail") + } + + manifest, err := LoadManifest(dir) + if err != nil { + t.Fatalf("LoadManifest() error = %v", err) + } + if _, ok := manifest.Files["/a.md"]; !ok || len(manifest.Files) != 2 { + t.Fatalf("manifest files = %+v, want /SKILL.md and /a.md", manifest.Files) + } + + fake.FailReadPath = "" + if _, err := repull(t, fake, dir, PullOptions{}); err != nil { + t.Fatalf("Pull() retry error = %v", err) + } + assertClean(t, fake, dir) +} + +func TestPullRefusesToReplaceSymlinkedFile(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/references/api.md", "api") + + dir := pullTestSkill(t, fake) + + target := filepath.Join(t.TempDir(), "api.md") + if err := os.WriteFile(target, []byte("api\n"), 0o644); err != nil { + t.Fatal(err) + } + apiPath := filepath.Join(dir, "references", "api.md") + if err := os.Remove(apiPath); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, apiPath); err != nil { + t.Fatal(err) + } + + if _, err := repull(t, fake, dir, PullOptions{}); err != nil { + t.Fatalf("Pull() with an unchanged symlinked file error = %v", err) + } + + fake.SetFile("/references/api.md", "api v2") + if _, err := repull(t, fake, dir, PullOptions{}); err == nil || !strings.Contains(err.Error(), "symbolic link") { + t.Fatalf("Pull() error = %v, want a symbolic link error", err) + } + + if info, err := os.Lstat(apiPath); err != nil || info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("api.md should still be a symlink, info = %v, err = %v", info, err) + } + if content, _ := os.ReadFile(target); string(content) != "api\n" { + t.Fatalf("link target = %q, want it untouched", content) + } +} + +func TestIgnoreFileWithTooManyPatternsIsAnError(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + patterns := []string{} + for index := range 31 { + patterns = append(patterns, fmt.Sprintf("draft-%d", index)) + } + writeLocal(t, dir, ".metorialignore", strings.Join(patterns, "\n")+"\n") + + want := "has 31 patterns; the limit is 30" + if _, err := GetStatus(fake.Client(t), dir); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("GetStatus() error = %v", err) + } + if _, err := Push(fake.Client(t), dir, PushOptions{}); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("Push() error = %v", err) + } + if _, err := repull(t, fake, dir, PullOptions{Force: true}); err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("Pull() error = %v", err) + } +} + +func TestPullAllowsExistingIgnoreFile(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := filepath.Join(t.TempDir(), "skill") + writeLocal(t, dir, ".metorialignore", "drafts/\n") + + if _, err := repull(t, fake, dir, PullOptions{}); err != nil { + t.Fatalf("Pull() error = %v", err) + } + status, err := GetStatus(fake.Client(t), dir) + if err != nil { + t.Fatalf("GetStatus() error = %v", err) + } + if len(status.Local) != 1 || status.Local[0] != (Change{Path: "/.metorialignore", Kind: ChangeAdded}) { + t.Fatalf("local = %+v, want .metorialignore to be pushed like any other file", status.Local) + } + + other := skillstest.NewServer(t) + other.SetFile("/SKILL.md", "# Release Notes\n") + other.SetFile("/.metorialignore", "scratch/\n") + + dir = filepath.Join(t.TempDir(), "skill") + writeLocal(t, dir, ".metorialignore", "drafts/\n") + if _, err := repull(t, other, dir, PullOptions{}); err == nil || !strings.Contains(err.Error(), ".metorialignore") { + t.Fatalf("Pull() error = %v, want the remote .metorialignore to be protected", err) + } + if content, _ := os.ReadFile(filepath.Join(dir, ".metorialignore")); string(content) != "drafts/\n" { + t.Fatalf("local .metorialignore = %q, want it untouched", content) + } +} + +func TestPushRejectsInvalidUTF8Markdown(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + writeLocal(t, dir, "notes.md", "caf\xe9\n") + + if _, err := Push(fake.Client(t), dir, PushOptions{}); err == nil || !strings.Contains(err.Error(), "UTF-8") { + t.Fatalf("Push() error = %v, want a UTF-8 error", err) + } + if fake.Batches != 0 { + t.Fatalf("no batch should be sent, got %d", fake.Batches) + } +} + +func TestPushRecordsPartiallyAppliedDocumentWithoutTrailingNewline(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + client := fake.Client(t) + + writeLocal(t, dir, "a.md", "a") + writeLocal(t, dir, "b.md", "b\n") + writeLocal(t, dir, "c.txt", "c\n") + fake.FailPath = "/b.md" + + if _, err := Push(client, dir, PushOptions{}); err == nil || !strings.Contains(err.Error(), "1 of 3") { + t.Fatalf("Push() error = %v, want a partial push error", err) + } + + status, err := GetStatus(client, dir) + if err != nil { + t.Fatalf("GetStatus() error = %v", err) + } + want := []Change{{Path: "/b.md", Kind: ChangeAdded}, {Path: "/c.txt", Kind: ChangeAdded}} + if len(status.Local) != 2 || status.Local[0] != want[0] || status.Local[1] != want[1] || len(status.Remote) != 0 { + t.Fatalf("status after partial push = %+v", status) + } +} + +func TestPushReportsEarlierBatchesWhenLaterBatchFails(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + for index := range maxBatchChanges { + writeLocal(t, dir, fmt.Sprintf("references/f%03d.md", index), "x\n") + } + writeLocal(t, dir, "z.bin", "\x00binary") + fake.FailUploads = true + + _, err := Push(fake.Client(t), dir, PushOptions{}) + if err == nil || !strings.Contains(err.Error(), "100 of 101") { + t.Fatalf("Push() error = %v, want the earlier batch to be reported", err) + } + if fake.Batches != 1 { + t.Fatalf("batches = %d, want 1", fake.Batches) + } + + status, err := GetStatus(fake.Client(t), dir) + if err != nil { + t.Fatalf("GetStatus() error = %v", err) + } + if len(status.Local) != 1 || status.Local[0] != (Change{Path: "/z.bin", Kind: ChangeAdded}) { + t.Fatalf("local = %+v", status.Local) + } +} + +func TestIgnorePatternsMatchFoldersAndAnchors(t *testing.T) { + dir := t.TempDir() + writeLocal(t, dir, ".metorialignore", "docs/old\n/drafts/\nscratch\nreferences/*\n/\n") + + rules, err := loadIgnoreRules(dir) + if err != nil { + t.Fatalf("loadIgnoreRules() error = %v", err) + } + + for remotePath, want := range map[string]bool{ + "/docs/old": true, + "/docs/old/a.md": true, + "/docs/old/deep/b.md": true, + "/docs/older/a.md": false, + "/notes/docs/old/a.md": false, + "/drafts": true, + "/drafts/idea.md": true, + "/notes/drafts/idea.md": false, + "/scratch/a.md": true, + "/notes/scratch": true, + "/references/api.md": true, + "/references/sub/deep.md": true, + "/references.md": false, + "/SKILL.md": false, + } { + if got := rules.matches(remotePath); got != want { + t.Errorf("matches(%q) = %v, want %v", remotePath, got, want) + } + } +} + +func TestIgnoredFolderStaysOutOfPullStatusAndPush(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/.metorialignore", "docs/old\n") + fake.SetFile("/docs/old/a.md", "old\n") + fake.SetFile("/docs/keep.md", "keep\n") + + dir := filepath.Join(t.TempDir(), "skill") + writeLocal(t, dir, ".metorialignore", "docs/old\n") + + result, err := repull(t, fake, dir, PullOptions{Force: true}) + if err != nil { + t.Fatalf("Pull() error = %v", err) + } + if len(result.Skipped) != 1 || result.Skipped[0] != "/docs/old/a.md" { + t.Fatalf("skipped = %v, want /docs/old/a.md", result.Skipped) + } + if _, err := os.Stat(filepath.Join(dir, "docs", "old", "a.md")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("docs/old/a.md should not be written, stat error = %v", err) + } + assertClean(t, fake, dir) + + writeLocal(t, dir, "docs/old/local.md", "local only\n") + fake.SetFile("/docs/old/a.md", "old v2\n") + assertClean(t, fake, dir) + + if pushed, err := Push(fake.Client(t), dir, PushOptions{}); err != nil || len(pushed.Changes) != 0 { + t.Fatalf("Push() = %+v, %v", pushed, err) + } + if _, exists := fake.Content("/docs/old/a.md"); !exists { + t.Fatal("docs/old/a.md must not be deleted remotely") + } + if fake.Batches != 0 { + t.Fatalf("batches = %d, want 0", fake.Batches) + } +} + +func TestPullIntoCurrentDirectoryReplacesFolderWithFile(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + fake.SetFile("/docs/a.md", "a\n") + + t.Chdir(t.TempDir()) + if _, err := repull(t, fake, ".", PullOptions{}); err != nil { + t.Fatalf("Pull() error = %v", err) + } + + fake.DeleteFile("/docs/a.md") + fake.SetFile("/docs", "file now") + + if _, err := repull(t, fake, ".", PullOptions{}); err != nil { + t.Fatalf("Pull() folder to file error = %v", err) + } + if content, err := os.ReadFile("docs"); err != nil || string(content) != "file now" { + t.Fatalf("docs = %q, %v", content, err) + } + assertClean(t, fake, ".") +} + +func TestPushValidatesEveryBatchBeforeSending(t *testing.T) { + fake := skillstest.NewServer(t) + fake.SetFile("/SKILL.md", "# Release Notes\n") + + dir := pullTestSkill(t, fake) + for index := range maxBatchChanges { + writeLocal(t, dir, fmt.Sprintf("references/f%03d.md", index), "x\n") + } + writeLocal(t, dir, "z.md", "caf\xe9\n") + + if _, err := Push(fake.Client(t), dir, PushOptions{}); err == nil || !strings.Contains(err.Error(), "UTF-8") { + t.Fatalf("Push() error = %v, want a UTF-8 error", err) + } + if fake.Batches != 0 { + t.Fatalf("batches = %d, want none before the invalid file is fixed", fake.Batches) + } +} diff --git a/manifest/admin.yaml b/manifest/admin.yaml index b3021a5..5f71e20 100644 --- a/manifest/admin.yaml +++ b/manifest/admin.yaml @@ -207,6 +207,9 @@ commands: - path: [skills, agents] resource: skills.agents + - path: [skills, files] + resource: skills.files + - path: [skills, participants] resource: skills.participants @@ -411,6 +414,8 @@ commands: - path: [integrations, instances] resource: integrations.instances + slateKeys: + create-session-template: integrations_instances_create_template - path: [integrations, setup-sessions] resource: integrations.setupSessions @@ -425,6 +430,7 @@ commands: create: type: { flag: type, required: false } instance_id: { flag: instance-id, required: false } + slateExclude: [update, revoke] - path: [outposts] resource: outposts