implement a bucket policy management system heavily "inspired" by aws s3

This commit is contained in:
Keiran 2025-08-07 18:42:34 +01:00
parent db776d34dd
commit b765dc6c88
6 changed files with 471 additions and 4 deletions

View File

@ -69,6 +69,83 @@ All API endpoints require `X-API-Key` header.
- `GET /api/v1/user` - Get user info and usage stats
**Bucket Policies:**
- `PUT /api/v1/buckets/:bucket/policy` - Set bucket policy `{"policy": "json-policy-string"}`
- `GET /api/v1/buckets/:bucket/policy` - Get bucket policy
- `DELETE /api/v1/buckets/:bucket/policy` - Delete bucket policy
## Bucket Policies
Bucket policies use JSON format similar to AWS S3 IAM policies to control access to buckets and objects.
### Policy Structure
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "StatementId",
"Effect": "Allow|Deny",
"Principal": {
"User": ["username1", "username2"]
},
"Action": [
"termcloud:GetObject",
"termcloud:PutObject",
"termcloud:DeleteObject",
"termcloud:ListObjects"
],
"Resource": [
"arn:termcloud:s3:::bucket-name/*"
]
}
]
}
```
### Supported Actions
- `termcloud:GetObject` - Download files
- `termcloud:PutObject` - Upload files
- `termcloud:DeleteObject` - Delete files
- `termcloud:ListObjects` - List files in bucket
- `termcloud:GetBucket` - Get bucket info
- `termcloud:DeleteBucket` - Delete bucket
- `*` - All actions
### Policy Examples
**Read-only access:**
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {"User": ["john"]},
"Action": ["termcloud:GetObject", "termcloud:ListObjects"],
"Resource": ["arn:termcloud:s3:::my-bucket/*"]
}
]
}
```
**Deny delete operations:**
```json
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Deny",
"Principal": {"User": ["*"]},
"Action": ["termcloud:DeleteObject"],
"Resource": ["arn:termcloud:s3:::my-bucket/*"]
}
]
}
```
### Example Usage
```bash

View File

@ -50,10 +50,17 @@ func main() {
api.GET("/buckets", h.ListBucketsHandler)
api.POST("/buckets", h.CreateBucketHandler)
api.DELETE("/buckets/:bucket", h.DeleteBucketHandler)
api.GET("/buckets/:bucket/objects", h.ListObjectsHandler)
api.PUT("/buckets/:bucket/objects/*", h.UploadObjectHandler)
api.GET("/buckets/:bucket/objects/*", h.GetObjectHandler)
api.DELETE("/buckets/:bucket/objects/*", h.DeleteObjectHandler)
api.PUT("/buckets/:bucket/policy", h.SetBucketPolicyHandler)
api.GET("/buckets/:bucket/policy", h.GetBucketPolicyHandler)
api.DELETE("/buckets/:bucket/policy", h.DeleteBucketPolicyHandler)
bucketRoutes := api.Group("/buckets/:bucket")
bucketRoutes.Use(h.PolicyEnforcementMiddleware)
bucketRoutes.GET("/objects", h.ListObjectsHandler)
bucketRoutes.PUT("/objects/*", h.UploadObjectHandler)
bucketRoutes.GET("/objects/*", h.GetObjectHandler)
bucketRoutes.DELETE("/objects/*", h.DeleteObjectHandler)
e.Logger.Fatal(e.Start(":" + cfg.Port))
}

View File

@ -0,0 +1,40 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowReadOnlyAccess",
"Effect": "Allow",
"Principal": {
"User": ["john", "jane"]
},
"Action": [
"termcloud:GetObject",
"termcloud:ListObjects"
],
"Resource": [
"arn:termcloud:s3:::my-bucket/*",
"arn:termcloud:s3:::my-bucket"
]
},
{
"Sid": "DenyDeleteOperations",
"Effect": "Deny",
"Principal": {
"User": ["*"]
},
"Action": [
"termcloud:DeleteObject",
"termcloud:DeleteBucket"
],
"Resource": [
"arn:termcloud:s3:::my-bucket/*",
"arn:termcloud:s3:::my-bucket"
],
"Condition": {
"StringNotEquals": {
"termcloud:username": "admin"
}
}
}
]
}

214
internal/db/policy.go Normal file
View File

@ -0,0 +1,214 @@
package db
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
)
type BucketPolicy struct {
Version string `json:"Version"`
Statement []PolicyStatement `json:"Statement"`
}
type PolicyStatement struct {
Sid string `json:"Sid,omitempty"`
Effect string `json:"Effect"`
Principal map[string]any `json:"Principal,omitempty"`
Action []string `json:"Action"`
Resource []string `json:"Resource"`
Condition map[string]any `json:"Condition,omitempty"`
}
type BucketPolicyRecord struct {
ID int64 `json:"id"`
BucketID int64 `json:"bucketId"`
Policy string `json:"policy"`
UpdatedAt time.Time `json:"updatedAt"`
}
func ValidateBucketPolicy(policyJSON string) (*BucketPolicy, error) {
var policy BucketPolicy
if err := json.Unmarshal([]byte(policyJSON), &policy); err != nil {
return nil, fmt.Errorf("invalid JSON format: %w", err)
}
if policy.Version == "" {
policy.Version = "2012-10-17"
}
if len(policy.Statement) == 0 {
return nil, fmt.Errorf("policy must contain at least one statement")
}
for i, stmt := range policy.Statement {
if err := validateStatement(stmt); err != nil {
return nil, fmt.Errorf("statement %d: %w", i, err)
}
}
return &policy, nil
}
func validateStatement(stmt PolicyStatement) error {
if stmt.Effect != "Allow" && stmt.Effect != "Deny" {
return fmt.Errorf("effect must be 'Allow' or 'Deny'")
}
if len(stmt.Action) == 0 {
return fmt.Errorf("statement must specify at least one action")
}
validActions := map[string]bool{
"termcloud:GetObject": true,
"termcloud:PutObject": true,
"termcloud:DeleteObject": true,
"termcloud:ListObjects": true,
"termcloud:GetBucket": true,
"termcloud:DeleteBucket": true,
"*": true,
}
for _, action := range stmt.Action {
if !validActions[action] && !strings.HasSuffix(action, "*") {
return fmt.Errorf("invalid action: %s", action)
}
}
if len(stmt.Resource) == 0 {
return fmt.Errorf("statement must specify at least one resource")
}
return nil
}
func (s *BucketService) SetBucketPolicy(ctx context.Context, bucketID int64, policyJSON string) error {
if _, err := ValidateBucketPolicy(policyJSON); err != nil {
return fmt.Errorf("invalid policy: %w", err)
}
_, err := s.pool.Exec(ctx, `
INSERT INTO bucket_policies (bucket_id, policy)
VALUES ($1, $2)
ON CONFLICT (bucket_id)
DO UPDATE SET policy = $2, updated_at = NOW()`,
bucketID, policyJSON)
if err != nil {
return fmt.Errorf("failed to set bucket policy: %w", err)
}
return nil
}
func (s *BucketService) GetBucketPolicy(ctx context.Context, bucketID int64) (*BucketPolicyRecord, error) {
var record BucketPolicyRecord
err := s.pool.QueryRow(ctx, `
SELECT id, bucket_id, policy, updated_at
FROM bucket_policies WHERE bucket_id = $1`,
bucketID).Scan(&record.ID, &record.BucketID, &record.Policy, &record.UpdatedAt)
if err != nil {
return nil, fmt.Errorf("failed to get bucket policy: %w", err)
}
return &record, nil
}
func (s *BucketService) DeleteBucketPolicy(ctx context.Context, bucketID int64) error {
_, err := s.pool.Exec(ctx, "DELETE FROM bucket_policies WHERE bucket_id = $1", bucketID)
if err != nil {
return fmt.Errorf("failed to delete bucket policy: %w", err)
}
return nil
}
func (s *BucketService) EvaluatePolicy(ctx context.Context, bucketID int64, action, resource, principal string) (bool, error) {
policyRecord, err := s.GetBucketPolicy(ctx, bucketID)
if err != nil {
if strings.Contains(err.Error(), "no rows") {
return true, nil
}
return false, err
}
policy, err := ValidateBucketPolicy(policyRecord.Policy)
if err != nil {
return false, fmt.Errorf("invalid stored policy: %w", err)
}
for _, stmt := range policy.Statement {
if matchesStatement(stmt, action, resource, principal) {
return stmt.Effect == "Allow", nil
}
}
return true, nil
}
func matchesStatement(stmt PolicyStatement, action, resource, principal string) bool {
if !matchesAction(stmt.Action, action) {
return false
}
if !matchesResource(stmt.Resource, resource) {
return false
}
if stmt.Principal != nil && !matchesPrincipal(stmt.Principal, principal) {
return false
}
return true
}
func matchesAction(allowedActions []string, action string) bool {
for _, allowed := range allowedActions {
if allowed == "*" || allowed == action {
return true
}
if strings.HasSuffix(allowed, "*") {
prefix := strings.TrimSuffix(allowed, "*")
if strings.HasPrefix(action, prefix) {
return true
}
}
}
return false
}
func matchesResource(allowedResources []string, resource string) bool {
for _, allowed := range allowedResources {
if allowed == "*" || allowed == resource {
return true
}
if strings.HasSuffix(allowed, "*") {
prefix := strings.TrimSuffix(allowed, "*")
if strings.HasPrefix(resource, prefix) {
return true
}
}
}
return false
}
func matchesPrincipal(allowedPrincipal map[string]any, principal string) bool {
if allowedPrincipal["*"] != nil {
return true
}
if users, ok := allowedPrincipal["User"].([]any); ok {
for _, user := range users {
if userStr, ok := user.(string); ok && userStr == principal {
return true
}
}
}
if userStr, ok := allowedPrincipal["User"].(string); ok && userStr == principal {
return true
}
return false
}

View File

@ -28,8 +28,16 @@ CREATE TABLE IF NOT EXISTS objects (
UNIQUE (bucket_id, key)
);
CREATE TABLE IF NOT EXISTS bucket_policies (
id SERIAL PRIMARY KEY,
bucket_id INT NOT NULL REFERENCES buckets(id) ON DELETE CASCADE UNIQUE,
policy JSONB NOT NULL,
updated_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_buckets_owner_id ON buckets (owner_id);
CREATE INDEX IF NOT EXISTS idx_users_api_key ON users (api_key);
CREATE INDEX IF NOT EXISTS idx_bucket_policies_bucket_id ON bucket_policies (bucket_id);
CREATE INDEX IF NOT EXISTS idx_objects_bucket_id_key ON objects (bucket_id, key);
CREATE INDEX IF NOT EXISTS idx_objects_bucket_id_last_modified ON objects (bucket_id, last_modified DESC);
CREATE INDEX IF NOT EXISTS idx_objects_custom_metadata_gin ON objects USING GIN (custom_metadata);

View File

@ -220,3 +220,124 @@ func (h *Handlers) GetUserInfoHandler(c echo.Context) error {
"bucketCount": len(buckets),
})
}
func (h *Handlers) SetBucketPolicyHandler(c echo.Context) error {
user := c.Get("user").(*db.User)
bucketName := c.Param("bucket")
bucket, err := h.bucketService.GetBucket(context.Background(), bucketName, user.ID)
if err != nil {
if strings.Contains(err.Error(), "no rows") {
return c.JSON(404, map[string]string{"error": "Bucket not found"})
}
return c.JSON(500, map[string]string{"error": "Failed to get bucket"})
}
var req struct {
Policy string `json:"policy"`
}
if err := c.Bind(&req); err != nil {
return c.JSON(400, map[string]string{"error": "Invalid request body"})
}
if err := h.bucketService.SetBucketPolicy(context.Background(), bucket.ID, req.Policy); err != nil {
if strings.Contains(err.Error(), "invalid policy") {
return c.JSON(400, map[string]string{"error": err.Error()})
}
return c.JSON(500, map[string]string{"error": "Failed to set bucket policy"})
}
return c.JSON(200, map[string]string{"message": "Bucket policy set successfully"})
}
func (h *Handlers) GetBucketPolicyHandler(c echo.Context) error {
user := c.Get("user").(*db.User)
bucketName := c.Param("bucket")
bucket, err := h.bucketService.GetBucket(context.Background(), bucketName, user.ID)
if err != nil {
if strings.Contains(err.Error(), "no rows") {
return c.JSON(404, map[string]string{"error": "Bucket not found"})
}
return c.JSON(500, map[string]string{"error": "Failed to get bucket"})
}
policyRecord, err := h.bucketService.GetBucketPolicy(context.Background(), bucket.ID)
if err != nil {
if strings.Contains(err.Error(), "no rows") {
return c.JSON(404, map[string]string{"error": "No policy found for bucket"})
}
return c.JSON(500, map[string]string{"error": "Failed to get bucket policy"})
}
return c.JSON(200, policyRecord)
}
func (h *Handlers) DeleteBucketPolicyHandler(c echo.Context) error {
user := c.Get("user").(*db.User)
bucketName := c.Param("bucket")
bucket, err := h.bucketService.GetBucket(context.Background(), bucketName, user.ID)
if err != nil {
if strings.Contains(err.Error(), "no rows") {
return c.JSON(404, map[string]string{"error": "Bucket not found"})
}
return c.JSON(500, map[string]string{"error": "Failed to get bucket"})
}
if err := h.bucketService.DeleteBucketPolicy(context.Background(), bucket.ID); err != nil {
return c.JSON(500, map[string]string{"error": "Failed to delete bucket policy"})
}
return c.JSON(200, map[string]string{"message": "Bucket policy deleted successfully"})
}
func (h *Handlers) PolicyEnforcementMiddleware(next echo.HandlerFunc) echo.HandlerFunc {
return func(c echo.Context) error {
user := c.Get("user").(*db.User)
bucketName := c.Param("bucket")
if bucketName == "" {
return next(c)
}
bucket, err := h.bucketService.GetBucket(context.Background(), bucketName, user.ID)
if err != nil {
return next(c)
}
action := mapHTTPMethodToAction(c.Request().Method, c.Path())
resource := fmt.Sprintf("arn:termcloud:s3:::%s/*", bucketName)
principal := user.Username
allowed, err := h.bucketService.EvaluatePolicy(context.Background(), bucket.ID, action, resource, principal)
if err != nil {
return c.JSON(500, map[string]string{"error": "Policy evaluation failed"})
}
if !allowed {
return c.JSON(403, map[string]string{"error": "Access denied by bucket policy"})
}
return next(c)
}
}
func mapHTTPMethodToAction(method, path string) string {
switch method {
case "GET":
if strings.Contains(path, "/objects") && !strings.HasSuffix(path, "/objects") {
return "termcloud:GetObject"
}
return "termcloud:ListObjects"
case "PUT":
return "termcloud:PutObject"
case "DELETE":
if strings.Contains(path, "/objects") {
return "termcloud:DeleteObject"
}
return "termcloud:DeleteBucket"
default:
return "termcloud:GetBucket"
}
}